@elizaos/ui 2.0.0-beta.2 → 2.0.3-beta.3

package/agent-surface/element-reporter.hooks.js

@@ -0,0 +1,79 @@
+/**
+ * Element-reporter internals — the payload builder and the subscribe/POST hook.
+ *
+ * Split out of element-reporter.tsx so that module exports only the
+ * `AgentSurfaceElementReporter` component: React Fast Refresh requires a
+ * component-only module to hot-reload without forcing a full page reload.
+ */
+import { useEffect } from "react";
+// The API client + url helper are loaded dynamically inside flush() (production
+// path only) so this module — re-exported from the widely-imported agent-surface
+// barrel and externalised into every plugin view bundle — stays import-light and
+// pulls no api/utils graph into its consumers.
+const REPORT_DEBOUNCE_MS = 400;
+export function buildPayload(registry) {
+    const snap = registry.snapshot();
+    const elements = snap.elements.map((e) => ({
+        id: e.id,
+        role: e.role,
+        label: e.label,
+        ...(typeof e.value === "string" ? { value: e.value } : {}),
+        ...(e.focused ? { focused: true } : {}),
+    }));
+    return { viewId: snap.viewId, elements };
+}
+/**
+ * Subscribe to a registry and push its element snapshot to the backend on mount
+ * and on every (debounced) change. Exported for direct/test use; the
+ * `AgentSurfaceElementReporter` component is the in-tree mount point.
+ */
+export function useAgentSurfaceElementReporter(registry) {
+    useEffect(() => {
+        // Inert without a registry, without fetch, or under the test runner (no
+        // backend to report to — mirrors the repo's NODE_ENV=test I/O convention).
+        const isTestEnv = typeof process !== "undefined" && process.env?.NODE_ENV === "test";
+        if (!registry || isTestEnv || typeof fetch === "undefined")
+            return;
+        let timer = null;
+        let cancelled = false;
+        const flush = () => {
+            const { viewId, elements } = buildPayload(registry);
+            // Nothing addressable yet (e.g. before any useAgentElement mounts, or a
+            // non-instrumented view) → skip the POST. Navigation clears server-side
+            // elements on view switch, so we never need to push an empty snapshot.
+            if (elements.length === 0)
+                return;
+            void (async () => {
+                try {
+                    const [{ fetchWithCsrf }, { resolveApiUrl }] = await Promise.all([
+                        import("../api/csrf-client.js"),
+                        import("../utils/asset-url.js"),
+                    ]);
+                    await fetchWithCsrf(resolveApiUrl(`/api/views/${encodeURIComponent(viewId)}/elements`), {
+                        method: "POST",
+                        headers: { "Content-Type": "application/json" },
+                        body: JSON.stringify({ elements }),
+                    });
+                }
+                catch {
+                    // Best-effort: planner awareness is an optimization, never a hard dep.
+                }
+            })();
+        };
+        const schedule = () => {
+            if (cancelled)
+                return;
+            if (timer)
+                clearTimeout(timer);
+            timer = setTimeout(flush, REPORT_DEBOUNCE_MS);
+        };
+        schedule(); // initial snapshot
+        const unsubscribe = registry.subscribe(schedule);
+        return () => {
+            cancelled = true;
+            if (timer)
+                clearTimeout(timer);
+            unsubscribe();
+        };
+    }, [registry]);
+}

package/agent-surface/element-reporter.js

@@ -0,0 +1,24 @@
+/**
+ * AgentSurfaceElementReporter — reports a view's addressable element snapshot to
+ * the agent backend (POST /api/views/:id/elements) so the planner's "# Active
+ * View" awareness block can list elements and act on them by id WITHOUT a
+ * list-elements round-trip.
+ *
+ * Best-effort + debounced: it never throws into the view, and the server gates
+ * the report on the id matching the active (navigated-to) view, so a background
+ * or stale surface's report is simply dropped. Mounted inside AgentSurfaceProvider
+ * by the host wrappers (DynamicViewLoader, ShellViewAgentSurface) — not by the
+ * provider itself, so unit tests that render the provider directly stay offline.
+ *
+ * The payload builder and the subscribe/POST hook live in
+ * `element-reporter.hooks` so this module stays component-only (a React Fast
+ * Refresh requirement).
+ */
+import { useAgentSurface } from "./AgentSurfaceContext.hooks.js";
+import { useAgentSurfaceElementReporter } from "./element-reporter.hooks.js";
+/** Renders nothing; reports the surrounding view's elements to the backend. */
+export function AgentSurfaceElementReporter() {
+    const surface = useAgentSurface();
+    useAgentSurfaceElementReporter(surface?.registry ?? null);
+    return null;
+}

package/agent-surface/index.d.ts

@@ -0,0 +1,16 @@
+/**
+ * agent-surface — the unified layer that makes every view fully controllable by
+ * the agent: addressable elements, focus awareness, programmatic fill/click,
+ * visual indicators, and a capability bridge to the floating-pill chat/voice.
+ */
+export { AgentElementOverlay } from "./AgentElementOverlay.js";
+export { AgentSurfaceProvider, type AgentSurfaceProviderProps, } from "./AgentSurfaceContext.js";
+export { AgentSurfaceContext, useAgentSurface, } from "./AgentSurfaceContext.hooks.js";
+export { handleAgentSurfaceCapability, isAgentSurfaceCapability, } from "./capabilities.js";
+export { AgentButton, type AgentButtonProps, AgentInput, type AgentInputProps, IconTag, type IconTagProps, } from "./components.js";
+export { AgentSurfaceElementReporter } from "./element-reporter.js";
+export { useAgentSurfaceElementReporter } from "./element-reporter.hooks.js";
+export { getOrCreateViewRegistry, getViewRegistry, removeViewRegistry, setNativeFieldValue, ViewAgentRegistry, } from "./registry.js";
+export { AGENT_SURFACE_CAPABILITY_IDS, type AgentActionResult, type AgentElementDescriptor, type AgentElementRole, type AgentElementSnapshot, type AgentSurfaceSnapshot, type AgentViewType, CLICKABLE_ROLES, FILLABLE_ROLES, } from "./types.js";
+export { type AgentElementHandle, useAgentElement, } from "./useAgentElement.js";
+//# sourceMappingURL=index.d.ts.map

package/agent-surface/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/agent-surface/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EACL,oBAAoB,EACpB,KAAK,yBAAyB,GAC/B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,mBAAmB,EACnB,eAAe,GAChB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,4BAA4B,EAC5B,wBAAwB,GACzB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,WAAW,EACX,KAAK,gBAAgB,EACrB,UAAU,EACV,KAAK,eAAe,EACpB,OAAO,EACP,KAAK,YAAY,GAClB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAC;AAC1E,OAAO,EACL,uBAAuB,EACvB,eAAe,EACf,kBAAkB,EAClB,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,4BAA4B,EAC5B,KAAK,iBAAiB,EACtB,KAAK,sBAAsB,EAC3B,KAAK,gBAAgB,EACrB,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,KAAK,aAAa,EAClB,eAAe,EACf,cAAc,GACf,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,KAAK,kBAAkB,EACvB,eAAe,GAChB,MAAM,mBAAmB,CAAC"}

package/agent-surface/index.js

@@ -0,0 +1,15 @@
+/**
+ * agent-surface — the unified layer that makes every view fully controllable by
+ * the agent: addressable elements, focus awareness, programmatic fill/click,
+ * visual indicators, and a capability bridge to the floating-pill chat/voice.
+ */
+export { AgentElementOverlay } from "./AgentElementOverlay.js";
+export { AgentSurfaceProvider, } from "./AgentSurfaceContext.js";
+export { AgentSurfaceContext, useAgentSurface, } from "./AgentSurfaceContext.hooks.js";
+export { handleAgentSurfaceCapability, isAgentSurfaceCapability, } from "./capabilities.js";
+export { AgentButton, AgentInput, IconTag, } from "./components.js";
+export { AgentSurfaceElementReporter } from "./element-reporter.js";
+export { useAgentSurfaceElementReporter } from "./element-reporter.hooks.js";
+export { getOrCreateViewRegistry, getViewRegistry, removeViewRegistry, setNativeFieldValue, ViewAgentRegistry, } from "./registry.js";
+export { AGENT_SURFACE_CAPABILITY_IDS, CLICKABLE_ROLES, FILLABLE_ROLES, } from "./types.js";
+export { useAgentElement, } from "./useAgentElement.js";

package/agent-surface/registry.d.ts

@@ -0,0 +1,48 @@
+/**
+ * ViewAgentRegistry — the live, per-view store of agent-addressable elements.
+ *
+ * One registry exists per mounted view (keyed by `viewType:viewId`).
+ * `useAgentElement` registers/updates/unregisters elements here; the
+ * DynamicViewLoader interact handler reads it to satisfy agent capabilities;
+ * `AgentElementOverlay` subscribes to it to draw indicators.
+ */
+import { type AgentActionResult, type AgentElementDescriptor, type AgentElementSnapshot, type AgentSurfaceSnapshot, type AgentViewType } from "./types.js";
+/**
+ * Set a native input/textarea/select value in a way React's controlled inputs
+ * observe — bypasses the React value setter then fires input/change events.
+ * Shared with the DynamicViewLoader selector path.
+ */
+export declare function setNativeFieldValue(target: HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement, value: string): void;
+export declare class ViewAgentRegistry {
+    readonly viewId: string;
+    readonly viewType: AgentViewType;
+    private readonly elements;
+    private readonly listeners;
+    private version;
+    private highlight;
+    constructor(viewId: string, viewType: AgentViewType);
+    register(descriptor: AgentElementDescriptor, getElement: () => HTMLElement | null): () => void;
+    update(id: string, patch: Partial<AgentElementDescriptor>): void;
+    subscribe: (listener: () => void) => (() => void);
+    getVersion: () => number;
+    /** Public version bump — used by `useAgentElement` when a live descriptor's
+     *  rendered fields (label/status/…) change so subscribers re-read snapshots. */
+    touch(): void;
+    private bump;
+    private orderedRecords;
+    private snapshotRecord;
+    snapshot(): AgentSurfaceSnapshot;
+    describe(id: string): AgentElementSnapshot | null;
+    getFocusedId(): string | null;
+    size(): number;
+    focus(id: string): AgentActionResult;
+    click(id: string): AgentActionResult;
+    fill(id: string, value: string): AgentActionResult;
+    scrollTo(id: string): AgentActionResult;
+    setHighlight(on: boolean): void;
+    isHighlighting(): boolean;
+}
+export declare function getOrCreateViewRegistry(viewId: string, viewType: AgentViewType): ViewAgentRegistry;
+export declare function getViewRegistry(viewId: string, viewType: AgentViewType): ViewAgentRegistry | undefined;
+export declare function removeViewRegistry(viewId: string, viewType: AgentViewType): void;
+//# sourceMappingURL=registry.d.ts.map

package/agent-surface/registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../../../../src/agent-surface/registry.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,sBAAsB,EAC3B,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,KAAK,aAAa,EAGnB,MAAM,SAAS,CAAC;AAkCjB;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,gBAAgB,GAAG,mBAAmB,GAAG,iBAAiB,EAClE,KAAK,EAAE,MAAM,GACZ,IAAI,CAWN;AAED,qBAAa,iBAAiB;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IAEjC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoC;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAyB;IACnD,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,SAAS,CAAS;gBAEd,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa;IAOnD,QAAQ,CACN,UAAU,EAAE,sBAAsB,EAClC,UAAU,EAAE,MAAM,WAAW,GAAG,IAAI,GACnC,MAAM,IAAI;IAkBb,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,sBAAsB,CAAC,GAAG,IAAI;IAShE,SAAS,GAAI,UAAU,MAAM,IAAI,KAAG,CAAC,MAAM,IAAI,CAAC,CAG9C;IAEF,UAAU,QAAO,MAAM,CAAiB;IAExC;oFACgF;IAChF,KAAK,IAAI,IAAI;IAIb,OAAO,CAAC,IAAI;IAOZ,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,cAAc;IAqCtB,QAAQ,IAAI,oBAAoB;IAahC,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,oBAAoB,GAAG,IAAI;IAKjD,YAAY,IAAI,MAAM,GAAG,IAAI;IAa7B,IAAI,IAAI,MAAM;IAMd,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,iBAAiB;IAQpC,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,iBAAiB;IAgBpC,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,iBAAiB;IAiClD,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,iBAAiB;IAUvC,YAAY,CAAC,EAAE,EAAE,OAAO,GAAG,IAAI;IAM/B,cAAc,IAAI,OAAO;CAG1B;AAUD,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,aAAa,GACtB,iBAAiB,CAQnB;AAED,wBAAgB,eAAe,CAC7B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,aAAa,GACtB,iBAAiB,GAAG,SAAS,CAE/B;AAED,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,aAAa,GACtB,IAAI,CAEN"}

package/agent-surface/registry.js

@@ -0,0 +1,274 @@
+/**
+ * ViewAgentRegistry — the live, per-view store of agent-addressable elements.
+ *
+ * One registry exists per mounted view (keyed by `viewType:viewId`).
+ * `useAgentElement` registers/updates/unregisters elements here; the
+ * DynamicViewLoader interact handler reads it to satisfy agent capabilities;
+ * `AgentElementOverlay` subscribes to it to draw indicators.
+ */
+import { CLICKABLE_ROLES, FILLABLE_ROLES, } from "./types.js";
+function isFillable(descriptor) {
+    if (typeof descriptor.fillable === "boolean")
+        return descriptor.fillable;
+    return FILLABLE_ROLES.has(descriptor.role ?? "region");
+}
+function isClickable(descriptor) {
+    if (typeof descriptor.clickable === "boolean")
+        return descriptor.clickable;
+    if (descriptor.onActivate)
+        return true;
+    return CLICKABLE_ROLES.has(descriptor.role ?? "region");
+}
+function readDomValue(el) {
+    if (!el)
+        return undefined;
+    if (el instanceof HTMLInputElement ||
+        el instanceof HTMLTextAreaElement ||
+        el instanceof HTMLSelectElement) {
+        if (el instanceof HTMLInputElement && el.type === "checkbox") {
+            return el.checked;
+        }
+        return el.value;
+    }
+    return undefined;
+}
+/**
+ * Set a native input/textarea/select value in a way React's controlled inputs
+ * observe — bypasses the React value setter then fires input/change events.
+ * Shared with the DynamicViewLoader selector path.
+ */
+export function setNativeFieldValue(target, value) {
+    const prototype = target instanceof HTMLTextAreaElement
+        ? HTMLTextAreaElement.prototype
+        : target instanceof HTMLSelectElement
+            ? HTMLSelectElement.prototype
+            : HTMLInputElement.prototype;
+    const setter = Object.getOwnPropertyDescriptor(prototype, "value")?.set;
+    setter?.call(target, value);
+    target.dispatchEvent(new Event("input", { bubbles: true }));
+    target.dispatchEvent(new Event("change", { bubbles: true }));
+}
+export class ViewAgentRegistry {
+    viewId;
+    viewType;
+    elements = new Map();
+    listeners = new Set();
+    version = 0;
+    highlight = false;
+    constructor(viewId, viewType) {
+        this.viewId = viewId;
+        this.viewType = viewType;
+    }
+    // ── registration ────────────────────────────────────────────────────────
+    register(descriptor, getElement) {
+        this.elements.set(descriptor.id, {
+            descriptor,
+            getElement,
+            registeredAt: this.version,
+        });
+        this.bump();
+        return () => {
+            const record = this.elements.get(descriptor.id);
+            // Only delete if this is still the same registration (guards against a
+            // remount registering before the prior unmount cleanup runs).
+            if (record && record.getElement === getElement) {
+                this.elements.delete(descriptor.id);
+                this.bump();
+            }
+        };
+    }
+    update(id, patch) {
+        const record = this.elements.get(id);
+        if (!record)
+            return;
+        record.descriptor = { ...record.descriptor, ...patch };
+        this.bump();
+    }
+    // ── reactivity (useSyncExternalStore) ─────────────────────────────────────
+    subscribe = (listener) => {
+        this.listeners.add(listener);
+        return () => this.listeners.delete(listener);
+    };
+    getVersion = () => this.version;
+    /** Public version bump — used by `useAgentElement` when a live descriptor's
+     *  rendered fields (label/status/…) change so subscribers re-read snapshots. */
+    touch() {
+        this.bump();
+    }
+    bump() {
+        this.version += 1;
+        for (const listener of this.listeners)
+            listener();
+    }
+    // ── introspection ─────────────────────────────────────────────────────────
+    orderedRecords() {
+        return [...this.elements.values()].sort((a, b) => {
+            const oa = a.descriptor.order ?? 100;
+            const ob = b.descriptor.order ?? 100;
+            if (oa !== ob)
+                return oa - ob;
+            return a.registeredAt - b.registeredAt;
+        });
+    }
+    snapshotRecord(record) {
+        const { descriptor } = record;
+        const el = record.getElement();
+        const role = descriptor.role ?? "region";
+        const value = descriptor.getValue
+            ? descriptor.getValue()
+            : readDomValue(el);
+        const rect = el?.getBoundingClientRect();
+        const visible = rect ? rect.width > 0 && rect.height > 0 : false;
+        const focused = typeof document !== "undefined" &&
+            el != null &&
+            (document.activeElement === el || el.contains(document.activeElement));
+        return {
+            id: descriptor.id,
+            role,
+            label: descriptor.label,
+            group: descriptor.group,
+            description: descriptor.description,
+            status: descriptor.status,
+            value,
+            fillable: isFillable(descriptor),
+            clickable: isClickable(descriptor),
+            focused,
+            visible,
+            options: descriptor.options,
+            bounds: rect
+                ? {
+                    x: Math.round(rect.x),
+                    y: Math.round(rect.y),
+                    width: Math.round(rect.width),
+                    height: Math.round(rect.height),
+                }
+                : undefined,
+        };
+    }
+    snapshot() {
+        const elements = this.orderedRecords().map((r) => this.snapshotRecord(r));
+        const focused = elements.find((e) => e.focused)?.id ?? null;
+        return {
+            viewId: this.viewId,
+            viewType: this.viewType,
+            elementCount: elements.length,
+            focusedId: focused,
+            elements,
+            updatedAt: this.version,
+        };
+    }
+    describe(id) {
+        const record = this.elements.get(id);
+        return record ? this.snapshotRecord(record) : null;
+    }
+    getFocusedId() {
+        if (typeof document === "undefined")
+            return null;
+        const active = document.activeElement;
+        if (!active)
+            return null;
+        for (const record of this.orderedRecords()) {
+            const el = record.getElement();
+            if (el && (el === active || el.contains(active))) {
+                return record.descriptor.id;
+            }
+        }
+        return null;
+    }
+    size() {
+        return this.elements.size;
+    }
+    // ── actions ───────────────────────────────────────────────────────────────
+    focus(id) {
+        const record = this.elements.get(id);
+        const el = record?.getElement();
+        if (!el)
+            return { ok: false, id, reason: "element not found" };
+        el.focus();
+        return { ok: true, id };
+    }
+    click(id) {
+        const record = this.elements.get(id);
+        if (!record)
+            return { ok: false, id, reason: "element not found" };
+        if (!isClickable(record.descriptor)) {
+            return { ok: false, id, reason: "element is not clickable" };
+        }
+        if (record.descriptor.onActivate) {
+            record.descriptor.onActivate();
+            return { ok: true, id };
+        }
+        const el = record.getElement();
+        if (!el)
+            return { ok: false, id, reason: "element not mounted" };
+        el.click();
+        return { ok: true, id };
+    }
+    fill(id, value) {
+        const record = this.elements.get(id);
+        if (!record)
+            return { ok: false, id, reason: "element not found" };
+        if (!isFillable(record.descriptor)) {
+            return { ok: false, id, reason: "element is not fillable" };
+        }
+        if (record.descriptor.options &&
+            record.descriptor.options.length > 0 &&
+            !record.descriptor.options.includes(value)) {
+            return {
+                ok: false,
+                id,
+                reason: `value must be one of: ${record.descriptor.options.join(", ")}`,
+            };
+        }
+        if (record.descriptor.onFill) {
+            record.descriptor.onFill(value);
+            return { ok: true, id, value };
+        }
+        const el = record.getElement();
+        if (el instanceof HTMLInputElement ||
+            el instanceof HTMLTextAreaElement ||
+            el instanceof HTMLSelectElement) {
+            setNativeFieldValue(el, value);
+            return { ok: true, id, value };
+        }
+        return { ok: false, id, reason: "element is not a native field" };
+    }
+    scrollTo(id) {
+        const record = this.elements.get(id);
+        const el = record?.getElement();
+        if (!el)
+            return { ok: false, id, reason: "element not found" };
+        el.scrollIntoView({ behavior: "smooth", block: "center" });
+        return { ok: true, id };
+    }
+    // ── highlight (agent indicator overlay) ────────────────────────────────────
+    setHighlight(on) {
+        if (this.highlight === on)
+            return;
+        this.highlight = on;
+        this.bump();
+    }
+    isHighlighting() {
+        return this.highlight;
+    }
+}
+// ── module-level map of live registries ─────────────────────────────────────
+const viewRegistries = new Map();
+function key(viewId, viewType) {
+    return `${viewType}:${viewId}`;
+}
+export function getOrCreateViewRegistry(viewId, viewType) {
+    const k = key(viewId, viewType);
+    let registry = viewRegistries.get(k);
+    if (!registry) {
+        registry = new ViewAgentRegistry(viewId, viewType);
+        viewRegistries.set(k, registry);
+    }
+    return registry;
+}
+export function getViewRegistry(viewId, viewType) {
+    return viewRegistries.get(key(viewId, viewType));
+}
+export function removeViewRegistry(viewId, viewType) {
+    viewRegistries.delete(key(viewId, viewType));
+}

package/agent-surface/types.d.ts

@@ -0,0 +1,93 @@
+/**
+ * agent-surface types — the contract that makes every view element addressable,
+ * focus-aware, fillable, and reactive to the agent.
+ *
+ * A view registers its interactive + informational elements through
+ * `useAgentElement`. The registry exposes them to the agent via the standard
+ * view-interact capabilities (`list-elements`, `agent-fill`, `agent-click`, …)
+ * so the floating pill can drive any view from voice or text without the view
+ * shipping its own chat surface.
+ */
+/** Semantic role of an addressable element — drives icons, affordances, search. */
+export type AgentElementRole = "button" | "link" | "text-input" | "number-input" | "textarea" | "select" | "toggle" | "slider" | "tab" | "menu-item" | "list-item" | "card" | "metric" | "status" | "image" | "chart" | "region" | "heading" | "custom";
+/** Roles the agent can type into. */
+export declare const FILLABLE_ROLES: ReadonlySet<AgentElementRole>;
+/** Roles the agent can activate (click). */
+export declare const CLICKABLE_ROLES: ReadonlySet<AgentElementRole>;
+/**
+ * Descriptor supplied by a view through `useAgentElement`. The registry keeps a
+ * live reference to the DOM node plus these hints so it can act on the element
+ * (focus/click/fill) and describe it to the agent.
+ */
+export interface AgentElementDescriptor {
+    /** Stable id unique within the view (e.g. "send.amount", "tab.positions"). */
+    id: string;
+    /** Semantic role; defaults to "region" when omitted. */
+    role?: AgentElementRole;
+    /** Human/agent-facing label (what the agent will say to target it). */
+    label: string;
+    /** Optional grouping key for related elements (e.g. a form or section id). */
+    group?: string;
+    /** One-line description for the planner / element search. */
+    description?: string;
+    /** Current status token rendered as `data-state` (e.g. "active", "error"). */
+    status?: string;
+    /** Sort priority for `list-elements` (lower first). Falls back to DOM order. */
+    order?: number;
+    /** Override fillability (else derived from role). */
+    fillable?: boolean;
+    /** Override clickability (else derived from role). */
+    clickable?: boolean;
+    /** Allowed values for selects / choice pickers. */
+    options?: readonly string[];
+    /** Current value accessor for controlled components. */
+    getValue?: () => unknown;
+    /** Controlled fill handler. When present the registry calls it instead of
+     *  driving the DOM input directly. */
+    onFill?: (value: string) => void;
+    /** Controlled activate handler. When present the registry calls it instead of
+     *  dispatching a DOM click. */
+    onActivate?: () => void;
+}
+/** Serialisable snapshot of one element, returned to the agent. */
+export interface AgentElementSnapshot {
+    id: string;
+    role: AgentElementRole;
+    label: string;
+    group?: string;
+    description?: string;
+    status?: string;
+    value?: unknown;
+    fillable: boolean;
+    clickable: boolean;
+    focused: boolean;
+    visible: boolean;
+    options?: readonly string[];
+    /** Viewport-relative bounds — lets the agent reason about layout/anchoring. */
+    bounds?: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+}
+/** Full snapshot of a view's agent surface. */
+export interface AgentSurfaceSnapshot {
+    viewId: string;
+    viewType: AgentViewType;
+    elementCount: number;
+    focusedId: string | null;
+    elements: AgentElementSnapshot[];
+    updatedAt: number;
+}
+export type AgentViewType = "gui" | "tui" | "xr";
+/** Result of an agent action on an element. */
+export interface AgentActionResult {
+    ok: boolean;
+    id?: string;
+    reason?: string;
+    value?: unknown;
+}
+/** Capability ids handled generically by the agent-surface registry. */
+export declare const AGENT_SURFACE_CAPABILITY_IDS: ReadonlySet<string>;
+//# sourceMappingURL=types.d.ts.map

package/agent-surface/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../../src/agent-surface/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,mFAAmF;AACnF,MAAM,MAAM,gBAAgB,GACxB,QAAQ,GACR,MAAM,GACN,YAAY,GACZ,cAAc,GACd,UAAU,GACV,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,KAAK,GACL,WAAW,GACX,WAAW,GACX,MAAM,GACN,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,OAAO,GACP,QAAQ,GACR,SAAS,GACT,QAAQ,CAAC;AAEb,qCAAqC;AACrC,eAAO,MAAM,cAAc,EAAE,WAAW,CAAC,gBAAgB,CAOrD,CAAC;AAEL,4CAA4C;AAC5C,eAAO,MAAM,eAAe,EAAE,WAAW,CAAC,gBAAgB,CAStD,CAAC;AAEL;;;;GAIG;AACH,MAAM,WAAW,sBAAsB;IACrC,8EAA8E;IAC9E,EAAE,EAAE,MAAM,CAAC;IACX,wDAAwD;IACxD,IAAI,CAAC,EAAE,gBAAgB,CAAC;IACxB,uEAAuE;IACvE,KAAK,EAAE,MAAM,CAAC;IACd,8EAA8E;IAC9E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,6DAA6D;IAC7D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8EAA8E;IAC9E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,sDAAsD;IACtD,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,mDAAmD;IACnD,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5B,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,OAAO,CAAC;IACzB;0CACsC;IACtC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACjC;mCAC+B;IAC/B,UAAU,CAAC,EAAE,MAAM,IAAI,CAAC;CACzB;AAED,mEAAmE;AACnE,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,gBAAgB,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5B,+EAA+E;IAC/E,MAAM,CAAC,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAClE;AAED,+CAA+C;AAC/C,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,aAAa,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,oBAAoB,EAAE,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC;AAEjD,+CAA+C;AAC/C,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,OAAO,CAAC;IACZ,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,wEAAwE;AACxE,eAAO,MAAM,4BAA4B,EAAE,WAAW,CAAC,MAAM,CAU3D,CAAC"}

package/agent-surface/types.js

@@ -0,0 +1,40 @@
+/**
+ * agent-surface types — the contract that makes every view element addressable,
+ * focus-aware, fillable, and reactive to the agent.
+ *
+ * A view registers its interactive + informational elements through
+ * `useAgentElement`. The registry exposes them to the agent via the standard
+ * view-interact capabilities (`list-elements`, `agent-fill`, `agent-click`, …)
+ * so the floating pill can drive any view from voice or text without the view
+ * shipping its own chat surface.
+ */
+/** Roles the agent can type into. */
+export const FILLABLE_ROLES = new Set([
+    "text-input",
+    "number-input",
+    "textarea",
+    "select",
+    "slider",
+]);
+/** Roles the agent can activate (click). */
+export const CLICKABLE_ROLES = new Set([
+    "button",
+    "link",
+    "toggle",
+    "tab",
+    "menu-item",
+    "list-item",
+    "card",
+]);
+/** Capability ids handled generically by the agent-surface registry. */
+export const AGENT_SURFACE_CAPABILITY_IDS = new Set([
+    "list-elements",
+    "describe-element",
+    "get-focus",
+    "get-agent-state",
+    "agent-click",
+    "agent-fill",
+    "agent-focus",
+    "agent-scroll-to",
+    "set-highlight",
+]);

package/agent-surface/useAgentElement.d.ts

@@ -0,0 +1,31 @@
+/**
+ * useAgentElement — register one element with the active view's agent surface.
+ *
+ * ```tsx
+ * const { ref, agentProps } = useAgentElement<HTMLButtonElement>({
+ *   id: "send",
+ *   role: "button",
+ *   label: "Send transaction",
+ *   onActivate: handleSend,
+ * });
+ * return <button ref={ref} {...agentProps} onClick={handleSend}>Send</button>;
+ * ```
+ *
+ * The returned `ref` gives the registry a live handle to the DOM node, and
+ * `agentProps` stamps `data-agent-*` / `data-state` so the element is both
+ * machine-addressable and counted as a visual signal. Outside a view (no
+ * provider) the hook still returns valid inert props.
+ */
+import type { RefObject } from "react";
+import type { AgentElementDescriptor } from "./types.js";
+export interface AgentElementHandle<T extends HTMLElement> {
+    ref: RefObject<T | null>;
+    agentProps: {
+        "data-agent-id": string;
+        "data-agent-role": string;
+        "data-agent-label": string;
+        "data-state"?: string;
+    };
+}
+export declare function useAgentElement<T extends HTMLElement = HTMLElement>(descriptor: AgentElementDescriptor): AgentElementHandle<T>;
+//# sourceMappingURL=useAgentElement.d.ts.map

package/agent-surface/useAgentElement.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"useAgentElement.d.ts","sourceRoot":"","sources":["../../../../../src/agent-surface/useAgentElement.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAGvC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,SAAS,CAAC;AAEtD,MAAM,WAAW,kBAAkB,CAAC,CAAC,SAAS,WAAW;IACvD,GAAG,EAAE,SAAS,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IACzB,UAAU,EAAE;QACV,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,kBAAkB,EAAE,MAAM,CAAC;QAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;CACH;AAED,wBAAgB,eAAe,CAAC,CAAC,SAAS,WAAW,GAAG,WAAW,EACjE,UAAU,EAAE,sBAAsB,GACjC,kBAAkB,CAAC,CAAC,CAAC,CA+EvB"}