@pixerate/editor-svelte 0.101.4 → 0.103.1

package/dist/vector/VectorCanvas.svelte

@@ -14,8 +14,12 @@
     toVectorKeyInput,
     toVectorPointerInput,
     vectorDiamondPath,
+    createVectorPlaybackLocator,
     type VectorEditorController
   } from '@pixerate/editor/vector';
+  import type { AgentPlayback } from '@pixerate/editor/tools';
+  import { createPlaybackSurface, getAgentPlayback } from '../playback/playback.svelte.js';
+  import PresenceLayer from '../playback/PresenceLayer.svelte';
   import VectorGradientDefs from './VectorGradientDefs.svelte';
   import VectorFilterDefs from './VectorFilterDefs.svelte';
   import VectorGradientAnnotator from './VectorGradientAnnotator.svelte';
@@ -39,6 +43,15 @@
     style?: string;
     /** Accessible label (default "Vector canvas"). */
     'aria-label'?: string;
+    /**
+     * Agent playback: act out the vector commands' calls here with the agent's cursor (default:
+     * `setAgentPlayback` above; `null` turns it off).
+     */
+    playback?: AgentPlayback | null;
+    /** The playback surface id (default `vector`, where the vector commands play). */
+    playbackId?: string;
+    /** Bring the canvas on screen (switch to its tab) before the agent acts. */
+    onplaybackreveal?: () => void | Promise<void>;
   }
 
   let {
@@ -50,7 +63,10 @@
     idPrefix = `ve${++nextId}-`,
     class: className = '',
     style = '',
-    'aria-label': ariaLabel = 'Vector canvas'
+    'aria-label': ariaLabel = 'Vector canvas',
+    playback,
+    playbackId = 'vector',
+    onplaybackreveal
   }: Props = $props();
 
   const store = vectorStateOf(() => controller);
@@ -70,6 +86,35 @@
   const as = THEME.anchorSize;
 
   let svg: SVGSVGElement | undefined = $state();
+
+  // Agent playback: the agent's cursor in document units, drawn through the viewport.
+  const ambientPlayback = getAgentPlayback();
+  const playing = $derived((playback === undefined ? ambientPlayback : playback) ?? null);
+  const svgBox = () => svg?.getBoundingClientRect() ?? null;
+  const presence = createPlaybackSurface(() => playing, () => playbackId, () => ({
+    defaultKind: 'vector',
+    locate: createVectorPlaybackLocator(controller),
+    bounds: () => {
+      const r = svgBox();
+      const v = controller.getState().viewport;
+      return r && r.width > 0 ? { x: -v.x / v.zoom, y: -v.y / v.zoom, width: r.width / v.zoom, height: r.height / v.zoom } : null;
+    },
+    visible: () => {
+      const r = svgBox();
+      return !!r && r.width > 0 && r.height > 0 && !(typeof document !== 'undefined' && document.visibilityState === 'hidden');
+    },
+    toViewport: (p) => {
+      const r = svgBox();
+      const v = controller.getState().viewport;
+      return { x: (r?.left ?? 0) + p.x * v.zoom + v.x, y: (r?.top ?? 0) + p.y * v.zoom + v.y };
+    },
+    fromViewport: (p) => {
+      const r = svgBox();
+      const v = controller.getState().viewport;
+      return { x: (p.x - (r?.left ?? 0) - v.x) / v.zoom, y: (p.y - (r?.top ?? 0) - v.y) / v.zoom };
+    },
+    reveal: () => onplaybackreveal?.()
+  }));
   $effect(() => {
     const el = svg;
     if (!el || typeof ResizeObserver === 'undefined') return;
@@ -611,4 +656,12 @@
       {/key}
     </foreignObject>
   {/if}
+  {#if playing}
+    <!-- The agent's cursor (agent playback), over everything and never in the way. -->
+    <foreignObject data-vector-presence="" x="0" y="0" width="100%" height="100%" pointer-events="none" style="overflow:visible;pointer-events:none">
+      <div style="position:relative;width:100%;height:100%">
+        <PresenceLayer {presence} viewport={view.viewport} />
+      </div>
+    </foreignObject>
+  {/if}
 </svg>

package/dist/vector/VectorCanvas.svelte.d.ts

@@ -1,4 +1,5 @@
 import { type VectorEditorController } from '@pixerate/editor/vector';
+import type { AgentPlayback } from '@pixerate/editor/tools';
 interface Props {
     controller: VectorEditorController;
     /** Fit the artboard into the canvas when it first gets a size (default `true`). */
@@ -15,6 +16,15 @@ interface Props {
     style?: string;
     /** Accessible label (default "Vector canvas"). */
     'aria-label'?: string;
+    /**
+     * Agent playback: act out the vector commands' calls here with the agent's cursor (default:
+     * `setAgentPlayback` above; `null` turns it off).
+     */
+    playback?: AgentPlayback | null;
+    /** The playback surface id (default `vector`, where the vector commands play). */
+    playbackId?: string;
+    /** Bring the canvas on screen (switch to its tab) before the agent acts. */
+    onplaybackreveal?: () => void | Promise<void>;
 }
 declare const VectorCanvas: import("svelte").Component<Props, {}, "">;
 type VectorCanvas = ReturnType<typeof VectorCanvas>;

package/dist/video-editor/VideoEditor.svelte

@@ -1,4 +1,5 @@
 <script lang="ts">
+  import type { AgentPlayback } from '@pixerate/editor/tools';
   import { onDestroy } from 'svelte';
   import { VideoEditorController, type VideoEditorOptions } from '@pixerate/editor/video-editor';
   import ClipInspector from './ClipInspector.svelte';
@@ -40,9 +41,13 @@
     /** Show the effects panel (default: when a provider is given or the project has effects). */
     showEffects?: boolean;
     class?: string;
+    /** Agent playback on the timeline (see `<VideoTimeline playback>`; default: `setAgentPlayback` above). */
+    playback?: AgentPlayback | null;
+    /** Bring the editor on screen (switch to its tab) before the agent acts. */
+    onplaybackreveal?: () => void | Promise<void>;
   }
 
-  let { controller, options, editable = true, pxPerSecond = $bindable(100), showIssues = true, showTranscript = true, showScopes = false, effects, showEffects, class: className = '' }: Props = $props();
+  let { controller, options, editable = true, pxPerSecond = $bindable(100), showIssues = true, showTranscript = true, showScopes = false, effects, showEffects, class: className = '', playback, onplaybackreveal }: Props = $props();
   let owned: VideoEditorController | null = null;
   // A controller is made only when none is passed, and kept for the component's lifetime.
   const ctl = $derived(controller ?? (owned ??= new VideoEditorController(options)));
@@ -73,7 +78,7 @@
     <VideoPreview controller={ctl} class="rounded-md" />
     <VideoTransport controller={ctl} editable={false} />
     {#if showScopes}<VideoScopes controller={ctl} />{/if}
-    <VideoTimeline controller={ctl} bind:pxPerSecond editable={false} />
+    <VideoTimeline controller={ctl} bind:pxPerSecond editable={false} {playback} {onplaybackreveal} />
     {#if showTranscript}<TranscriptPanel controller={ctl} editable={false} />{/if}
     {@render issueList()}
   </div>
@@ -90,7 +95,7 @@
       {#if effectsShown}<VideoEffectsPanel controller={ctl} class="min-w-0" />{/if}
     </div>
     <div class="flex min-w-0 flex-col gap-2 lg:col-span-3">
-      <VideoTimeline controller={ctl} bind:pxPerSecond />
+      <VideoTimeline controller={ctl} bind:pxPerSecond {playback} {onplaybackreveal} />
       {#if showTranscript}<TranscriptPanel controller={ctl} />{/if}
       {@render issueList()}
     </div>

package/dist/video-editor/VideoEditor.svelte.d.ts

@@ -1,3 +1,4 @@
+import type { AgentPlayback } from '@pixerate/editor/tools';
 import { VideoEditorController, type VideoEditorOptions } from '@pixerate/editor/video-editor';
 import type { EffectProvider } from '@pixerate/editor/effects';
 interface Props {
@@ -27,6 +28,10 @@ interface Props {
     /** Show the effects panel (default: when a provider is given or the project has effects). */
     showEffects?: boolean;
     class?: string;
+    /** Agent playback on the timeline (see `<VideoTimeline playback>`; default: `setAgentPlayback` above). */
+    playback?: AgentPlayback | null;
+    /** Bring the editor on screen (switch to its tab) before the agent acts. */
+    onplaybackreveal?: () => void | Promise<void>;
 }
 declare const VideoEditor: import("svelte").Component<Props, {}, "pxPerSecond">;
 type VideoEditor = ReturnType<typeof VideoEditor>;

package/dist/video-editor/VideoTimeline.svelte

@@ -20,7 +20,11 @@
     type VideoMediaClip,
     type VideoTrack,
     type VideoWaveformMode,
+    parseVideoTimeRef,
   } from '@pixerate/editor/video-editor';
+  import type { AgentPlayback, PlaybackRect, PlaybackRef } from '@pixerate/editor/tools';
+  import { createPlaybackSurface, getAgentPlayback } from '../playback/playback.svelte.js';
+  import PresenceLayer from '../playback/PresenceLayer.svelte';
   import { onDestroy, untrack } from 'svelte';
   import { videoEditorStateOf } from './createVideoEditor.svelte.js';
 
@@ -35,9 +39,18 @@
     /** Draw waveforms in audio clips (default), in every clip with sound (`all`), or not at all. Sound is decoded once per asset through `controller.audioPeaks`. */
     waveforms?: VideoWaveformMode;
     class?: string;
+    /**
+     * Agent playback: act out the video tools' calls here with the agent's cursor (default:
+     * `setAgentPlayback` above; `null` turns it off).
+     */
+    playback?: AgentPlayback | null;
+    /** The playback surface id (default `video`, where the video tool providers play). */
+    playbackId?: string;
+    /** Bring the timeline on screen (switch to its tab) before the agent acts. */
+    onplaybackreveal?: () => void | Promise<void>;
   }
 
-  let { controller, pxPerSecond = $bindable(100), onpxpersecondchange, editable = true, waveforms = 'audio', class: className = '' }: Props = $props();
+  let { controller, pxPerSecond = $bindable(100), onpxpersecondchange, editable = true, waveforms = 'audio', class: className = '', playback, playbackId = 'video', onplaybackreveal }: Props = $props();
   /** Bumped when more sound peaks have loaded, so waveforms redraw. */
   let peaksVersion = $state(0);
   $effect(() => controller.audioPeaks.subscribe(() => peaksVersion++));
@@ -90,6 +103,51 @@
   const btn = 'rounded border border-border px-1.5 py-0.5 text-[11px] disabled:opacity-40';
 
   let container: HTMLDivElement | undefined = $state();
+
+  // Agent playback: the agent's cursor in pixels of the timeline's box, finding clips, tracks and times.
+  const ambientPlayback = getAgentPlayback();
+  const playing = $derived((playback === undefined ? ambientPlayback : playback) ?? null);
+  const rectIn = (el: Element | null | undefined): PlaybackRect | null => {
+    const box = container?.getBoundingClientRect();
+    const r = el?.getBoundingClientRect();
+    return box && r && r.width > 0 ? { x: r.left - box.left, y: r.top - box.top, width: r.width, height: r.height } : null;
+  };
+  const q = (selector: string) => container?.querySelector(selector);
+  const agentPresence = createPlaybackSurface(() => playing, () => playbackId, () => ({
+    defaultKind: 'clip',
+    locate: (ref: PlaybackRef) => {
+      if (ref.kind === 'clip') return rectIn(q(`[data-clip="${CSS.escape(ref.id)}"]`));
+      if (ref.kind === 'track') return rectIn(q(`[data-track-lane="${CSS.escape(ref.id)}"]`));
+      if (ref.kind === 'marker') return rectIn(q(`[data-marker="${CSS.escape(ref.id)}"]`));
+      const at = parseVideoTimeRef(ref);
+      if (!at) return null;
+      // The row: the named track, the row of the named clip, else the ruler.
+      const lane = at.track
+        ? q(`[data-track-lane="${CSS.escape(at.track)}"]`)
+        : at.clip
+          ? q(`[data-clip="${CSS.escape(at.clip)}"]`)?.closest('[data-track-lane]')
+          : ruler;
+      const row = rectIn(lane);
+      return row ? { x: row.x + (at.time * px) / 1000 - 3, y: row.y, width: 6, height: row.height } : null;
+    },
+    bounds: () => {
+      const r = container?.getBoundingClientRect();
+      return r && r.width > 0 ? { x: 0, y: 0, width: r.width, height: r.height } : null;
+    },
+    visible: () => {
+      const r = container?.getBoundingClientRect();
+      return !!r && r.width > 0 && r.height > 0 && !(typeof document !== 'undefined' && document.visibilityState === 'hidden');
+    },
+    toViewport: (p: { x: number; y: number }) => {
+      const r = container?.getBoundingClientRect();
+      return { x: (r?.left ?? 0) + p.x, y: (r?.top ?? 0) + p.y };
+    },
+    fromViewport: (p: { x: number; y: number }) => {
+      const r = container?.getBoundingClientRect();
+      return { x: p.x - (r?.left ?? 0), y: p.y - (r?.top ?? 0) };
+    },
+    reveal: () => onplaybackreveal?.()
+  }));
   let ruler: HTMLDivElement | undefined = $state();
 
   interface Drag {
@@ -262,7 +320,7 @@
 <!-- svelte-ignore a11y_no_noninteractive_tabindex, a11y_no_noninteractive_element_interactions -->
 <div
   bind:this={container}
-  class="flex flex-col gap-1 text-xs outline-none focus-visible:ring-1 focus-visible:ring-rose-500 {className}"
+  class="relative flex flex-col gap-1 text-xs outline-none focus-visible:ring-1 focus-visible:ring-rose-500 {className}"
   tabindex="0"
   role="group"
   aria-label="Timeline"
@@ -346,6 +404,7 @@
             class="relative shrink-0 {track.locked ? 'bg-muted/60' : ''} {track.hidden || track.muted ? 'opacity-60' : ''}"
             style="width:{width}px"
             data-lane={track.id}
+            data-track-lane={track.id}
             onpointerdown={(e) => {
               if (e.button === 0 && !(e.shiftKey || e.metaKey || e.ctrlKey)) controller.select(null);
             }}
@@ -394,4 +453,7 @@
   {#if editor.error}
     <p class="text-red-500" role="alert" data-edit-error="">{editor.error}</p>
   {/if}
+  {#if playing}
+    <PresenceLayer presence={agentPresence} />
+  {/if}
 </div>

package/dist/video-editor/VideoTimeline.svelte.d.ts

@@ -1,4 +1,5 @@
 import { type VideoEditorController, type VideoWaveformMode } from '@pixerate/editor/video-editor';
+import type { AgentPlayback } from '@pixerate/editor/tools';
 interface Props {
     controller: VideoEditorController;
     /** Zoom: pixels per second of timeline (default 100). Bindable. */
@@ -10,6 +11,15 @@ interface Props {
     /** Draw waveforms in audio clips (default), in every clip with sound (`all`), or not at all. Sound is decoded once per asset through `controller.audioPeaks`. */
     waveforms?: VideoWaveformMode;
     class?: string;
+    /**
+     * Agent playback: act out the video tools' calls here with the agent's cursor (default:
+     * `setAgentPlayback` above; `null` turns it off).
+     */
+    playback?: AgentPlayback | null;
+    /** The playback surface id (default `video`, where the video tool providers play). */
+    playbackId?: string;
+    /** Bring the timeline on screen (switch to its tab) before the agent acts. */
+    onplaybackreveal?: () => void | Promise<void>;
 }
 declare const VideoTimeline: import("svelte").Component<Props, {}, "pxPerSecond">;
 type VideoTimeline = ReturnType<typeof VideoTimeline>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pixerate/editor-svelte",
-  "version": "0.101.4",
+  "version": "0.103.1",
   "description": "Svelte bindings and prompt/rich-text components for @pixerate/editor",
   "author": "Pixerate",
   "license": "MIT",
@@ -144,7 +144,7 @@
     "@tiptap/extension-bubble-menu": "^2.27.3",
     "@tiptap/extension-placeholder": "^2.27.3",
     "@tiptap/starter-kit": "^2.27.3",
-    "@pixerate/editor": "0.140.4"
+    "@pixerate/editor": "0.142.1"
   },
   "peerDependencies": {
     "@dagrejs/dagre": "^1.1.4",