@hyperframes/studio 0.8.49 → 0.8.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/dist/assets/{hyperframes-player-CqON2Rzo.js → hyperframes-player-CxxDxHTW.js} +1 -1
  2. package/dist/assets/{index-mIgSsKqs.js → index-DUAIWxkI.js} +1 -1
  3. package/dist/assets/{index-BGovDksE.js → index-Dt3qBI-9.js} +1 -1
  4. package/dist/assets/{index-Bgmx2h7W.js → index-n2ml4tUn.js} +218 -217
  5. package/dist/assets/index-pO6p0H27.css +1 -0
  6. package/dist/index.d.ts +12 -4
  7. package/dist/index.html +2 -2
  8. package/dist/index.js +5899 -5754
  9. package/dist/index.js.map +1 -1
  10. package/package.json +7 -7
  11. package/src/App.tsx +10 -8
  12. package/src/components/EditorShell.selectionSync.test.tsx +4 -0
  13. package/src/components/EditorShell.tsx +24 -0
  14. package/src/components/StudioOverlays.tsx +1 -5
  15. package/src/components/editor/PreviewGuides.test.tsx +88 -0
  16. package/src/components/editor/PreviewGuides.tsx +125 -0
  17. package/src/components/editor/SnapToolbar.test.tsx +21 -0
  18. package/src/components/editor/SnapToolbar.tsx +25 -1
  19. package/src/components/editor/domEditingAgentPrompt.ts +5 -0
  20. package/src/components/editor/previewGuidesStore.ts +26 -0
  21. package/src/components/nle/NLEPreview.test.ts +13 -0
  22. package/src/components/nle/NLEPreview.tsx +9 -4
  23. package/src/components/nle/PreviewPane.test.tsx +52 -0
  24. package/src/components/nle/PreviewPane.tsx +2 -0
  25. package/src/components/nle/TimelinePane.tsx +14 -0
  26. package/src/components/nle/useCanvasZOrderTimelineMirror.test.tsx +1 -1
  27. package/src/components/nle/useCanvasZOrderTimelineMirror.ts +3 -3
  28. package/src/hooks/appHotkeysDispatch.ts +11 -0
  29. package/src/hooks/timelineAudioGroupCreate.ts +2 -2
  30. package/src/hooks/timelineTrackVisibility.ts +8 -11
  31. package/src/hooks/useAppHotkeys.textEditing.test.tsx +1 -0
  32. package/src/hooks/useAppHotkeys.ts +3 -0
  33. package/src/hooks/useAskAgentModal.ts +2 -0
  34. package/src/hooks/useClipboard.test.ts +234 -0
  35. package/src/hooks/useClipboard.ts +264 -70
  36. package/src/hooks/useGroupCommits.test.ts +0 -1
  37. package/src/hooks/useRazorSplit.history.test.tsx +0 -1
  38. package/src/hooks/useRazorSplit.test.ts +0 -2
  39. package/src/hooks/useRazorSplit.test.tsx +0 -1
  40. package/src/hooks/useStudioContextValue.ts +5 -19
  41. package/src/hooks/useTimelineEditing.test.tsx +0 -1
  42. package/src/player/components/ClipContextMenu.tsx +87 -7
  43. package/src/player/components/Timeline.tsx +14 -14
  44. package/src/player/components/TimelineCanvas.tsx +30 -0
  45. package/src/player/components/TimelineGestureOverlay.test.tsx +35 -1
  46. package/src/player/components/TimelineGestureOverlay.tsx +1 -1
  47. package/src/player/components/TimelineLanes.test.tsx +22 -1
  48. package/src/player/components/TimelineOverlays.tsx +12 -0
  49. package/src/player/components/TimelineTypes.ts +4 -0
  50. package/src/player/components/timelineAuthoredMoveTarget.ts +1 -2
  51. package/src/player/components/timelineClipChildren.test.ts +2 -2
  52. package/src/player/components/timelineClipChildren.tsx +2 -1
  53. package/src/player/components/timelineClipDragCommit.ts +2 -2
  54. package/src/player/components/timelineDragDrop.test.tsx +31 -0
  55. package/src/player/components/timelineDragDrop.ts +30 -23
  56. package/src/player/components/timelineEditCapabilities.ts +1 -2
  57. package/src/player/components/timelineEditing.test.ts +0 -16
  58. package/src/player/components/timelineGroupEditing.test.ts +1 -3
  59. package/src/player/components/timelineGroupEditing.ts +0 -1
  60. package/src/player/components/timelineTrackDisplay.ts +1 -1
  61. package/src/player/components/timelineTrackPersistPipeline.test.ts +0 -53
  62. package/src/player/components/useTimelineClipContextMenu.ts +32 -0
  63. package/src/player/components/useTimelineTrackDerivations.ts +1 -1
  64. package/src/player/hooks/timelineDomBinding.test.ts +0 -3
  65. package/src/player/hooks/timelineSyncHydration.test.ts +18 -0
  66. package/src/player/hooks/timelineSyncHydration.ts +37 -18
  67. package/src/player/hooks/useTimelinePlayer.test.ts +3 -20
  68. package/src/player/hooks/useTimelineRowElements.test.ts +28 -0
  69. package/src/player/hooks/useTimelineRowElements.ts +17 -0
  70. package/src/player/hooks/useTimelineSyncCallbacks.emptyManifest.test.tsx +41 -0
  71. package/src/player/hooks/useTimelineSyncCallbacks.ts +9 -10
  72. package/src/player/lib/describeClips.test.ts +95 -0
  73. package/src/player/lib/describeClips.ts +103 -0
  74. package/src/player/lib/timelineDOM.test.ts +0 -80
  75. package/src/player/lib/timelineDOM.ts +1 -61
  76. package/src/player/lib/timelineElementHelpers.ts +2 -53
  77. package/src/player/store/playerStore.ts +5 -0
  78. package/src/player/store/timelineElement.ts +1 -3
  79. package/src/shims/canvasBrowserStub.js +50 -0
  80. package/src/utils/clipboardPayload.test.ts +42 -2
  81. package/src/utils/clipboardPayload.ts +57 -32
  82. package/src/utils/previewSafeMargins.test.ts +34 -0
  83. package/src/utils/previewSafeMargins.ts +10 -0
  84. package/src/utils/razorSplitTransaction.test.ts +0 -1
  85. package/src/utils/sdkCanvasComposition.test.ts +77 -0
  86. package/src/utils/studioUiPreferences.test.ts +43 -0
  87. package/src/utils/studioUiPreferences.ts +27 -6
  88. package/src/utils/timelineElementSplit.test.ts +1 -2
  89. package/src/utils/timelineElementSplit.ts +0 -1
  90. package/src/webmcp/tools/lookTools.test.ts +40 -0
  91. package/src/webmcp/tools/lookTools.ts +32 -1
  92. package/dist/assets/index-B9LfWNyM.css +0 -1
  93. package/src/components/StudioGlobalDragOverlay.tsx +0 -24
  94. package/src/player/hooks/useExpandedTimelineElements.test.ts +0 -774
  95. package/src/player/hooks/useExpandedTimelineElements.ts +0 -460
@@ -2,7 +2,6 @@ import { describe, expect, it, vi } from "vitest";
2
2
  import type { TimelineElement } from "../store/playerStore";
3
3
  import type { ClipManifestClip } from "../lib/playbackTypes";
4
4
  import { createTimelineElementFromManifestClip } from "../lib/timelineDOM";
5
- import { buildExpandedElements } from "../hooks/useExpandedTimelineElements";
6
5
  import { normalizeToZones } from "./timelineZones";
7
6
  import { commitDraggedClipMove, type TimelineMoveEdit } from "./timelineClipDragCommit";
8
7
  import type { DraggedClipState } from "./useTimelineClipDrag";
@@ -113,56 +112,4 @@ describe("track persist pipeline (manifest → factory → lanes → drag commit
113
112
  const up = commitLaneChange(byId.get("g")!, 0, elements, [0, 1, 2]);
114
113
  expect(up[0].updates.track).toBe(3); // authored, NOT display lane 0
115
114
  });
116
-
117
- it("an expanded sub-comp child's lane change persists the sibling's authored track from ITS file", () => {
118
- // Host timeline: the sub-comp host plus a root clip, discovered through the
119
- // factory and lane-normalized like the store does.
120
- const hostManifest = [
121
- manifestClip({
122
- id: "scene",
123
- kind: "composition",
124
- tagName: "div",
125
- track: 0,
126
- start: 0,
127
- duration: 10,
128
- compositionId: "scene",
129
- compositionSrc: "scene.html",
130
- }),
131
- manifestClip({ id: "root-clip", kind: "video", tagName: "video", track: 1, start: 0 }),
132
- ];
133
- // scene.html has SPARSE authored tracks 3 and 7.
134
- const childClips = [
135
- manifestClip({ id: "c3", track: 3, start: 1, duration: 2, parentCompositionId: "scene" }),
136
- manifestClip({ id: "c7", track: 7, start: 4, duration: 2, parentCompositionId: "scene" }),
137
- ];
138
- const storeElements = normalizeToZones(fromManifest(hostManifest));
139
- const parentMap = new Map([
140
- ["c3", "scene"],
141
- ["c7", "scene"],
142
- ]);
143
- const expanded = buildExpandedElements(
144
- storeElements,
145
- [...hostManifest, ...childClips],
146
- parentMap,
147
- "scene",
148
- "scene",
149
- );
150
-
151
- // The children replaced the host row: synthetic display lanes, but the
152
- // authored track (in scene.html's coordinate space) survived the expansion.
153
- const c3 = expanded.find((e) => e.domId === "c3")!;
154
- const c7 = expanded.find((e) => e.domId === "c7")!;
155
- expect(c3).toMatchObject({ authoredTrack: 3, sourceFile: "scene.html" });
156
- expect(c7).toMatchObject({ authoredTrack: 7, sourceFile: "scene.html" });
157
- expect(c3.stackingContextId).toBe("root");
158
- expect(c3.track).not.toBe(3); // display row is synthetic
159
-
160
- // Drag c3 onto c7's display lane: the persist target is c3's OWN file, so
161
- // the written track must be c7's authored 7 — not the display-lane integer.
162
- const trackOrder = [...new Set(expanded.map((e) => e.track))].sort((a, b) => a - b);
163
- const edits = commitLaneChange(c3, c7.track, expanded, trackOrder);
164
- expect(edits).toHaveLength(1);
165
- expect(edits[0].updates.track).toBe(7);
166
- expect(edits[0].updates.track).not.toBe(c7.track);
167
- });
168
115
  });
@@ -0,0 +1,32 @@
1
+ import { useCallback } from "react";
2
+ import { usePlayerStore, type TimelineElement } from "../store/playerStore";
3
+ import type { ClipContextMenuState } from "./TimelineOverlays";
4
+
5
+ /** A clicked member of an active multi-selection keeps the whole group
6
+ * selected, so the context menu it opens acts on all of it. */
7
+ export function useClipContextMenu(
8
+ onSelectElement: ((element: TimelineElement | null) => void) | undefined,
9
+ dismissGapMenu: () => void,
10
+ setClipContextMenu: (state: ClipContextMenuState | null) => void,
11
+ ) {
12
+ const selectedElementIds = usePlayerStore((s) => s.selectedElementIds);
13
+ const setSelectedElementId = usePlayerStore((s) => s.setSelectedElementId);
14
+ return useCallback(
15
+ (e: React.MouseEvent, el: TimelineElement) => {
16
+ e.preventDefault();
17
+ const id = el.key ?? el.id;
18
+ if (!(selectedElementIds.size > 1 && selectedElementIds.has(id))) {
19
+ setSelectedElementId(id);
20
+ onSelectElement?.(el);
21
+ }
22
+ dismissGapMenu();
23
+ setClipContextMenu({
24
+ x: e.clientX,
25
+ y: e.clientY,
26
+ element: el,
27
+ sessionEpoch: usePlayerStore.getState().timelineSessionEpoch,
28
+ });
29
+ },
30
+ [selectedElementIds, setSelectedElementId, onSelectElement, dismissGapMenu, setClipContextMenu],
31
+ );
32
+ }
@@ -98,7 +98,7 @@ function buildGroupInfo(
98
98
  id: groupId,
99
99
  label: membership.labelByGroup.get(groupId) ?? groupId,
100
100
  // Exactly x.5, which sub-composition child rows are now kept strictly below
101
- // (`useExpandedTimelineElements`) — they used to be able to land here and
101
+ // (`useTimelineRowElements`) — they used to be able to land here and
102
102
  // collide, duplicating the group header.
103
103
  anchorKey: (memberTracks[0] ?? fallbackTrackNum) - 0.5,
104
104
  memberTracks,
@@ -73,9 +73,6 @@ describe("live DOM manifest hydration", () => {
73
73
  canTrimEnd: true,
74
74
  });
75
75
  expect(getTimelineEditCapabilities({ ...input, timelineLocked: true }).canMove).toBe(false);
76
- expect(getTimelineEditCapabilities({ ...input, timingSource: "implicit" }).canTrimEnd).toBe(
77
- false,
78
- );
79
76
  });
80
77
  it("uses at most one fallback candidate snapshot per hydration pass", () => {
81
78
  const doc = documentWith(
@@ -3,6 +3,7 @@
3
3
  import { describe, expect, it } from "vitest";
4
4
  import {
5
5
  collectSubCompositionDomChildren,
6
+ collectTopLevelElementIds,
6
7
  collectSubCompositionHostState,
7
8
  } from "./timelineSyncHydration";
8
9
  import type { ClipManifestClip } from "../lib/playbackTypes";
@@ -94,3 +95,20 @@ describe("collectSubCompositionHostState", () => {
94
95
  expect(state.size).toBe(0);
95
96
  });
96
97
  });
98
+
99
+ describe("collectTopLevelElementIds", () => {
100
+ const docOf = (html: string) => new DOMParser().parseFromString(html, "text/html");
101
+
102
+ it("names the sub-composition host and a timed element, not what is nested inside them", () => {
103
+ const doc = docOf(`<div data-composition-id="root">
104
+ <div id="wrap"><div id="a" data-start="0" data-duration="2"><div id="inner" data-start="0"></div></div></div>
105
+ <div id="scene" data-composition-id="scene" data-composition-src="scene.html"><div id="box" data-start="0"></div></div>
106
+ </div>`);
107
+ expect([...collectTopLevelElementIds(doc)!].sort()).toEqual(["a", "scene"]);
108
+ });
109
+
110
+ it("is null when the document has no composition root", () => {
111
+ expect(collectTopLevelElementIds(docOf("<p>hi</p>"))).toBeNull();
112
+ expect(collectTopLevelElementIds(null)).toBeNull();
113
+ });
114
+ });
@@ -14,12 +14,12 @@ import { usePlayerStore } from "../store/playerStore";
14
14
  import type { TimelineElement, DomClipChild, SubCompositionHostState } from "../store/playerStore";
15
15
  import { resolveCssStackingContextId } from "@hyperframes/core/runtime/stacking-context";
16
16
  import type { ClipTree } from "@hyperframes/core/runtime/clipTree";
17
+ import { topLevelElements, type StructureNode } from "@hyperframes/parsers/top-level-elements";
17
18
  import { HF_AUDIO_GROUP_ATTR } from "@hyperframes/core/audio-groups";
18
19
  import { groupInfoFor } from "../lib/timelineGroupInfo";
19
20
  import type { PlaybackAdapter, ClipManifestClip, IframeWindow } from "../lib/playbackTypes";
20
21
  import {
21
22
  buildStandaloneRootTimelineElement,
22
- createImplicitTimelineLayersFromDOM,
23
23
  createTimelineElementFromManifestClip,
24
24
  getTimelineElementSelector,
25
25
  parseTimelineFromDOM,
@@ -137,6 +137,28 @@ export function collectSubCompositionDomChildren(
137
137
  return out;
138
138
  }
139
139
 
140
+ interface DomStructureNode extends StructureNode<DomStructureNode> {
141
+ id: string;
142
+ }
143
+
144
+ function toStructureNode(el: Element): DomStructureNode {
145
+ const attrs: Record<string, string> = {};
146
+ for (const attr of Array.from(el.attributes)) attrs[attr.name] = attr.value;
147
+ return {
148
+ tag: el.tagName,
149
+ attrs,
150
+ id: el.id,
151
+ children: Array.from(el.children).map(toStructureNode),
152
+ };
153
+ }
154
+
155
+ /** DOM ids of the timeline's rows, by the definition the structure lint shares; null when there is no readable root. */
156
+ export function collectTopLevelElementIds(doc: Document | null): Set<string> | null {
157
+ const root = doc?.querySelector("[data-composition-id]");
158
+ if (!root) return null;
159
+ return new Set(topLevelElements(toStructureNode(root)).flatMap((n) => (n.id ? [n.id] : [])));
160
+ }
161
+
140
162
  /** The host-element `data-*` state one element carries, or null when it has none. */
141
163
  function readSubCompositionHostState(el: Element): SubCompositionHostState | null {
142
164
  const state: SubCompositionHostState = {};
@@ -214,28 +236,25 @@ export function buildTimelineElementsFromClips(
214
236
  });
215
237
  }
216
238
 
217
- /**
218
- * The clamped manifest elements plus the layers that exist only in the DOM.
219
- * Both halves need the same resolved duration, which is why they land together.
220
- */
221
- export function withImplicitDomLayers(
222
- els: readonly TimelineElement[],
223
- iframeDoc: Document | null,
224
- effectiveDuration: number,
225
- ): TimelineElement[] {
226
- const clamped = clampElementsToDuration(els, effectiveDuration);
227
- if (!iframeDoc || effectiveDuration <= 0) return clamped;
228
- return [
229
- ...clamped,
230
- ...createImplicitTimelineLayersFromDOM(iframeDoc, effectiveDuration, clamped),
231
- ];
232
- }
233
-
234
239
  /**
235
240
  * Drop elements that start past the composition's end and trim the ones that
236
241
  * straddle it. A non-positive duration means "not known yet" — pass through
237
242
  * untouched rather than clamping everything to nothing.
238
243
  */
244
+ /** Commits the manifest elements, including none. An empty manifest carries a 1s floor, not a duration. */
245
+ export function syncManifestTimeline(
246
+ els: readonly TimelineElement[],
247
+ manifestDuration: number,
248
+ storeDuration: number,
249
+ sync: (els: TimelineElement[], duration?: number) => void,
250
+ ): void {
251
+ const hasDuration = manifestDuration > 0 && els.length > 0;
252
+ sync(
253
+ clampElementsToDuration(els, hasDuration ? manifestDuration : storeDuration),
254
+ hasDuration ? manifestDuration : undefined,
255
+ );
256
+ }
257
+
239
258
  function clampElementsToDuration(
240
259
  els: readonly TimelineElement[],
241
260
  effectiveDuration: number,
@@ -330,34 +330,17 @@ describe("findTimelineDomNodeForClip", () => {
330
330
  });
331
331
 
332
332
  describe("anonymous timeline identity", () => {
333
- it("adds root-level untimed DOM layers as implicit full-duration layers", () => {
333
+ it("does not invent rows for root-level untimed wrappers", () => {
334
334
  const doc = createDocument(`
335
335
  <div data-composition-id="compare" data-start="0" data-duration="18">
336
- <link rel="stylesheet" href="styles.css" />
337
- <div class="scene-shell">
338
- <div class="topline">Title</div>
339
- </div>
336
+ <div class="scene-shell"><div class="topline">Title</div></div>
340
337
  <video id="main-video" class="clip main-video" data-start="0" data-duration="18" data-track-index="1"></video>
341
- <script></script>
342
338
  </div>
343
339
  `);
344
340
 
345
341
  const elements = parseTimelineFromDOM(doc, 18);
346
342
 
347
- expect(elements).toEqual(
348
- expect.arrayContaining([
349
- expect.objectContaining({
350
- duration: 18,
351
- label: "Scene Shell",
352
- selector: ".scene-shell",
353
- start: 0,
354
- tag: "div",
355
- timingSource: "implicit",
356
- }),
357
- ]),
358
- );
359
- expect(elements.find((element) => element.tag === "link")).toBeUndefined();
360
- expect(elements.find((element) => element.tag === "script")).toBeUndefined();
343
+ expect(elements.map((element) => element.tag)).toEqual(["video"]);
361
344
  });
362
345
 
363
346
  it("keeps fallback-parsed anonymous clips distinct when labels match", () => {
@@ -0,0 +1,28 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import type { TimelineElement } from "../store/playerStore";
3
+ import { selectTimelineRowElements } from "./useTimelineRowElements";
4
+
5
+ const el = (id: string, domId?: string): TimelineElement => ({
6
+ id,
7
+ domId,
8
+ start: 0,
9
+ duration: 1,
10
+ track: 0,
11
+ tag: "div",
12
+ });
13
+
14
+ describe("selectTimelineRowElements", () => {
15
+ it("drops an element the shared definition does not call top-level", () => {
16
+ const rows = selectTimelineRowElements([el("a", "a"), el("b", "b")], new Set(["a"]));
17
+ expect(rows.map((r) => r.id)).toEqual(["a"]);
18
+ });
19
+
20
+ it("keeps everything until the document has been read", () => {
21
+ const all = [el("a", "a"), el("b", "b")];
22
+ expect(selectTimelineRowElements(all, null)).toBe(all);
23
+ });
24
+
25
+ it("keeps an element with no DOM id", () => {
26
+ expect(selectTimelineRowElements([el("k")], new Set())).toHaveLength(1);
27
+ });
28
+ });
@@ -0,0 +1,17 @@
1
+ import { useMemo } from "react";
2
+ import { usePlayerStore, type TimelineElement } from "../store/playerStore";
3
+
4
+ /** Rows are the ids `topLevelElements` names, as the lint does; id-less ones are kept. */
5
+ export function selectTimelineRowElements(
6
+ elements: TimelineElement[],
7
+ topLevelIds: ReadonlySet<string> | null,
8
+ ): TimelineElement[] {
9
+ if (!topLevelIds) return elements;
10
+ return elements.filter((el) => el.domId === undefined || topLevelIds.has(el.domId));
11
+ }
12
+
13
+ export function useTimelineRowElements(): TimelineElement[] {
14
+ const elements = usePlayerStore((s) => s.elements);
15
+ const topLevelIds = usePlayerStore((s) => s.topLevelIds);
16
+ return useMemo(() => selectTimelineRowElements(elements, topLevelIds), [elements, topLevelIds]);
17
+ }
@@ -0,0 +1,41 @@
1
+ // @vitest-environment happy-dom
2
+
3
+ import React, { act } from "react";
4
+ import { createRoot } from "react-dom/client";
5
+ import { describe, expect, it, vi } from "vitest";
6
+ import type { TimelineElement } from "../store/playerStore";
7
+ import { useTimelineSyncCallbacks } from "./useTimelineSyncCallbacks";
8
+
9
+ (globalThis as unknown as { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
10
+
11
+ describe("processTimelineMessage with an empty manifest", () => {
12
+ it("commits zero rows without adopting the runtime 1s floor as the duration", () => {
13
+ const sync = vi.fn<(els: TimelineElement[], duration?: number) => void>();
14
+ let processTimelineMessage: ReturnType<
15
+ typeof useTimelineSyncCallbacks
16
+ >["processTimelineMessage"];
17
+ function Harness() {
18
+ ({ processTimelineMessage } = useTimelineSyncCallbacks({
19
+ iframeRef: { current: null },
20
+ probeIntervalRef: { current: undefined },
21
+ pendingSeekRef: { current: null },
22
+ isRefreshingRef: { current: false },
23
+ getAdapter: () => null,
24
+ syncTimelineElements: sync,
25
+ setDuration: () => {},
26
+ setCurrentTime: () => {},
27
+ requestTimelineReady: () => {},
28
+ setIsPlaying: () => {},
29
+ attachIframeShortcutListeners: () => {},
30
+ applyPreviewAudioState: () => {},
31
+ }));
32
+ return null;
33
+ }
34
+ const host = document.createElement("div");
35
+ act(() => createRoot(host).render(<Harness />));
36
+
37
+ processTimelineMessage!({ clips: [], durationInFrames: 30 });
38
+
39
+ expect(sync).toHaveBeenCalledWith([], undefined);
40
+ });
41
+ });
@@ -17,7 +17,9 @@ import { buildMissingCompositionElements } from "../lib/timelineIframeHelpers";
17
17
  import { acceptedRuntimeMessageFps } from "../lib/runtimeProtocol";
18
18
  import {
19
19
  buildTimelineElementsFromClips,
20
+ syncManifestTimeline,
20
21
  clipTreeParentMap,
22
+ collectTopLevelElementIds,
21
23
  collectSubCompositionDomChildren,
22
24
  collectSubCompositionHostState,
23
25
  hydrateTimelineFromPreview,
@@ -26,7 +28,6 @@ import {
26
28
  sanitizeDurationSeconds,
27
29
  seekAdapterToRestorePoint,
28
30
  syncAdapterDuration,
29
- withImplicitDomLayers,
30
31
  type RuntimeTimelineMessage,
31
32
  } from "./timelineSyncHydration";
32
33
 
@@ -114,9 +115,7 @@ export function useTimelineSyncCallbacks({
114
115
  // Convert a runtime timeline message (from iframe postMessage) into TimelineElements
115
116
  const processTimelineMessage = useCallback(
116
117
  (data: RuntimeTimelineMessage) => {
117
- if (!data.clips || data.clips.length === 0) {
118
- return;
119
- }
118
+ if (!data.clips) return;
120
119
 
121
120
  usePlayerStore.getState().setClipManifest(data.clips);
122
121
 
@@ -131,11 +130,13 @@ export function useTimelineSyncCallbacks({
131
130
  const parentMap = clipTreeParentMap(iframeRef.current?.contentWindow ?? null);
132
131
  const domClipChildren = collectSubCompositionDomChildren(iframeDoc, data.clips, parentMap);
133
132
  usePlayerStore.getState().setClipParentMap(parentMap);
133
+ usePlayerStore.getState().setTopLevelIds(collectTopLevelElementIds(iframeDoc));
134
134
  usePlayerStore.getState().setDomClipChildren(domClipChildren);
135
135
  usePlayerStore
136
136
  .getState()
137
137
  .setSubCompositionHostState(collectSubCompositionHostState(iframeDoc, data.clips));
138
138
  } catch {
139
+ usePlayerStore.getState().setTopLevelIds(null);
139
140
  // cross-origin or __clipTree not available — maps stay empty
140
141
  }
141
142
 
@@ -150,14 +151,12 @@ export function useTimelineSyncCallbacks({
150
151
  manifestDurationSeconds: data.durationInFrames / acceptedRuntimeMessageFps(data),
151
152
  authoredRootDurationSeconds: readTimelineDurationFromDocument(iframeDoc),
152
153
  });
153
- const timelineEls = withImplicitDomLayers(
154
+ syncManifestTimeline(
154
155
  els,
155
- iframeDoc,
156
- newDuration > 0 ? newDuration : usePlayerStore.getState().duration,
156
+ newDuration,
157
+ usePlayerStore.getState().duration,
158
+ syncTimelineElements,
157
159
  );
158
- if (timelineEls.length > 0) {
159
- syncTimelineElements(timelineEls, newDuration > 0 ? newDuration : undefined);
160
- }
161
160
  },
162
161
  [iframeRef, syncTimelineElements],
163
162
  );
@@ -0,0 +1,95 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import type { TimelineElement } from "../store/timelineElement";
3
+ import { describeClips, formatTimelineBlock } from "./describeClips";
4
+
5
+ function audio(id: string, start: number, duration: number, extra: Partial<TimelineElement> = {}) {
6
+ return {
7
+ id,
8
+ domId: id,
9
+ tag: "audio",
10
+ kind: "audio",
11
+ start,
12
+ duration,
13
+ track: 0,
14
+ ...extra,
15
+ } satisfies TimelineElement;
16
+ }
17
+
18
+ // Shaped like claude-paper-launch: two voiceovers, clicks, and dozens of keystroke sounds, each on its own track.
19
+ function paperLaunchClips(): TimelineElement[] {
20
+ const clips = [
21
+ audio("vo", 29.6, 2.17, { src: "voiceover.mp3", authoredTrack: 8, volume: 0.009772 }),
22
+ audio("sfx-click-0", 1.6, 0.07, { src: "click.mp3", authoredTrack: 100, volume: 0.85 }),
23
+ audio("sfx-type-0", 7.8, 0.57, { src: "typenew.mp3", authoredTrack: 187, volume: 0.2 }),
24
+ ];
25
+ for (let i = 1; i <= 84; i++) {
26
+ clips.push(audio(`sfx-extra-${i}`, 8 + i * 0.1, 0.5, { authoredTrack: 200 + i, volume: 0.2 }));
27
+ }
28
+ return clips;
29
+ }
30
+
31
+ describe("formatTimelineBlock", () => {
32
+ it("lists every audio clip with its start and volume", () => {
33
+ const block = formatTimelineBlock(paperLaunchClips());
34
+ const audioLines = block.split("\n").filter((line) => line.startsWith("- audio "));
35
+ expect(audioLines).toHaveLength(87);
36
+ expect(
37
+ audioLines.every((line) => /start=[\d.]+/.test(line) && /volume=[\d.]+/.test(line)),
38
+ ).toBe(true);
39
+ expect(block).toContain(
40
+ '- audio "sfx-click-0" src=click.mp3 start=1.6 duration=0.07 end=1.67 track=100 volume=0.85',
41
+ );
42
+ expect(block).toContain(
43
+ '- audio "vo" src=voiceover.mp3 start=29.6 duration=2.17 end=31.77 track=8 volume=0.01',
44
+ );
45
+ });
46
+
47
+ it("orders clips by start time", () => {
48
+ const lines = formatTimelineBlock(paperLaunchClips()).split("\n");
49
+ expect(lines[1]).toContain('"sfx-click-0"');
50
+ });
51
+
52
+ it("prints volume automation points, rate and group", () => {
53
+ const block = formatTimelineBlock([
54
+ audio("bed", 8, 20, {
55
+ automation:
56
+ '{"version":1,"lanes":[{"target":"volume","points":[{"t":0,"v":0.1},{"t":10,"v":0.5}]}]}',
57
+ playbackRate: 0.5,
58
+ audioGroup: "music",
59
+ timelineRole: "music",
60
+ }),
61
+ ]);
62
+ expect(block).toContain("rate=0.5 group=music role=music volume-lane=[0:0.1, 10:0.5]");
63
+ });
64
+
65
+ it("prints volume 0 and a project-relative src, and omits rate at normal speed", () => {
66
+ const block = formatTimelineBlock([
67
+ audio("mute", 0, 1, {
68
+ src: "http://localhost:5190/api/projects/p1/preview/assets/a.mp3",
69
+ volume: 0,
70
+ playbackRate: 1,
71
+ }),
72
+ ]);
73
+ expect(block).toContain(
74
+ '- audio "mute" src=assets/a.mp3 start=0 duration=1 end=1 track=0 volume=0',
75
+ );
76
+ expect(block).not.toContain("rate=");
77
+ });
78
+
79
+ it("is empty for an empty timeline and caps long ones", () => {
80
+ expect(formatTimelineBlock([])).toBe("");
81
+ const many = Array.from({ length: 250 }, (_, i) => audio(`a${i}`, i, 1));
82
+ expect(formatTimelineBlock(many)).toContain("(50 more clips not listed)");
83
+ });
84
+ });
85
+
86
+ describe("describeClips", () => {
87
+ it("keeps 'not authored' distinct from zero for volume and rate", () => {
88
+ const [plain, silent] = describeClips([
89
+ audio("a", 0, 1),
90
+ audio("b", 1, 1, { volume: 0, playbackRate: 0 }),
91
+ ]);
92
+ expect([plain?.volume, plain?.playbackRate]).toEqual([null, null]);
93
+ expect([silent?.volume, silent?.playbackRate]).toEqual([0, 0]);
94
+ });
95
+ });
@@ -0,0 +1,103 @@
1
+ /** Plain-field clip facts from the player store; feeds the Ask-agent prompt and `studio_look`. */
2
+ import type { TimelineElement } from "../store/playerStore";
3
+ import { elementAutomationLanes } from "../components/automationLaneData";
4
+ import { roundTo3 } from "../../utils/rounding";
5
+
6
+ export interface ClipLane {
7
+ /** `volume`, or `fx.<nodeId>.<param>`. */
8
+ target: string;
9
+ /** `t` is seconds from the start of the clip, not the composition. */
10
+ points: { t: number; v: number }[];
11
+ }
12
+
13
+ export interface ClipFact {
14
+ id: string;
15
+ label: string | null;
16
+ kind: string;
17
+ start: number;
18
+ duration: number;
19
+ end: number;
20
+ /** The `data-track-index` as written in the source file. */
21
+ trackIndex: number;
22
+ src: string | null;
23
+ sourceFile: string | null;
24
+ /** `null` when `data-volume` is not authored (the clip plays at 1). */
25
+ volume: number | null;
26
+ lanes: ClipLane[];
27
+ /** `null` at normal speed: not authored, or 1 (the manifest defaults it to 1). */
28
+ playbackRate: number | null;
29
+ audioGroup: string | null;
30
+ role: string | null;
31
+ }
32
+
33
+ /** The store holds preview URLs; the agent edits project files, so drop the origin and preview prefix. */
34
+ function projectRelativeSrc(src: string): string {
35
+ return src.replace(/^https?:\/\/[^/]+/, "").replace(/^\/api\/projects\/[^/]+\/preview\//, "");
36
+ }
37
+
38
+ export function describeClip(element: TimelineElement): ClipFact {
39
+ return {
40
+ id: element.domId ?? element.id,
41
+ label: element.label ?? null,
42
+ kind: element.kind ?? element.tag.toLowerCase(),
43
+ start: element.start,
44
+ duration: element.duration,
45
+ end: element.start + element.duration,
46
+ trackIndex: element.authoredTrack ?? element.track,
47
+ src: element.src ? projectRelativeSrc(element.src) : null,
48
+ sourceFile: element.sourceFile ?? null,
49
+ volume: element.volume ?? null,
50
+ lanes: elementAutomationLanes(element).map((lane) => ({
51
+ target: lane.target,
52
+ points: lane.points.map(({ t, v }) => ({ t, v })),
53
+ })),
54
+ playbackRate: element.playbackRate === 1 ? null : (element.playbackRate ?? null),
55
+ audioGroup: element.audioGroup ?? null,
56
+ role: element.timelineRole ?? null,
57
+ };
58
+ }
59
+
60
+ export const byStart = (a: ClipFact, b: ClipFact) =>
61
+ a.start - b.start || a.trackIndex - b.trackIndex;
62
+
63
+ export function describeClips(elements: readonly TimelineElement[]): ClipFact[] {
64
+ return elements.map(describeClip).sort(byStart);
65
+ }
66
+
67
+ const num = (n: number) => String(roundTo3(n));
68
+
69
+ function formatClipLine(clip: ClipFact): string {
70
+ const parts = [
71
+ `${clip.kind} "${clip.id}"`,
72
+ clip.src && `src=${clip.src}`,
73
+ `start=${num(clip.start)}`,
74
+ `duration=${num(clip.duration)}`,
75
+ `end=${num(clip.end)}`,
76
+ `track=${clip.trackIndex}`,
77
+ clip.volume !== null && `volume=${num(clip.volume)}`,
78
+ clip.playbackRate !== null && `rate=${num(clip.playbackRate)}`,
79
+ clip.audioGroup && `group=${clip.audioGroup}`,
80
+ clip.role && `role=${clip.role}`,
81
+ clip.sourceFile && `file=${clip.sourceFile}`,
82
+ ...clip.lanes.map(
83
+ (lane) =>
84
+ `${lane.target}-lane=[${lane.points.map((p) => `${num(p.t)}:${num(p.v)}`).join(", ")}]`,
85
+ ),
86
+ ];
87
+ return `- ${parts.filter(Boolean).join(" ")}`;
88
+ }
89
+
90
+ const PROMPT_CLIP_CAP = 200;
91
+
92
+ /** The Ask-agent prompt's Timeline block; empty when the timeline has no clips. */
93
+ export function formatTimelineBlock(elements: readonly TimelineElement[]): string {
94
+ const clips = describeClips(elements);
95
+ if (clips.length === 0) return "";
96
+ const shown = clips.slice(0, PROMPT_CLIP_CAP).map(formatClipLine);
97
+ const more = clips.length - shown.length;
98
+ return [
99
+ "Timeline (composition seconds; lane points are seconds from the clip start):",
100
+ ...shown,
101
+ ...(more > 0 ? [`(${more} more clips not listed)`] : []),
102
+ ].join("\n");
103
+ }