@hyperframes/studio 0.9.58 → 0.9.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/hyperframes-player-Cm0hYGm2.js +573 -0
- package/dist/assets/{index-D7ICtobF.js → index-BGrNWeLK.js} +199 -199
- package/dist/assets/{index-ww7-RmuM.js → index-REqr_dLu.js} +1 -1
- package/dist/assets/{pausedVideoDecoder-Dbe8Cmab.js → pausedVideoDecoder-DqLDSq8J.js} +1 -1
- package/dist/{chunk-5EEM5WO4.js → chunk-3KCEIESB.js} +1 -1
- package/dist/{chunk-G6FJSL7X.js → chunk-JTUKMELR.js} +34 -6
- package/dist/index.d.ts +29 -1
- package/dist/index.html +1 -1
- package/dist/index.js +734 -217
- package/dist/player.js +2 -2
- package/dist/ui.js +1 -1
- package/package.json +8 -8
- package/src/captions/components/CaptionOverlay.sealed.test.ts +273 -0
- package/src/captions/components/CaptionOverlay.tsx +112 -57
- package/src/captions/components/CaptionOverlayUtils.ts +319 -33
- package/src/captions/components/CaptionPropertyPanel.tsx +17 -0
- package/src/components/editor/LayersPanel.opaque.test.ts +2 -2
- package/src/components/editor/previewDrafts.test.ts +2 -2
- package/src/components/editor/previewGestureStartRead.test.ts +2 -2
- package/src/components/editor/previewLayers.test.ts +2 -2
- package/src/components/editor/previewOverlayRects.test.ts +2 -2
- package/src/components/editor/previewPointerTargets.test.ts +2 -2
- package/src/components/editor/previewSelection.test.ts +2 -2
- package/src/components/editor/useColorGradingController.sealed.test.ts +165 -0
- package/src/components/editor/useColorGradingController.ts +20 -1
- package/src/components/editor/useColorGradingPreviews.ts +127 -43
- package/src/components/nle/NLEPreview.test.ts +24 -0
- package/src/components/nle/NLEPreview.tsx +2 -6
- package/src/components/nle/previewStage.ts +11 -1
- package/src/components/panels/VariablesPanel.test.tsx +90 -1
- package/src/components/panels/VariablesPanel.tsx +41 -17
- package/src/components/panels/renderHandoff.test.ts +86 -0
- package/src/components/panels/renderHandoff.ts +28 -0
- package/src/components/ui/Menu.test.tsx +77 -0
- package/src/components/ui/Menu.tsx +47 -4
- package/src/components/ui/Popover.tsx +5 -2
- package/src/elementAnimations.test.ts +13 -0
- package/src/hooks/useCaptionDetection.ts +175 -54
- package/src/hooks/useMusicBeatAnalysis.test.tsx +131 -0
- package/src/hooks/useMusicBeatAnalysis.ts +51 -66
- package/src/hooks/usePersistentEditHistory.test.ts +84 -0
- package/src/hooks/usePersistentEditHistory.ts +36 -3
- package/src/hooks/useRenderClipContent.test.ts +21 -0
- package/src/hooks/useRenderClipContent.ts +3 -1
- package/src/index.ts +2 -0
- package/src/player/hooks/useShadowPreviewReload.ts +7 -0
- package/src/player/hooks/useTimelinePlayer.shadowReload.test.ts +20 -0
- package/src/player/lib/previewElements.test.ts +4 -0
- package/src/player/lib/previewElements.ts +1 -0
- package/src/sealedPreviewExports.test.ts +1 -0
- package/src/utils/gsapUndoRestore.sealed.test.ts +73 -0
- package/src/utils/gsapUndoRestore.ts +66 -0
- package/src/utils/studioPendingEdits.test.ts +20 -0
- package/src/utils/studioPendingEdits.ts +11 -4
- package/src/utils/timelineInspector.test.ts +49 -2
- package/src/utils/timelineInspector.ts +21 -2
- package/dist/assets/hyperframes-player-B78JDyI_.js +0 -565
|
@@ -5,7 +5,16 @@ import { useCaptionStore } from "../captions/store";
|
|
|
5
5
|
import { acceptStudioRuntimeMessage } from "../player/lib/runtimeProtocol";
|
|
6
6
|
import { useCaptionSync } from "../captions/hooks/useCaptionSync";
|
|
7
7
|
import { parseCaptionComposition } from "../captions/parser";
|
|
8
|
+
import type { CaptionModel } from "../captions/types";
|
|
8
9
|
import { studioApiFetch } from "../utils/studioApiFetch";
|
|
10
|
+
import { MAX_DESCRIBE_HANDLES } from "@hyperframes/core/runtime/editing-port";
|
|
11
|
+
import { previewComputedStyles, previewFrameOpaque } from "../player/lib/previewChannel";
|
|
12
|
+
import {
|
|
13
|
+
describePreviewElements,
|
|
14
|
+
previewAncestors,
|
|
15
|
+
queryPreviewElements,
|
|
16
|
+
type PreviewElement,
|
|
17
|
+
} from "../player/lib/previewElements";
|
|
9
18
|
|
|
10
19
|
interface UseCaptionDetectionParams {
|
|
11
20
|
projectId: string | null;
|
|
@@ -18,6 +27,91 @@ interface UseCaptionDetectionParams {
|
|
|
18
27
|
setRightCollapsed: (collapsed: boolean) => void;
|
|
19
28
|
}
|
|
20
29
|
|
|
30
|
+
// The parser's GROUP_SELECTOR and WORD_SELECTOR, the elements it reads.
|
|
31
|
+
const GROUP_SELECTOR = ".caption-group, .caption-line, .caption-block";
|
|
32
|
+
const WORD_SELECTOR = ".word, .caption-word";
|
|
33
|
+
|
|
34
|
+
const attribute = (el: PreviewElement | null | undefined, name: string) =>
|
|
35
|
+
el?.attributes.find(([key]) => key === name)?.[1] ?? null;
|
|
36
|
+
|
|
37
|
+
/** The styles `parseCaptionComposition` reads off its first word and group. */
|
|
38
|
+
const CAPTION_STYLES = [
|
|
39
|
+
"font-size",
|
|
40
|
+
"font-weight",
|
|
41
|
+
"font-family",
|
|
42
|
+
"color",
|
|
43
|
+
"text-transform",
|
|
44
|
+
"letter-spacing",
|
|
45
|
+
"background-color",
|
|
46
|
+
"border-radius",
|
|
47
|
+
"padding-top",
|
|
48
|
+
"padding-right",
|
|
49
|
+
"padding-bottom",
|
|
50
|
+
"padding-left",
|
|
51
|
+
];
|
|
52
|
+
|
|
53
|
+
async function frameStyles(iframe: HTMLIFrameElement, handle: string | undefined) {
|
|
54
|
+
if (!handle) return null;
|
|
55
|
+
const styles = await previewComputedStyles(iframe, { handle }, CAPTION_STYLES);
|
|
56
|
+
if (!styles) return null;
|
|
57
|
+
const camel = (name: string) => name.replace(/-(\w)/g, (_, c: string) => c.toUpperCase());
|
|
58
|
+
return Object.fromEntries(CAPTION_STYLES.map((name) => [camel(name), styles[name] ?? ""]));
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** A frame's caption groups and words rebuilt in order in a detached document, with a stand-in
|
|
62
|
+
* `getComputedStyle` for the parser; null past one query's matches or with no answer. */
|
|
63
|
+
async function frameCaptionDom(
|
|
64
|
+
iframe: HTMLIFrameElement,
|
|
65
|
+
): Promise<{ doc: Document; win: Window } | null> {
|
|
66
|
+
const [all, groups, words] = await Promise.all(
|
|
67
|
+
[`${GROUP_SELECTOR}, ${WORD_SELECTOR}`, GROUP_SELECTOR, WORD_SELECTOR].map((selector) =>
|
|
68
|
+
queryPreviewElements(iframe, selector),
|
|
69
|
+
),
|
|
70
|
+
);
|
|
71
|
+
if (!all || !groups || !words || all.truncated) return null;
|
|
72
|
+
const handles = all.matches.map((match) => match.handle);
|
|
73
|
+
const chains: Array<string[] | null> = [];
|
|
74
|
+
for (let i = 0; i < handles.length; i += MAX_DESCRIBE_HANDLES) {
|
|
75
|
+
const part = await previewAncestors(iframe, handles.slice(i, i + MAX_DESCRIBE_HANDLES));
|
|
76
|
+
if (!part) return null;
|
|
77
|
+
chains.push(...part);
|
|
78
|
+
}
|
|
79
|
+
const isGroup = new Set(groups.matches.map((match) => match.handle));
|
|
80
|
+
const isWord = new Set(words.matches.map((match) => match.handle));
|
|
81
|
+
const doc = document.implementation.createHTMLDocument("");
|
|
82
|
+
const mirrored = new Map<string, Element>();
|
|
83
|
+
handles.forEach((handle, i) => {
|
|
84
|
+
const el = doc.createElement("span");
|
|
85
|
+
if (isGroup.has(handle)) el.classList.add("caption-group");
|
|
86
|
+
if (isWord.has(handle)) el.classList.add("word");
|
|
87
|
+
const parent = chains[i]?.find((ancestor) => mirrored.has(ancestor));
|
|
88
|
+
(parent ? mirrored.get(parent)! : doc.body).append(el);
|
|
89
|
+
mirrored.set(handle, el);
|
|
90
|
+
});
|
|
91
|
+
const firstWord = words.matches[0]?.handle;
|
|
92
|
+
const firstGroup = groups.matches[0]?.handle;
|
|
93
|
+
const styles = new Map<Element | undefined, Record<string, string> | null>([
|
|
94
|
+
[mirrored.get(firstWord ?? ""), await frameStyles(iframe, firstWord)],
|
|
95
|
+
[mirrored.get(firstGroup ?? ""), await frameStyles(iframe, firstGroup)],
|
|
96
|
+
]);
|
|
97
|
+
const win = { getComputedStyle: (el: Element) => styles.get(el) ?? {} } as unknown as Window;
|
|
98
|
+
return { doc, win };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async function frameCaptionModel(
|
|
102
|
+
iframe: HTMLIFrameElement,
|
|
103
|
+
content: string,
|
|
104
|
+
): Promise<CaptionModel | null> {
|
|
105
|
+
const [root] = (await queryPreviewElements(iframe, "[data-composition-id]"))?.matches ?? [];
|
|
106
|
+
const rootEl = root ? (await describePreviewElements(iframe, [root.handle]))?.[0] : null;
|
|
107
|
+
const frame = await frameCaptionDom(iframe);
|
|
108
|
+
if (!frame) return null;
|
|
109
|
+
const w = parseInt(attribute(rootEl, "data-width") ?? "1920", 10);
|
|
110
|
+
const h = parseInt(attribute(rootEl, "data-height") ?? "1080", 10);
|
|
111
|
+
const dur = parseFloat(attribute(rootEl, "data-duration") ?? "0");
|
|
112
|
+
return parseCaptionComposition(frame.doc, frame.win, content, w, h, dur);
|
|
113
|
+
}
|
|
114
|
+
|
|
21
115
|
export function useCaptionDetection({
|
|
22
116
|
projectId,
|
|
23
117
|
activeCompPath,
|
|
@@ -52,6 +146,51 @@ export function useCaptionDetection({
|
|
|
52
146
|
if (!projectId) return;
|
|
53
147
|
|
|
54
148
|
let activating = false;
|
|
149
|
+
// A detection still in flight when the composition changes must not enter edit mode for the old one.
|
|
150
|
+
let disposed = false;
|
|
151
|
+
|
|
152
|
+
const pickCaptionSource = (hostSrcs: Array<string | null>, captionCompId: string | null) => {
|
|
153
|
+
const host = hostSrcs.find((src) => src && src.includes("captions"));
|
|
154
|
+
if (host) return host;
|
|
155
|
+
for (const [id, src] of compIdToSrc) {
|
|
156
|
+
if (id.includes("caption") || src.includes("caption")) return src;
|
|
157
|
+
}
|
|
158
|
+
if (activeCompPath?.includes("captions")) return activeCompPath;
|
|
159
|
+
return captionCompId === null ? null : compIdToSrc.get(captionCompId) || null;
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
const loadCaptions = (
|
|
163
|
+
srcPath: string,
|
|
164
|
+
parse: (content: string) => Promise<CaptionModel | null>,
|
|
165
|
+
) =>
|
|
166
|
+
studioApiFetch(buildProjectApiPath(projectId, `/files/${encodeURIComponent(srcPath)}`))
|
|
167
|
+
.then((r) => r.json())
|
|
168
|
+
.then(async (data: { content?: string }) => {
|
|
169
|
+
if (disposed || !data.content || useCaptionStore.getState().isEditMode) return;
|
|
170
|
+
const model = await parse(data.content);
|
|
171
|
+
if (disposed || !model || useCaptionStore.getState().isEditMode) return;
|
|
172
|
+
const store = useCaptionStore.getState();
|
|
173
|
+
store.setModel(model);
|
|
174
|
+
store.setSourceFilePath(srcPath);
|
|
175
|
+
store.setEditMode(true);
|
|
176
|
+
captionSync.loadOverrides();
|
|
177
|
+
})
|
|
178
|
+
.catch(() => {});
|
|
179
|
+
|
|
180
|
+
const frameCaptionSource = async (iframe: HTMLIFrameElement) => {
|
|
181
|
+
if (!(await queryPreviewElements(iframe, ".caption-group"))?.matches.length) return null;
|
|
182
|
+
const hosts = await queryPreviewElements(
|
|
183
|
+
iframe,
|
|
184
|
+
"[data-composition-src], [data-composition-file]",
|
|
185
|
+
);
|
|
186
|
+
const handles = (hosts?.matches ?? []).slice(0, MAX_DESCRIBE_HANDLES).map((m) => m.handle);
|
|
187
|
+
const described = handles.length ? await describePreviewElements(iframe, handles) : [];
|
|
188
|
+
const hostSrcs = (described ?? []).map(
|
|
189
|
+
(el) => attribute(el, "data-composition-src") || attribute(el, "data-composition-file"),
|
|
190
|
+
);
|
|
191
|
+
const captionComp = await queryPreviewElements(iframe, '[data-composition-id*="caption"]');
|
|
192
|
+
return pickCaptionSource(hostSrcs, captionComp?.matches[0]?.source.compositionId ?? null);
|
|
193
|
+
};
|
|
55
194
|
|
|
56
195
|
const tryActivateCaptions = () => {
|
|
57
196
|
const captionState = useCaptionStore.getState();
|
|
@@ -61,6 +200,19 @@ export function useCaptionDetection({
|
|
|
61
200
|
}
|
|
62
201
|
|
|
63
202
|
const iframe = previewIframeRef.current;
|
|
203
|
+
if (iframe && previewFrameOpaque(iframe)) {
|
|
204
|
+
activating = true;
|
|
205
|
+
void frameCaptionSource(iframe)
|
|
206
|
+
.then((src) =>
|
|
207
|
+
src && !disposed
|
|
208
|
+
? loadCaptions(src, (content) => frameCaptionModel(iframe, content))
|
|
209
|
+
: null,
|
|
210
|
+
)
|
|
211
|
+
.finally(() => {
|
|
212
|
+
activating = false;
|
|
213
|
+
});
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
64
216
|
let doc: Document | null = null;
|
|
65
217
|
let win: Window | null = null;
|
|
66
218
|
try {
|
|
@@ -74,63 +226,31 @@ export function useCaptionDetection({
|
|
|
74
226
|
const groups = doc.querySelectorAll(".caption-group");
|
|
75
227
|
if (groups.length === 0) return;
|
|
76
228
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
if (!captionSrcPath) {
|
|
90
|
-
for (const [id, src] of compIdToSrc) {
|
|
91
|
-
if (id.includes("caption") || src.includes("caption")) {
|
|
92
|
-
captionSrcPath = src;
|
|
93
|
-
break;
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
if (!captionSrcPath && activeCompPath?.includes("captions")) {
|
|
99
|
-
captionSrcPath = activeCompPath;
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
if (!captionSrcPath) {
|
|
103
|
-
const captionComp = doc.querySelector('[data-composition-id*="caption"]');
|
|
104
|
-
if (captionComp) {
|
|
105
|
-
const compId = captionComp.getAttribute("data-composition-id") || "";
|
|
106
|
-
captionSrcPath = compIdToSrc.get(compId) || null;
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
|
|
229
|
+
const hostSrcs = [
|
|
230
|
+
...doc.querySelectorAll("[data-composition-src], [data-composition-file]"),
|
|
231
|
+
].map(
|
|
232
|
+
(host) =>
|
|
233
|
+
host.getAttribute("data-composition-src") || host.getAttribute("data-composition-file"),
|
|
234
|
+
);
|
|
235
|
+
const captionComp = doc.querySelector('[data-composition-id*="caption"]');
|
|
236
|
+
const captionCompId = captionComp
|
|
237
|
+
? captionComp.getAttribute("data-composition-id") || ""
|
|
238
|
+
: null;
|
|
239
|
+
const captionSrcPath = pickCaptionSource(hostSrcs, captionCompId);
|
|
110
240
|
if (!captionSrcPath) return;
|
|
111
241
|
|
|
112
242
|
activating = true;
|
|
113
|
-
const
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
.
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
const store = useCaptionStore.getState();
|
|
125
|
-
store.setModel(model);
|
|
126
|
-
store.setSourceFilePath(srcPath);
|
|
127
|
-
store.setEditMode(true);
|
|
128
|
-
captionSync.loadOverrides();
|
|
129
|
-
})
|
|
130
|
-
.catch(() => {})
|
|
131
|
-
.finally(() => {
|
|
132
|
-
activating = false;
|
|
133
|
-
});
|
|
243
|
+
const frameDoc = doc;
|
|
244
|
+
const frameWin = win;
|
|
245
|
+
void loadCaptions(captionSrcPath, async (content) => {
|
|
246
|
+
const root = frameDoc.querySelector("[data-composition-id]");
|
|
247
|
+
const w = parseInt(root?.getAttribute("data-width") ?? "1920", 10);
|
|
248
|
+
const h = parseInt(root?.getAttribute("data-height") ?? "1080", 10);
|
|
249
|
+
const dur = parseFloat(root?.getAttribute("data-duration") ?? "0");
|
|
250
|
+
return parseCaptionComposition(frameDoc, frameWin, content, w, h, dur);
|
|
251
|
+
}).finally(() => {
|
|
252
|
+
activating = false;
|
|
253
|
+
});
|
|
134
254
|
};
|
|
135
255
|
|
|
136
256
|
const handleMessage = (e: MessageEvent) => {
|
|
@@ -146,6 +266,7 @@ export function useCaptionDetection({
|
|
|
146
266
|
tryActivateCaptions();
|
|
147
267
|
|
|
148
268
|
return () => {
|
|
269
|
+
disposed = true;
|
|
149
270
|
window.removeEventListener("message", handleMessage);
|
|
150
271
|
};
|
|
151
272
|
}, [activeCompPath, projectId, compIdToSrc, captionSync, previewIframeRef, livePreviewIframe]);
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
// @vitest-environment happy-dom
|
|
2
|
+
import React, { act } from "react";
|
|
3
|
+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
4
|
+
import type { TimelineElement } from "../player";
|
|
5
|
+
import { usePlayerStore } from "../player/store/playerStore";
|
|
6
|
+
import { FileManagerProvider } from "../contexts/FileManagerContext";
|
|
7
|
+
import { mountReactHarness } from "./domSelectionTestHarness";
|
|
8
|
+
|
|
9
|
+
const analyze = vi.hoisted(() => vi.fn());
|
|
10
|
+
vi.mock("@hyperframes/core/beats", async (actual) => ({
|
|
11
|
+
...(await actual<typeof import("@hyperframes/core/beats")>()),
|
|
12
|
+
analyzeMusicFromUrl: analyze,
|
|
13
|
+
}));
|
|
14
|
+
|
|
15
|
+
const { useMusicBeatAnalysis } = await import("./useMusicBeatAnalysis");
|
|
16
|
+
const { serializeBeats } = await import("@hyperframes/core/beats");
|
|
17
|
+
|
|
18
|
+
Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true });
|
|
19
|
+
|
|
20
|
+
const clip = (overrides: Partial<TimelineElement>) =>
|
|
21
|
+
({
|
|
22
|
+
id: "talk",
|
|
23
|
+
domId: "talk",
|
|
24
|
+
key: "talk",
|
|
25
|
+
tag: "audio",
|
|
26
|
+
src: "assets/talk.m4a",
|
|
27
|
+
start: 0,
|
|
28
|
+
duration: 180,
|
|
29
|
+
track: 3,
|
|
30
|
+
...overrides,
|
|
31
|
+
}) as TimelineElement;
|
|
32
|
+
|
|
33
|
+
let savedBeats = "";
|
|
34
|
+
const files = {
|
|
35
|
+
readOptionalProjectFile: async () => savedBeats,
|
|
36
|
+
writeProjectFile: async () => {},
|
|
37
|
+
} as unknown as React.ComponentProps<typeof FileManagerProvider>["value"];
|
|
38
|
+
|
|
39
|
+
function Probe() {
|
|
40
|
+
useMusicBeatAnalysis();
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Studio's timeline opened on `elements`: which files beat analysis decoded, and the beat times it shows. The hook
|
|
45
|
+
* caches an analysis per file, so each case names its own. */
|
|
46
|
+
async function opened(elements: TimelineElement[]) {
|
|
47
|
+
usePlayerStore.setState({ elements, beatAnalysis: null });
|
|
48
|
+
const root = mountReactHarness(
|
|
49
|
+
<FileManagerProvider value={files}>
|
|
50
|
+
<Probe />
|
|
51
|
+
</FileManagerProvider>,
|
|
52
|
+
);
|
|
53
|
+
await act(async () => {
|
|
54
|
+
for (let i = 0; i < 5; i += 1) await Promise.resolve();
|
|
55
|
+
});
|
|
56
|
+
const beats = usePlayerStore.getState().beatAnalysis?.beatTimes ?? null;
|
|
57
|
+
act(() => root.unmount());
|
|
58
|
+
return { decoded: analyze.mock.calls.map(([src]) => src), beats };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
beforeEach(() => {
|
|
62
|
+
savedBeats = "";
|
|
63
|
+
analyze.mockReset();
|
|
64
|
+
analyze.mockResolvedValue({ beatTimes: [1, 2], beatStrengths: [1, 1], bpm: 120 });
|
|
65
|
+
});
|
|
66
|
+
afterEach(() =>
|
|
67
|
+
usePlayerStore.setState({ elements: [], beatEdits: null, beatUndo: [], beatRedo: [] }),
|
|
68
|
+
);
|
|
69
|
+
|
|
70
|
+
// fails without the fix: a 62-minute recording with no role was picked as the music and decoded whole (1.4 GB on
|
|
71
|
+
// a Mac, which ran a person's page out of memory while their talk was still being transcribed).
|
|
72
|
+
describe("beat analysis decodes only a file no longer than a song", () => {
|
|
73
|
+
it("leaves a 62-minute unlabelled recording alone", async () => {
|
|
74
|
+
const talk = clip({ src: "assets/talk.m4a", duration: 3720, sourceDuration: 3720 });
|
|
75
|
+
expect((await opened([talk])).decoded).toEqual([]);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("leaves a 3-minute clip of a 62-minute file alone, since the decode reads the whole file", async () => {
|
|
79
|
+
const cut = clip({ src: "assets/cut.m4a", duration: 180, sourceDuration: 3720 });
|
|
80
|
+
expect((await opened([cut])).decoded).toEqual([]);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
it("leaves a recording alone while its file's length is unknown", async () => {
|
|
84
|
+
expect((await opened([clip({ src: "assets/unread.m4a", duration: 180 })])).decoded).toEqual([]);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it("shows a 62-minute music track's saved beats without decoding it", async () => {
|
|
88
|
+
savedBeats = serializeBeats([1, 2], [1, 1], "assets/mix.m4a");
|
|
89
|
+
const mix = clip({
|
|
90
|
+
src: "assets/mix.m4a",
|
|
91
|
+
timelineRole: "music",
|
|
92
|
+
duration: 3720,
|
|
93
|
+
sourceDuration: 3720,
|
|
94
|
+
});
|
|
95
|
+
expect(await opened([mix])).toEqual({ decoded: [], beats: [1, 2] });
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it("still finds the beats of a 3-minute song", async () => {
|
|
99
|
+
const song = clip({ src: "assets/song.mp3", sourceDuration: 180 });
|
|
100
|
+
expect(await opened([song])).toEqual({ decoded: ["assets/song.mp3"], beats: [1, 2] });
|
|
101
|
+
});
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
// fails without the fix: the length arriving after the saved beats showed loaded the track again, which dropped the
|
|
105
|
+
// person's beat edit and its undo.
|
|
106
|
+
it("keeps a music track's beat edits and undo when its file's length arrives", async () => {
|
|
107
|
+
savedBeats = serializeBeats([1, 2], [1, 1], "assets/edited.m4a");
|
|
108
|
+
const music = { src: "assets/edited.m4a", timelineRole: "music" as const, duration: 180 };
|
|
109
|
+
const settle = () =>
|
|
110
|
+
act(async () => {
|
|
111
|
+
for (let i = 0; i < 5; i += 1) await Promise.resolve();
|
|
112
|
+
});
|
|
113
|
+
usePlayerStore.setState({ elements: [clip(music)], beatAnalysis: null, beatEdits: null });
|
|
114
|
+
const root = mountReactHarness(
|
|
115
|
+
<FileManagerProvider value={files}>
|
|
116
|
+
<Probe />
|
|
117
|
+
</FileManagerProvider>,
|
|
118
|
+
);
|
|
119
|
+
await settle();
|
|
120
|
+
const edit = { src: "assets/edited.m4a", added: [{ time: 1.5, strength: 1 }], removed: [] };
|
|
121
|
+
act(() => usePlayerStore.getState().commitBeatEdits(edit, "add beat"));
|
|
122
|
+
act(() => usePlayerStore.setState({ elements: [clip({ ...music, sourceDuration: 180 })] }));
|
|
123
|
+
await settle();
|
|
124
|
+
const { beatEdits, beatUndo, beatAnalysis } = usePlayerStore.getState();
|
|
125
|
+
act(() => root.unmount());
|
|
126
|
+
expect({ beatEdits, undo: beatUndo.length, beats: beatAnalysis?.beatTimes }).toEqual({
|
|
127
|
+
beatEdits: edit,
|
|
128
|
+
undo: 1,
|
|
129
|
+
beats: [1, 2],
|
|
130
|
+
});
|
|
131
|
+
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { useEffect, useMemo, useRef } from "react";
|
|
2
2
|
import { usePlayerStore } from "../player/store/playerStore";
|
|
3
|
-
import { resolveBeatSourceTrack } from "../utils/timelineInspector";
|
|
3
|
+
import { fitsBeatAnalysis, resolveBeatSourceTrack } from "../utils/timelineInspector";
|
|
4
4
|
import { analyzeMusicFromUrl } from "@hyperframes/core/beats";
|
|
5
5
|
import { whenTimelineIdle } from "../player/components/timelineMotion";
|
|
6
6
|
import { useFileManagerContextOptional } from "../contexts/FileManagerContext";
|
|
@@ -32,67 +32,66 @@ function cacheAnalysis(url: string, promise: ReturnType<typeof analyzeMusicFromU
|
|
|
32
32
|
|
|
33
33
|
type ProjectIo = { readOptionalProjectFile: (p: string) => Promise<string> };
|
|
34
34
|
|
|
35
|
-
|
|
36
|
-
* Resolve the effective beat list for a track: a saved file with real beats
|
|
37
|
-
* wins; otherwise the detected beats are used (and `hasFile` is false so the
|
|
38
|
-
* caller seeds a new file). An empty saved file is ignored so detection retries.
|
|
39
|
-
*/
|
|
40
|
-
async function resolveBeats(
|
|
41
|
-
beatPath: string | null,
|
|
42
|
-
detected: { times: number[]; strengths: number[] },
|
|
43
|
-
io: ProjectIo,
|
|
44
|
-
): Promise<{ times: number[]; strengths: number[]; hasFile: boolean }> {
|
|
45
|
-
if (!beatPath) return { ...detected, hasFile: false };
|
|
46
|
-
try {
|
|
47
|
-
const content = await io.readOptionalProjectFile(beatPath);
|
|
48
|
-
const parsed = content ? parseBeats(content) : null;
|
|
49
|
-
if (parsed && parsed.times.length > 0) {
|
|
50
|
-
return { times: parsed.times, strengths: parsed.strengths, hasFile: true };
|
|
51
|
-
}
|
|
52
|
-
} catch {
|
|
53
|
-
/* fall back to detected beats */
|
|
54
|
-
}
|
|
55
|
-
return { ...detected, hasFile: false };
|
|
56
|
-
}
|
|
35
|
+
type Beats = { times: number[]; strengths: number[] };
|
|
57
36
|
|
|
58
|
-
/**
|
|
59
|
-
async function
|
|
37
|
+
/** The track's saved beats file, or null when it is missing, unreadable or holds no beat (detection then retries). */
|
|
38
|
+
async function readSavedBeats(io: ProjectIo, beatPath: string): Promise<Beats | null> {
|
|
60
39
|
try {
|
|
61
40
|
const content = await io.readOptionalProjectFile(beatPath);
|
|
62
41
|
const parsed = content ? parseBeats(content) : null;
|
|
63
|
-
return
|
|
42
|
+
return parsed && parsed.times.length > 0
|
|
43
|
+
? { times: parsed.times, strengths: parsed.strengths }
|
|
44
|
+
: null;
|
|
64
45
|
} catch {
|
|
65
|
-
return
|
|
46
|
+
return null;
|
|
66
47
|
}
|
|
67
48
|
}
|
|
68
49
|
|
|
69
50
|
type MusicAnalysis = Awaited<ReturnType<typeof analyzeMusicFromUrl>>;
|
|
70
51
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
*/
|
|
76
|
-
async function loadBeatAnalysis(
|
|
77
|
-
musicSrc: string,
|
|
78
|
-
beatPath: string,
|
|
79
|
-
io: ProjectIo,
|
|
80
|
-
): Promise<{ analysis: MusicAnalysis; times: number[]; strengths: number[] } | null> {
|
|
52
|
+
type Loaded = { analysis: MusicAnalysis } & Beats;
|
|
53
|
+
|
|
54
|
+
/** A track's analysis (memoized per URL), or null when decode or analysis fails. */
|
|
55
|
+
async function decodeTrack(musicSrc: string): Promise<MusicAnalysis | null> {
|
|
81
56
|
let promise = analysisCache.get(musicSrc);
|
|
82
57
|
if (!promise) {
|
|
83
58
|
promise = analyzeMusicFromUrl(musicSrc, { pause: whenTimelineIdle });
|
|
84
59
|
cacheAnalysis(musicSrc, promise);
|
|
85
60
|
}
|
|
86
|
-
|
|
87
|
-
const analysis = await promise;
|
|
88
|
-
const detected = { times: analysis.beatTimes, strengths: analysis.beatStrengths };
|
|
89
|
-
const { times, strengths } = await resolveBeats(beatPath, detected, io);
|
|
90
|
-
return { analysis, times, strengths };
|
|
91
|
-
} catch {
|
|
92
|
-
return null;
|
|
93
|
-
}
|
|
61
|
+
return promise.catch(() => null);
|
|
94
62
|
}
|
|
95
63
|
|
|
64
|
+
/** What a track shows. A fallback track (audio dropped in with no role) is analysed so the Beat tool works at once; a
|
|
65
|
+
* marked music track only once the person saved beats for it, so unrelated assets add no beat lines. A file that does
|
|
66
|
+
* not fit the decode keeps its saved beats, with no strengths measured from the audio. */
|
|
67
|
+
async function loadTrack(
|
|
68
|
+
musicSrc: string,
|
|
69
|
+
beatPath: string,
|
|
70
|
+
io: ProjectIo,
|
|
71
|
+
isFallbackTrack: boolean,
|
|
72
|
+
decodable: boolean,
|
|
73
|
+
): Promise<Loaded | null> {
|
|
74
|
+
const saved = await readSavedBeats(io, beatPath);
|
|
75
|
+
if (!isFallbackTrack && !saved) return null;
|
|
76
|
+
if (!decodable) return saved && { analysis: savedOnly(saved), ...saved };
|
|
77
|
+
const analysis = await decodeTrack(musicSrc);
|
|
78
|
+
if (!analysis) return null;
|
|
79
|
+
return {
|
|
80
|
+
analysis,
|
|
81
|
+
...(saved ?? { times: analysis.beatTimes, strengths: analysis.beatStrengths }),
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const savedOnly = ({ times, strengths }: Beats): MusicAnalysis => ({
|
|
86
|
+
beatTimes: times,
|
|
87
|
+
beatStrengths: strengths,
|
|
88
|
+
bpm: null,
|
|
89
|
+
bpmConfidence: "uncertain",
|
|
90
|
+
channelData: null,
|
|
91
|
+
sampleRate: 0,
|
|
92
|
+
peak: 0,
|
|
93
|
+
});
|
|
94
|
+
|
|
96
95
|
/** The current beat analysis, remapped onto the composition's edited beat grid. */
|
|
97
96
|
export function useAdjustedBeatAnalysis() {
|
|
98
97
|
const beatAnalysis = usePlayerStore((s) => s.beatAnalysis);
|
|
@@ -123,16 +122,19 @@ export function useMusicBeatAnalysis(): void {
|
|
|
123
122
|
? { readOptionalProjectFile, writeProjectFile }
|
|
124
123
|
: null;
|
|
125
124
|
|
|
126
|
-
const { musicSrc, isFallbackTrack } = useMemo(() => {
|
|
125
|
+
const { musicSrc, isFallbackTrack, decodable } = useMemo(() => {
|
|
127
126
|
const resolved = resolveBeatSourceTrack(elements);
|
|
128
127
|
return {
|
|
129
128
|
musicSrc: resolved?.element.src ?? null,
|
|
130
129
|
isFallbackTrack: resolved?.isFallback ?? false,
|
|
130
|
+
decodable: resolved ? fitsBeatAnalysis(resolved.element) : false,
|
|
131
131
|
};
|
|
132
132
|
}, [elements]);
|
|
133
|
+
// Read at load, never a reason to load again: a file length arriving for the same track must keep its edits.
|
|
134
|
+
const decodableRef = useRef(decodable);
|
|
135
|
+
decodableRef.current = decodable;
|
|
133
136
|
|
|
134
|
-
// ── Load:
|
|
135
|
-
// otherwise seed it from detection. Resets edits + history on track change. ──
|
|
137
|
+
// ── Load: the track's beats (loadTrack). Resets edits + history on track change. ──
|
|
136
138
|
useEffect(() => {
|
|
137
139
|
if (!musicSrc) {
|
|
138
140
|
setBeatAnalysis(null);
|
|
@@ -150,26 +152,9 @@ export function useMusicBeatAnalysis(): void {
|
|
|
150
152
|
const beatPath = beatFilePathForSrc(musicSrc);
|
|
151
153
|
const io = ioRef.current;
|
|
152
154
|
|
|
153
|
-
// For explicitly tagged/named music tracks: only run expensive audio decode
|
|
154
|
-
// + beat analysis when the user has an explicit beats file saved. Without
|
|
155
|
-
// one, skip entirely — no surprise green lines on the timeline after
|
|
156
|
-
// dragging unrelated assets.
|
|
157
|
-
//
|
|
158
|
-
// For fallback tracks (audio dropped from Finder with no role/music-id):
|
|
159
|
-
// always run analysis so the Beat tool becomes usable immediately.
|
|
160
155
|
(async () => {
|
|
161
156
|
if (!beatPath || !io) return;
|
|
162
|
-
|
|
163
|
-
const hasSavedBeats = await readHasSavedBeats(io, beatPath);
|
|
164
|
-
if (cancelled) return;
|
|
165
|
-
if (!hasSavedBeats) {
|
|
166
|
-
setBeatAnalysis(null);
|
|
167
|
-
return;
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
if (cancelled) return;
|
|
171
|
-
|
|
172
|
-
const result = await loadBeatAnalysis(musicSrc, beatPath, io);
|
|
157
|
+
const result = await loadTrack(musicSrc, beatPath, io, isFallbackTrack, decodableRef.current);
|
|
173
158
|
if (cancelled) return;
|
|
174
159
|
if (!result) {
|
|
175
160
|
setBeatAnalysis(null);
|
|
@@ -724,3 +724,87 @@ it("stores DOM targets in existing history and retains them through undo and red
|
|
|
724
724
|
await act(() => hook().redo({ readFile }));
|
|
725
725
|
expect(history.list().at(-1)?.targets).toEqual(targets);
|
|
726
726
|
});
|
|
727
|
+
|
|
728
|
+
it("three quick drags are three entries in the list as soon as each drag's saves are in", async () => {
|
|
729
|
+
const { hook, save, history } = await studio();
|
|
730
|
+
const moves = [
|
|
731
|
+
["A", "B"],
|
|
732
|
+
["B", "C"],
|
|
733
|
+
["C", "D"],
|
|
734
|
+
] as const;
|
|
735
|
+
for (const [i, [before, after]] of moves.entries()) {
|
|
736
|
+
// A drag claims under its own key with no idle limit, as a plain move does, and settles after its save.
|
|
737
|
+
const drag = beginStudioPendingEdit(() => () => {});
|
|
738
|
+
save(after);
|
|
739
|
+
const saved = hook().recordEdit({
|
|
740
|
+
label: "Move layer",
|
|
741
|
+
coalesceKey: `move:${i}`,
|
|
742
|
+
coalesceMs: Number.POSITIVE_INFINITY,
|
|
743
|
+
files: { "index.html": { before, after } },
|
|
744
|
+
});
|
|
745
|
+
drag.settle(saved);
|
|
746
|
+
await act(() => saved);
|
|
747
|
+
}
|
|
748
|
+
await vi.waitFor(() =>
|
|
749
|
+
expect(history.list().filter((entry) => entry.label === "Move layer")).toHaveLength(3),
|
|
750
|
+
);
|
|
751
|
+
});
|
|
752
|
+
|
|
753
|
+
it("the next drag's claim reaches the server after the settled drag's empty claim, however slow that one is", async () => {
|
|
754
|
+
const { hook, save } = await studio();
|
|
755
|
+
const forward = globalThis.fetch;
|
|
756
|
+
const arrived: string[] = [];
|
|
757
|
+
let release = () => {};
|
|
758
|
+
const held = new Promise<void>((resolve) => (release = resolve));
|
|
759
|
+
vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => {
|
|
760
|
+
const claim = url.endsWith("/claim")
|
|
761
|
+
? (JSON.parse(String(init?.body)) as { paths: string[] })
|
|
762
|
+
: null;
|
|
763
|
+
if (claim && !claim.paths.length) await held;
|
|
764
|
+
if (claim) arrived.push(claim.paths.length ? "claim" : "empty");
|
|
765
|
+
return forward(url, init);
|
|
766
|
+
});
|
|
767
|
+
const drag = (i: number, before: string, after: string) => {
|
|
768
|
+
const pending = beginStudioPendingEdit(() => () => {});
|
|
769
|
+
save(after);
|
|
770
|
+
const saved = hook().recordEdit({
|
|
771
|
+
label: "Move layer",
|
|
772
|
+
coalesceKey: `move:${i}`,
|
|
773
|
+
coalesceMs: Number.POSITIVE_INFINITY,
|
|
774
|
+
files: { "index.html": { before, after } },
|
|
775
|
+
});
|
|
776
|
+
pending.settle(saved);
|
|
777
|
+
return saved;
|
|
778
|
+
};
|
|
779
|
+
await act(() => drag(0, "A", "B"));
|
|
780
|
+
const next = drag(1, "B", "C");
|
|
781
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
782
|
+
release();
|
|
783
|
+
await act(() => next);
|
|
784
|
+
expect(arrived.slice(0, 3)).toEqual(["claim", "empty", "claim"]);
|
|
785
|
+
});
|
|
786
|
+
|
|
787
|
+
it("a gesture that writes twice under one key stays one entry once it settles", async () => {
|
|
788
|
+
const { hook, save, history } = await studio();
|
|
789
|
+
const resize = beginStudioPendingEdit(() => () => {});
|
|
790
|
+
const writes = (async () => {
|
|
791
|
+
for (const [before, after] of [
|
|
792
|
+
["A", "B"],
|
|
793
|
+
["B", "C"],
|
|
794
|
+
] as const) {
|
|
795
|
+
save(after);
|
|
796
|
+
await hook().recordEdit({
|
|
797
|
+
label: "Resize layer",
|
|
798
|
+
coalesceKey: "tx:resize",
|
|
799
|
+
coalesceMs: Number.POSITIVE_INFINITY,
|
|
800
|
+
files: { "index.html": { before, after } },
|
|
801
|
+
});
|
|
802
|
+
}
|
|
803
|
+
})();
|
|
804
|
+
resize.settle(writes);
|
|
805
|
+
await act(() => writes);
|
|
806
|
+
await vi.waitFor(() =>
|
|
807
|
+
expect(history.list().filter((entry) => entry.label === "Resize layer")).toHaveLength(1),
|
|
808
|
+
);
|
|
809
|
+
expect(history.list().find((entry) => entry.label === "Resize layer")?.files.length).toBe(1);
|
|
810
|
+
});
|