@hyperframes/studio 0.8.139 → 0.8.141

package/src/player/lib/thumbnailVideoDecoder.test.ts

@@ -39,10 +39,20 @@ function recordDecodes(decoded: number[][]): void {
   });
 }
 
+const endOf = (start: number, duration: number) => start + Math.max(0, duration - 0.001);
+
+// Decoded frames are shared per source for the module's life, so each test decodes its own file.
+let sources = 0;
+let source = "";
+
 beforeEach(() => {
+  source = `/clip-${++sources}.mp4`;
   vi.clearAllMocks();
   getKeyPacket.mockImplementation(async () => null);
-  vi.spyOn(URL, "createObjectURL").mockReturnValueOnce("blob:one").mockReturnValueOnce("blob:two");
+  vi.spyOn(URL, "createObjectURL")
+    .mockReturnValueOnce("blob:one")
+    .mockReturnValueOnce("blob:two")
+    .mockReturnValueOnce("blob:end");
   vi.spyOn(URL, "revokeObjectURL").mockImplementation(() => {});
   HTMLCanvasElement.prototype.toBlob = function toBlob(callback) {
     callback(new Blob(["frame"], { type: "image/jpeg" }));
@@ -55,9 +65,15 @@ beforeEach(() => {
 });
 
 describe("videoThumbnailTimestamps", () => {
-  it("uses the midpoint for a poster and sorted sparse points for a strip", () => {
+  it("uses the midpoint for a poster, and each slice's left edge then the end for a strip", () => {
     expect(videoThumbnailTimestamps(2, 6, 1)).toEqual([5]);
-    expect(videoThumbnailTimestamps(2, 6, 4)).toEqual([2, 4, 6, 8]);
+    expect(videoThumbnailTimestamps(2, 6, 4)).toEqual([2, 3.5, 5, 6.5, endOf(2, 6)]);
+  });
+
+  it("nests each strip in the strip twice as long, and the poster in every strip", () => {
+    const strip = videoThumbnailTimestamps(0, 8, 4);
+    expect(videoThumbnailTimestamps(0, 8, 8).filter((_, i) => i % 2 === 0)).toEqual(strip);
+    expect(strip).toContain(videoThumbnailTimestamps(0, 8, 1)[0]);
   });
 
   it("clamps invalid source ranges", () => {
@@ -71,31 +87,31 @@ describe("decodeVideoThumbnail", () => {
     const decoded: number[][] = [];
     recordDecodes(decoded);
     const result = await decodeVideoThumbnail(
-      { source: "/clip.mp4", sourceStart: 2, sourceRangeDuration: 6, frameCount: 2 },
+      { source, sourceStart: 2, sourceRangeDuration: 6, frameCount: 2 },
       new AbortController().signal,
     );
 
-    expect(decoded).toEqual([[2, 8]]);
+    expect(decoded).toEqual([[2, 5, endOf(2, 6)]]);
     expect(result.value).toEqual({
       kind: "filmstrip",
-      urls: ["blob:one", "blob:two"],
+      urls: ["blob:one", "blob:two", "blob:end"],
       aspect: 9 / 16,
     });
     result.dispose?.();
     result.dispose?.();
-    expect(URL.revokeObjectURL).toHaveBeenCalledTimes(2);
+    expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3);
     expect(dispose).toHaveBeenCalledTimes(1);
   });
 
   it("decodes each strip frame at its keyframe unless that keyframe is before the clip's range", async () => {
-    getKeyPacket.mockImplementation(async (time) => ({ timestamp: Math.floor(time / 4) * 4 }));
+    getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 0.5 }));
     const decoded: number[][] = [];
     recordDecodes(decoded);
     await decodeVideoThumbnail(
-      { source: "/clip.mp4", sourceStart: 2, sourceRangeDuration: 8, frameCount: 3 },
+      { source, sourceStart: 2, sourceRangeDuration: 8, frameCount: 4 },
       new AbortController().signal,
     );
-    expect(decoded).toEqual([[2, 4, 8]]);
+    expect(decoded).toEqual([[2, 3.5, 5.5, 7.5, endOf(2, 8)]]);
   });
 
   it("keeps a slot's own time when its keyframe is more than half a slot earlier", async () => {
@@ -103,14 +119,10 @@ describe("decodeVideoThumbnail", () => {
     const decoded: number[][] = [];
     recordDecodes(decoded);
     await decodeVideoThumbnail(
-      { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },
-      new AbortController().signal,
-    );
-    await decodeVideoThumbnail(
-      { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 1 },
+      { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 4 },
       new AbortController().signal,
     );
-    expect(decoded).toEqual([[0, 5, 10], [5]]);
+    expect(decoded).toEqual([[0, 2.5, 5, 7.5, endOf(0, 10)]]);
   });
 
   it("looks up each keyframe just before decoding it", async () => {
@@ -126,10 +138,11 @@ describe("decodeVideoThumbnail", () => {
       }
     });
     await decodeVideoThumbnail(
-      { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },
+      { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 2 },
       new AbortController().signal,
     );
-    expect(events).toEqual(["key 0", "frame 0", "key 5", "frame 5", "key 10", "frame 10"]);
+    const end = endOf(0, 10);
+    expect(events).toEqual(["key 0", "frame 0", "key 5", "frame 5", `key ${end}`, `frame ${end}`]);
   });
 
   it("ends the decode times without throwing when cancelled during a keyframe lookup", async () => {
@@ -146,7 +159,7 @@ describe("decodeVideoThumbnail", () => {
     });
     await expect(
       decodeVideoThumbnail(
-        { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },
+        { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },
         controller.signal,
       ),
     ).rejects.toThrow("Aborted");
@@ -164,10 +177,198 @@ describe("decodeVideoThumbnail", () => {
     const decoded: number[][] = [];
     recordDecodes(decoded);
     await decodeVideoThumbnail(
-      { source: "/clip.mp4", sourceStart: 5, sourceRangeDuration: 5, frameCount: 3 },
+      { source, sourceStart: 5, sourceRangeDuration: 5, frameCount: 2 },
       new AbortController().signal,
     );
-    expect(decoded).toEqual([[5, 7.5, 10]]);
+    expect(decoded).toEqual([[5, 7.5, endOf(5, 5)]]);
+  });
+
+  it("reuses the frames a strip of the same source shows, so a zoom decodes only new times", async () => {
+    vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`);
+    const decoded: number[][] = [];
+    recordDecodes(decoded);
+    const range = { source, sourceStart: 0, sourceRangeDuration: 8 };
+    const signal = new AbortController().signal;
+    const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal);
+    // Twice the frames decodes only the new half; half the frames decodes nothing. All share the end.
+    const eight = await decodeVideoThumbnail({ ...range, frameCount: 8 }, signal);
+    const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal);
+    expect(decoded).toEqual([
+      [0, 2, 4, 6, endOf(0, 8)],
+      [1, 3, 5, 7],
+    ]);
+    const urls = (result: typeof four) =>
+      result.value.kind === "filmstrip" ? result.value.urls : [];
+    expect(urls(eight).filter((_, i) => i % 2 === 0)).toEqual(urls(four));
+    expect(urls(two)).toEqual([urls(four)[0], urls(four)[2], urls(four)[4]]);
+    for (const result of [four, eight, two]) result.dispose?.();
+  });
+
+  it("decodes a frame again when the one shared for its time sits too far before it", async () => {
+    // Every keyframe is 2 s before the time asked: a 2-frame strip of 8 s accepts that (its
+    // frames stand for 4 s each), a 4-frame strip does not, or its frames would run backwards.
+    getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 2 }));
+    const decoded: number[][] = [];
+    recordDecodes(decoded);
+    const range = { source, sourceStart: 0, sourceRangeDuration: 8 };
+    const signal = new AbortController().signal;
+    const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal);
+    const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal);
+    expect(decoded).toEqual([
+      [0, 2, endOf(0, 8)],
+      [2, 4, 6],
+    ]);
+    two.dispose?.();
+    four.dispose?.();
+  });
+
+  it("returns a strip whose frames are all shared without opening the file again", async () => {
+    recordDecodes([]);
+    const range = { source, sourceStart: 0, sourceRangeDuration: 8 };
+    const signal = new AbortController().signal;
+    const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal);
+    const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal);
+    expect(input.getPrimaryVideoTrack).toHaveBeenCalledTimes(1);
+    four.dispose?.();
+    two.dispose?.();
+  });
+
+  it("reopens the same asset after a project round trip and keeps new-session frames separate", async () => {
+    let frame = 0;
+    vi.mocked(URL.createObjectURL)
+      .mockReset()
+      .mockImplementation(() => `blob:version-${++frame}`);
+    const decoded: number[][] = [];
+    recordDecodes(decoded);
+    const signal = new AbortController().signal;
+    const original = await decodeVideoThumbnail(
+      { source, contentVersion: "A:1", frameCount: 1 },
+      signal,
+    );
+    const other = await decodeVideoThumbnail(
+      { source: `${source}?B`, contentVersion: "B:2", frameCount: 1 },
+      signal,
+    );
+    input.getPrimaryVideoTrack.mockResolvedValue({
+      getDisplayWidth: vi.fn(async () => 1280),
+      getDisplayHeight: vi.fn(async () => 360),
+      getDurationFromMetadata: vi.fn(async () => 10),
+    });
+    const reopened = await decodeVideoThumbnail(
+      { source, contentVersion: "A:3", frameCount: 1 },
+      signal,
+    );
+    const shared = await decodeVideoThumbnail(
+      { source, contentVersion: "A:3", frameCount: 1 },
+      signal,
+    );
+    expect(reopened.value).toEqual({ kind: "image", url: "blob:version-3", aspect: 32 / 9 });
+    expect(shared.value).toEqual(reopened.value);
+    expect(decoded).toEqual([[5], [5], [5]]);
+    expect(input.getPrimaryVideoTrack).toHaveBeenCalledTimes(3);
+    original.dispose?.();
+    expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:version-1");
+    expect(URL.revokeObjectURL).not.toHaveBeenCalledWith("blob:version-3");
+    other.dispose?.();
+    reopened.dispose?.();
+    expect(URL.revokeObjectURL).not.toHaveBeenCalledWith("blob:version-3");
+    shared.dispose?.();
+    expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3);
+  });
+
+  it("keeps one copy of a frame two strips decode at once, and revokes the other", async () => {
+    vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`);
+    // The first strip holds its first frame until the second is decoding too.
+    let release!: () => void;
+    const bothDecoding = new Promise<void>((resolve) => (release = resolve));
+    canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable<number>) {
+      const first = canvasesAtTimestamps.mock.calls.length === 1;
+      for await (const _time of timestamps) {
+        if (first) await bothDecoding;
+        else release();
+        yield { canvas: document.createElement("canvas") };
+      }
+    });
+    const range = { source, sourceStart: 0, sourceRangeDuration: 8 };
+    const signal = new AbortController().signal;
+    const decodingTwo = decodeVideoThumbnail({ ...range, frameCount: 2 }, signal);
+    await vi.waitFor(() => expect(canvasesAtTimestamps).toHaveBeenCalledOnce());
+    const [two, four] = await Promise.all([
+      decodingTwo,
+      decodeVideoThumbnail({ ...range, frameCount: 4 }, signal),
+    ]);
+    // Both decoded the frames at 0 s, 4 s and the end; the second copy of each goes at once.
+    expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3);
+    two.dispose?.();
+    expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3);
+    four.dispose?.();
+    expect(URL.revokeObjectURL).toHaveBeenCalledTimes(8);
+  });
+
+  it("fills a frame that failed to decode from the one before it, so tiles stay on their times", async () => {
+    canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable<number>) {
+      let slot = 0;
+      for await (const _time of timestamps) {
+        yield slot++ === 2 ? null : { canvas: document.createElement("canvas") };
+      }
+    });
+    const signal = new AbortController().signal;
+    const result = await decodeVideoThumbnail(
+      { source, sourceStart: 0, sourceRangeDuration: 8, frameCount: 2 },
+      signal,
+    );
+    expect(result.value).toMatchObject({
+      kind: "filmstrip",
+      urls: ["blob:one", "blob:two", "blob:two"],
+    });
+    // The cache is charged for the two decoded frames, not the filled copy.
+    recordDecodes([]);
+    const whole = await decodeVideoThumbnail(
+      { source: `${source}?whole`, sourceStart: 0, sourceRangeDuration: 8, frameCount: 2 },
+      signal,
+    );
+    expect(result.weight * 3).toBe(whole.weight * 2);
+    result.dispose?.();
+    whole.dispose?.();
+  });
+
+  it("snaps a poster up to a quarter of its range and a strip frame up to half a slice", async () => {
+    const decoded: number[][] = [];
+    recordDecodes(decoded);
+    const signal = new AbortController().signal;
+    getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 2.4 }));
+    await decodeVideoThumbnail(
+      { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 1 },
+      signal,
+    );
+    getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 2.6 }));
+    await decodeVideoThumbnail(
+      { source: `${source}?b`, sourceStart: 0, sourceRangeDuration: 10, frameCount: 1 },
+      signal,
+    );
+    // Four slices of 8 s: a keyframe 0.9 s early snaps, one 1.1 s early does not.
+    getKeyPacket.mockImplementation(async (time) => ({
+      timestamp: time - (time === 4 ? 0.9 : 1.1),
+    }));
+    await decodeVideoThumbnail(
+      { source: `${source}?c`, sourceStart: 0, sourceRangeDuration: 8, frameCount: 4 },
+      signal,
+    );
+    expect(decoded).toEqual([[5 - 2.4], [5], [0, 2, 4 - 0.9, 6, endOf(0, 8)]]);
+  });
+
+  it("revokes a shared frame only once no strip shows it", async () => {
+    vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`);
+    recordDecodes([]);
+    const range = { source, sourceStart: 0, sourceRangeDuration: 8 };
+    const signal = new AbortController().signal;
+    const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal);
+    const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal);
+    four.dispose?.();
+    // The three frames the 2-slice strip still shows stay; the other two go.
+    expect(URL.revokeObjectURL).toHaveBeenCalledTimes(2);
+    two.dispose?.();
+    expect(URL.revokeObjectURL).toHaveBeenCalledTimes(5);
   });
 
   it("releases input and degrades when the source has no video track", async () => {
@@ -181,13 +382,15 @@ describe("decodeVideoThumbnail", () => {
   it("revokes partial results when cancellation lands during extraction", async () => {
     const controller = new AbortController();
     const canvas = document.createElement("canvas");
-    canvasesAtTimestamps.mockImplementation(async function* () {
-      yield { canvas, timestamp: 1, duration: 1 };
-      controller.abort();
-      yield { canvas, timestamp: 2, duration: 1 };
+    canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable<number>) {
+      let frames = 0;
+      for await (const timestamp of timestamps) {
+        if (frames++ === 1) controller.abort();
+        yield { canvas, timestamp, duration: 1 };
+      }
     });
     await expect(
-      decodeVideoThumbnail({ source: "/clip.mp4", frameCount: 2 }, controller.signal),
+      decodeVideoThumbnail({ source, frameCount: 2 }, controller.signal),
     ).rejects.toMatchObject({ name: "AbortError" });
     expect(URL.revokeObjectURL).toHaveBeenCalledTimes(1);
     expect(dispose).toHaveBeenCalledTimes(1);
@@ -207,10 +410,7 @@ describe("decodeVideoThumbnail", () => {
       getDurationFromMetadata,
     });
 
-    const decoding = decodeVideoThumbnail(
-      { source: "/clip.mp4", frameCount: 1 },
-      controller.signal,
-    );
+    const decoding = decodeVideoThumbnail({ source, frameCount: 1 }, controller.signal);
     await vi.waitFor(() => expect(getDisplayWidth).toHaveBeenCalledOnce());
     controller.abort();
     resolveWidth(1080);