@editframe/cli 0.11.0-beta.9 → 0.12.0-beta.10

package/src/operations/syncAssetsDirectory/SyncTrack.test.ts

@@ -1,10 +1,12 @@
+import { useMSW } from "TEST/useMSW.ts";
 import { describe, expect, test } from "vitest";
 import { fixture, withFixtures } from "../../../test-fixtures/fixture.ts";
 import {
   mockCreateIsobmffFile,
   mockCreateIsobmffTrack,
   mockGetIsobmffTrackUpload,
-  useMSW,
+  mockLookupISOBMFFFileByMd5,
+  mockLookupISOBMFFFileByMd5NotFound,
 } from "../../../test-fixtures/network.ts";
 import { SyncTrack } from "./SyncTrack.ts";
 
@@ -21,8 +23,28 @@ describe("SyncTrack", async () => {
         await expect(syncTrack.byteSize()).resolves.toEqual(26434);
       });
       describe("prepare()", () => {
+        test("Uses existing ISOBMFF file if md5 matches", async () => {
+          server.use(
+            mockLookupISOBMFFFileByMd5({
+              complete: true,
+              id: "123",
+              md5: video!.md5,
+              fixture: video!,
+            }),
+          );
+          const syncTrack = new SyncTrack(
+            await generateTrack(video!, 1),
+            video!.md5,
+          );
+          await syncTrack.prepare();
+          expect(syncTrack.isoFile).toBeDefined();
+        });
+
         test("Creates ISOBMFF file", async () => {
           server.use(
+            mockLookupISOBMFFFileByMd5NotFound({
+              md5: video!.md5,
+            }),
             mockCreateIsobmffFile({
               complete: true,
               id: "123",
@@ -40,6 +62,9 @@ describe("SyncTrack", async () => {
 
         test("Probes track", async () => {
           server.use(
+            mockLookupISOBMFFFileByMd5NotFound({
+              md5: video!.md5,
+            }),
             mockCreateIsobmffFile({
               complete: true,
               id: "123",
@@ -81,6 +106,9 @@ describe("SyncTrack", async () => {
 
         test("isComplete() returns false when not complete", async () => {
           server.use(
+            mockLookupISOBMFFFileByMd5NotFound({
+              md5: video!.md5,
+            }),
             mockCreateIsobmffFile({
               complete: true,
               id: "123",
@@ -106,6 +134,9 @@ describe("SyncTrack", async () => {
 
         test("isComplete() returns true when complete", async () => {
           server.use(
+            mockLookupISOBMFFFileByMd5NotFound({
+              md5: video!.md5,
+            }),
             mockCreateIsobmffFile({
               complete: true,
               id: "123",
@@ -140,6 +171,9 @@ describe("SyncTrack", async () => {
         });
         test("uploads track", async () => {
           server.use(
+            mockLookupISOBMFFFileByMd5NotFound({
+              md5: video!.md5,
+            }),
             mockCreateIsobmffFile({
               complete: true,
               id: "123",
@@ -178,6 +212,9 @@ describe("SyncTrack", async () => {
         });
         test("marks synced", async () => {
           server.use(
+            mockLookupISOBMFFFileByMd5NotFound({
+              md5: video!.md5,
+            }),
             mockCreateIsobmffFile({
               complete: true,
               id: "123",
@@ -203,7 +240,6 @@ describe("SyncTrack", async () => {
             complete: true,
             id: "123:track-1",
             md5: video!.md5,
-            asset_id: `${video!.md5}:test.mp4`,
             byte_size: 26434,
           });
 
@@ -212,7 +248,6 @@ describe("SyncTrack", async () => {
             complete: true,
             id: "123",
             md5: video!.md5,
-            asset_id: `${video!.md5}:test.mp4`,
             byte_size: 26434,
           });
         });

package/src/operations/syncAssetsDirectory/SyncTrack.ts

@@ -1,6 +1,6 @@
 import { createReadStream } from "node:fs";
 import fs from "node:fs/promises";
-import { dirname, join } from "node:path";
+import { basename, dirname, join } from "node:path";
 
 import type { z } from "zod";
 
@@ -8,12 +8,15 @@ import {
   type CreateISOBMFFFileResult,
   type CreateISOBMFFTrackPayload,
   type CreateISOBMFFTrackResult,
+  type LookupISOBMFFFileByMd5Result,
   createISOBMFFFile,
   createISOBMFFTrack,
+  lookupISOBMFFFileByMd5,
   uploadISOBMFFTrack,
 } from "@editframe/api";
 import { Probe } from "@editframe/assets";
 
+import { createReadableStreamFromReadable } from "../../utils/createReadableStreamFromReadable.ts";
 import { getClient } from "../../utils/index.ts";
 import type { SubAssetSync } from "./SubAssetSync.ts";
 import { SyncStatus } from "./SyncStatus.ts";
@@ -21,10 +24,8 @@ import { SyncStatus } from "./SyncStatus.ts";
 export class SyncTrack implements SubAssetSync<CreateISOBMFFTrackResult> {
   icon = "📼";
   label = "track";
-  syncStatus: SyncStatus = new SyncStatus(this.path);
-  fileSyncStatus: SyncStatus = new SyncStatus(
-    join(dirname(this.path), "isobmff"),
-  );
+  syncStatus = new SyncStatus(this.path);
+  fileSyncStatus = new SyncStatus(join(dirname(this.path), "isobmff"));
   created: CreateISOBMFFTrackResult | null = null;
 
   constructor(
@@ -32,7 +33,10 @@ export class SyncTrack implements SubAssetSync<CreateISOBMFFTrackResult> {
     public md5: string,
   ) {}
 
-  private _isoFile: CreateISOBMFFFileResult | null = null;
+  private _isoFile:
+    | CreateISOBMFFFileResult
+    | LookupISOBMFFFileByMd5Result
+    | null = null;
 
   get isoFile() {
     if (this._isoFile) {
@@ -62,10 +66,15 @@ export class SyncTrack implements SubAssetSync<CreateISOBMFFTrackResult> {
   }
 
   async prepare() {
-    this._isoFile = await createISOBMFFFile(getClient(), {
-      md5: this.md5,
-      filename: this.path.replace(/\.track-[\d]+.mp4$/, ""),
-    });
+    const maybeIsoFile = await lookupISOBMFFFileByMd5(getClient(), this.md5);
+    if (maybeIsoFile) {
+      this._isoFile = maybeIsoFile;
+    } else {
+      this._isoFile = await createISOBMFFFile(getClient(), {
+        md5: this.md5,
+        filename: basename(this.path).replace(/\.track-[\d]+.mp4$/, ""),
+      });
+    }
     this._probeResult = await Probe.probePath(this.path);
   }
 
@@ -131,7 +140,7 @@ export class SyncTrack implements SubAssetSync<CreateISOBMFFTrackResult> {
       getClient(),
       this.isoFile.id,
       Number(this.trackId),
-      createReadStream(this.path),
+      createReadableStreamFromReadable(createReadStream(this.path)),
       this.created?.byte_size,
     ).whenUploaded();
   }
@@ -148,7 +157,6 @@ export class SyncTrack implements SubAssetSync<CreateISOBMFFTrackResult> {
         complete: true,
         id: `${this.created.file_id}:${this.created.track_id}`,
         md5: this.md5,
-        asset_id: this.created.asset_id,
         byte_size: byteSize,
       }),
       this.fileSyncStatus.markSynced({
@@ -156,7 +164,6 @@ export class SyncTrack implements SubAssetSync<CreateISOBMFFTrackResult> {
         complete: true,
         id: this.created.file_id,
         md5: this.md5,
-        asset_id: this.created.asset_id,
         byte_size: byteSize,
       }),
     ]);

package/src/operations/syncAssetsDirectory.test.ts

@@ -1,13 +1,15 @@
 import { describe, it } from "vitest";
 
+import { useMSW } from "TEST/useMSW.ts";
 import { fixture, withFixtures } from "../../test-fixtures/fixture.ts";
 import {
   mockCreateImageFile,
   mockCreateIsobmffFile,
   mockCreateIsobmffTrack,
   mockGetUploadImageFile,
+  mockLookupISOBMFFFileByMd5NotFound,
+  mockLookupImageFileByMd5NotFound,
   mockUploadIsobmffFileIndex,
-  useMSW,
 } from "../../test-fixtures/network.ts";
 
 describe("syncAssetsDirectory", () => {
@@ -25,6 +27,9 @@ describe("syncAssetsDirectory", () => {
           cacheImage,
         }) => {
           server.use(
+            mockLookupImageFileByMd5NotFound({
+              md5: testPng!.md5,
+            }),
             mockCreateImageFile({
               complete: true,
               id: "123",
@@ -55,6 +60,9 @@ describe("syncAssetsDirectory", () => {
           syncAssetsDirectory,
         }) => {
           server.use(
+            mockLookupImageFileByMd5NotFound({
+              md5: testPng!.md5,
+            }),
             mockCreateImageFile({
               complete: false,
               id: "123",
@@ -91,6 +99,9 @@ describe("syncAssetsDirectory", () => {
           expectInfoFileContent,
         }) => {
           server.use(
+            mockLookupImageFileByMd5NotFound({
+              md5: testPng!.md5,
+            }),
             mockCreateImageFile({
               complete: false,
               id: "123",
@@ -160,6 +171,10 @@ describe("syncAssetsDirectory", () => {
             fixture: testMp4!,
           };
           server.use(
+            mockLookupISOBMFFFileByMd5NotFound({
+              md5: testMp4!.md5,
+            }),
+
             mockCreateIsobmffFile(creationArgs),
 
             mockCreateIsobmffTrack({
@@ -247,6 +262,9 @@ describe("syncAssetsDirectory", () => {
             fixture: testMp4!,
           };
           server.use(
+            mockLookupISOBMFFFileByMd5NotFound({
+              md5: testMp4!.md5,
+            }),
             mockCreateIsobmffFile(creationArgs),
             mockUploadIsobmffFileIndex({
               id: "123",
@@ -326,6 +344,9 @@ describe("syncAssetsDirectory", () => {
           expectCacheFiles,
         }) => {
           server.use(
+            mockLookupISOBMFFFileByMd5NotFound({
+              md5: testMp4!.md5,
+            }),
             mockCreateIsobmffFile({
               complete: true,
               id: "123",

package/src/utils/createReadableStreamFromReadable.ts

@@ -0,0 +1,117 @@
+import { type Readable, Stream } from "node:stream";
+
+export const createReadableStreamFromReadable = (
+  source: Readable & { readableHighWaterMark?: number },
+) => {
+  const pump = new StreamPump(source);
+  const stream = new ReadableStream(pump, pump);
+  return stream;
+};
+
+class StreamPump {
+  public highWaterMark: number;
+  public accumalatedSize: number;
+  private stream: Stream & {
+    readableHighWaterMark?: number;
+    readable?: boolean;
+    resume?: () => void;
+    pause?: () => void;
+    destroy?: (error?: Error) => void;
+  };
+  private controller?: ReadableStreamController<Uint8Array>;
+
+  constructor(
+    stream: Stream & {
+      readableHighWaterMark?: number;
+      readable?: boolean;
+      resume?: () => void;
+      pause?: () => void;
+      destroy?: (error?: Error) => void;
+    },
+  ) {
+    this.highWaterMark =
+      stream.readableHighWaterMark ||
+      new Stream.Readable().readableHighWaterMark;
+    this.accumalatedSize = 0;
+    this.stream = stream;
+    this.enqueue = this.enqueue.bind(this);
+    this.error = this.error.bind(this);
+    this.close = this.close.bind(this);
+  }
+
+  size(chunk: Uint8Array) {
+    return chunk?.byteLength || 0;
+  }
+
+  start(controller: ReadableStreamController<Uint8Array>) {
+    this.controller = controller;
+    this.stream.on("data", this.enqueue);
+    this.stream.once("error", this.error);
+    this.stream.once("end", this.close);
+    this.stream.once("close", this.close);
+  }
+
+  pull() {
+    this.resume();
+  }
+
+  cancel(reason?: Error) {
+    if (this.stream.destroy) {
+      this.stream.destroy(reason);
+    }
+
+    this.stream.off("data", this.enqueue);
+    this.stream.off("error", this.error);
+    this.stream.off("end", this.close);
+    this.stream.off("close", this.close);
+  }
+
+  enqueue(chunk: Uint8Array | string) {
+    if (this.controller) {
+      try {
+        const bytes = chunk instanceof Uint8Array ? chunk : Buffer.from(chunk);
+
+        const available = (this.controller.desiredSize || 0) - bytes.byteLength;
+        this.controller.enqueue(bytes);
+        if (available <= 0) {
+          this.pause();
+        }
+      } catch (error: any) {
+        this.controller.error(
+          new Error(
+            "Could not create Buffer, chunk must be of type string or an instance of Buffer, ArrayBuffer, or Array or an Array-like Object",
+          ),
+        );
+        this.cancel();
+      }
+    }
+  }
+
+  pause() {
+    if (this.stream.pause) {
+      this.stream.pause();
+    }
+  }
+
+  resume() {
+    if (this.stream.readable && this.stream.resume) {
+      this.stream.resume();
+    }
+  }
+
+  close() {
+    if (this.controller) {
+      this.controller.close();
+      // biome-ignore lint/performance/noDelete: infrequent use
+      delete this.controller;
+    }
+  }
+
+  error(error: Error) {
+    if (this.controller) {
+      this.controller.error(error);
+      // biome-ignore lint/performance/noDelete: infrequent use
+      delete this.controller;
+    }
+  }
+}

package/src/utils/launchBrowserAndWaitForSDK.ts

@@ -1,6 +1,6 @@
 import chalk from "chalk";
-import { type Browser, type Page, chromium } from "playwright";
 import debug from "debug";
+import { type Browser, type Page, chromium } from "playwright";
 
 import { withSpinner } from "./withSpinner.ts";
 
@@ -26,7 +26,7 @@ export async function launchBrowserAndWaitForSDK(
     });
   });
 
-  const page = await withSpinner("Loading EditFrame SDK", async () => {
+  const page = await withSpinner("Loading Editframe SDK", async () => {
     const pageOptions: Parameters<Browser["newPage"]>[0] = {};
     if (options.interactive === true) {
       // By default, playwright uses its own viewport, so resizing the browser window

package/src/utils/startDevServer.ts

@@ -1,7 +1,7 @@
 import path from "node:path";
 
-import { type ViteDevServer, createServer } from "vite";
 import tailwindcss from "tailwindcss";
+import { type ViteDevServer, createServer } from "vite";
 
 import { vitePluginEditframe as editframe } from "@editframe/vite-plugin";
 

package/test-fixtures/fixture.ts

@@ -80,7 +80,6 @@ export const withFixtures = async (
       const filePath = join(cacheDir, fixture.md5, fileName);
       const content = JSON.parse(await readFile(filePath, "utf-8"));
       expect(content).toMatchObject({
-        asset_id: `${fixture.md5}:${fixture.name}`,
         md5: fixture.md5,
         version: "1",
         byte_size: fixture.content.length,

package/test-fixtures/network.ts

@@ -1,44 +1,42 @@
 import { http, HttpResponse } from "msw";
-import { setupServer } from "msw/node";
-import { afterAll, afterEach, beforeAll } from "vitest";
 import type { Fixture } from "./fixture.ts";
 
-const server = setupServer();
-
-export const useMSW = () => {
-  beforeAll(() => {
-    server.listen();
-    process.env.EF_TOKEN = "ef_SECRET_TOKEN";
-    process.env.EF_HOST = "http://localhost:3000";
-  });
-  afterAll(() => server.close());
-  afterEach(() => server.resetHandlers());
-  return server;
-};
-
 export const mockCreateImageFile = ({
   complete = true,
   id = "123",
-  filename = "test.png",
-  fixture,
 }: { complete?: boolean; id?: string; filename?: string; fixture: Fixture }) =>
   http.post(
     "http://localhost:3000/api/v1/image_files",
     async () => {
       return HttpResponse.json({
         id,
-        asset_id: `${fixture.md5}:${filename}`,
         complete,
       });
     },
     { once: true },
   );
 
+export const mockLookupImageFileByMd5 = ({
+  md5 = "test-md5",
+}: { complete?: boolean; id?: string; md5?: string; fixture: Fixture }) =>
+  http.get(`http://localhost:3000/api/v1/image_files/md5/${md5}`, async () => {
+    return HttpResponse.json({
+      id: "123",
+      complete: true,
+    });
+  });
+
+export const mockLookupImageFileByMd5NotFound = ({
+  md5 = "test-md5",
+}: { md5?: string }) =>
+  http.get(`http://localhost:3000/api/v1/image_files/md5/${md5}`, async () => {
+    return HttpResponse.json({}, { status: 404 });
+  });
+
 export const mockGetUploadImageFile = ({
   complete = true,
   id = "123",
   filename = "test.png",
-  fixture,
 }: { complete?: boolean; id?: string; filename?: string; fixture: Fixture }) =>
   http.get(
     `http://localhost:3000/api/v1/image_files/${id}/upload`,
@@ -47,7 +45,6 @@ export const mockGetUploadImageFile = ({
         id,
         complete,
         filename,
-        asset_id: `${fixture.md5}:${filename}`,
       });
     },
   );
@@ -55,8 +52,6 @@ export const mockGetUploadImageFile = ({
 export const mockCreateIsobmffFile = ({
   complete = true,
   id = "123",
-  filename = "test.mp4",
-  fixture,
 }: {
   complete?: boolean;
   id?: string;
@@ -68,19 +63,46 @@ export const mockCreateIsobmffFile = ({
     async () => {
       return HttpResponse.json({
         id,
-        asset_id: `${fixture.md5}:${filename}`,
         fragment_index_complete: complete,
       });
     },
     { once: true },
   );
 
+export const mockLookupISOBMFFFileByMd5 = ({
+  complete = true,
+  id = "123",
+  md5 = "test-md5",
+}: {
+  complete?: boolean;
+  id?: string;
+  md5?: string;
+  fixture: Fixture;
+}) =>
+  http.get(
+    `http://localhost:3000/api/v1/isobmff_files/md5/${md5}`,
+    async () => {
+      return HttpResponse.json({
+        id,
+        complete,
+      });
+    },
+  );
+
+export const mockLookupISOBMFFFileByMd5NotFound = ({
+  md5 = "test-md5",
+}: { md5?: string }) =>
+  http.get(
+    `http://localhost:3000/api/v1/isobmff_files/md5/${md5}`,
+    async () => {
+      return HttpResponse.json({}, { status: 404 });
+    },
+  );
+
 export const mockCreateIsobmffTrack = ({
   complete = true,
   id = "123",
   fileId = "123",
-  filename = "test.mp4",
-  fixture,
 }: {
   complete?: boolean;
   id?: string;
@@ -93,7 +115,6 @@ export const mockCreateIsobmffTrack = ({
     async () => {
       return HttpResponse.json({
         id,
-        asset_id: `${fixture.md5}:${filename}`,
         complete,
         file_id: fileId,
         track_id: id,
@@ -137,11 +158,34 @@ export const mockUploadIsobmffFileIndex = ({
     { once: true },
   );
 
+export const mockLookupCaptionFileByMd5 = ({
+  complete = true,
+  id = "123",
+  md5 = "test-md5",
+}: { complete?: boolean; id?: string; md5?: string; fixture: Fixture }) =>
+  http.get(
+    `http://localhost:3000/api/v1/caption_files/md5/${md5}`,
+    async () => {
+      return HttpResponse.json({
+        id,
+        complete,
+      });
+    },
+  );
+
+export const mockLookupCaptionFileByMd5NotFound = ({
+  md5 = "test-md5",
+}: { md5?: string }) =>
+  http.get(
+    `http://localhost:3000/api/v1/caption_files/md5/${md5}`,
+    async () => {
+      return HttpResponse.json({}, { status: 404 });
+    },
+  );
+
 export const mockCreateCaptionFile = ({
   complete = true,
   id = "123",
-  filename = "test.captions.json",
-  fixture,
 }: {
   complete?: boolean;
   id?: string;
@@ -154,7 +198,6 @@ export const mockCreateCaptionFile = ({
       return HttpResponse.json({
         id,
         complete,
-        asset_id: `${fixture.md5}:${filename}`,
       });
     },
     { once: true },