@editframe/cli 0.11.0-beta.8 → 0.12.0-beta.1

package/src/commands/process-file.ts

@@ -1,18 +1,22 @@
-import { basename } from "node:path";
-import { stat } from "node:fs/promises";
 import { createReadStream } from "node:fs";
+import { stat } from "node:fs/promises";
+import { basename } from "node:path";
 
 import { program } from "commander";
 
 import { withSpinner } from "../utils/withSpinner.ts";
 
-import { getClient } from "../utils/index.ts";
 import {
   createUnprocessedFile,
-  updateUnprocessedFile,
+  getIsobmffProcessInfo,
+  getIsobmffProcessProgress,
+  processIsobmffFile,
   uploadUnprocessedFile,
 } from "@editframe/api";
 import { md5FilePath } from "@editframe/assets";
+import ora from "ora";
+import { createReadableStreamFromReadable } from "../utils/createReadableStreamFromReadable.ts";
+import { getClient } from "../utils/index.ts";
 
 program
   .command("process-file <file>")
@@ -29,7 +33,6 @@ program
       async () =>
         await createUnprocessedFile(client, {
           md5,
-          processes: [],
           filename: basename(path),
           byte_size,
         }),
@@ -37,25 +40,36 @@ program
 
     const readStream = createReadStream(path);
 
-    await withSpinner("Uploading file", async () => {
-      await uploadUnprocessedFile(
-        client,
-        unprocessedFile.id,
-        readStream,
-        byte_size,
-      );
-    });
-
-    const updatedUnprocessedFile = await withSpinner(
+    const upload = uploadUnprocessedFile(
+      client,
+      unprocessedFile.id,
+      createReadableStreamFromReadable(readStream),
+      byte_size,
+    );
+    const uploadSpinner = ora("Uploading file");
+    for await (const event of upload) {
+      uploadSpinner.text = `Uploading file: ${(100 * event.progress).toFixed(2)}%`;
+    }
+    uploadSpinner.succeed("Upload complete");
+    const processor = await withSpinner(
       "Marking for processing",
-      async () => {
-        return await updateUnprocessedFile(client, unprocessedFile.id, {
-          processes: ["isobmff"],
-        });
-      },
+      async () => await processIsobmffFile(client, unprocessedFile.id),
     );
 
-    process.stderr.write("File uploaded and marked for processing.\n");
+    const processSpinner = ora("Waiting for processing to complete");
+    processSpinner.start();
+    const progress = await getIsobmffProcessProgress(client, processor.id);
+
+    for await (const event of progress) {
+      if (event.type === "progress") {
+        processSpinner.text = `Processing: ${(100 * event.data.progress).toFixed(2)}%`;
+      } else if (event.type === "complete") {
+        processSpinner.succeed("Processing complete");
+      }
+    }
+
+    const info = await getIsobmffProcessInfo(client, processor.id);
 
-    console.log(JSON.stringify(updatedUnprocessedFile, null, 2));
+    console.log("Processed file info");
+    console.log(info);
   });

package/src/commands/render.ts

@@ -16,6 +16,7 @@ import { RenderInfo, getRenderInfo } from "../operations/getRenderInfo.ts";
 import { processRenderInfo } from "../operations/processRenderInfo.ts";
 import { syncAssetDirectory } from "../operations/syncAssetsDirectory.ts";
 import { SyncStatus } from "../operations/syncAssetsDirectory/SyncStatus.ts";
+import { createReadableStreamFromReadable } from "../utils/createReadableStreamFromReadable.ts";
 import { getFolderSize } from "../utils/getFolderSize.ts";
 import { getClient } from "../utils/index.ts";
 import { launchBrowserAndWaitForSDK } from "../utils/launchBrowserAndWaitForSDK.ts";
@@ -41,7 +42,7 @@ export const buildAssetId = async (
     throw new Error(`SyncStatus info is not found for ${syncStatus.infoPath}`);
   }
 
-  return info.asset_id;
+  return info.id;
 };
 
 program
@@ -179,8 +180,14 @@ program
         );
         const readable = new PassThrough();
         tarStream.pipe(readable);
+
         const folderSize = await getFolderSize(distDir);
-        await uploadRender(getClient(), render.id, readable, folderSize);
+        await uploadRender(
+          getClient(),
+          render.id,
+          createReadableStreamFromReadable(readable),
+          folderSize,
+        );
         process.stderr.write("Render assets uploaded\n");
         process.stderr.write(inspect(render));
         process.stderr.write("\n");

package/src/operations/syncAssetsDirectory/SubAssetSync.ts

@@ -1,3 +1,4 @@
+import debug from "debug";
 import type { SyncStatus } from "./SyncStatus.ts";
 
 import { SyncCaption } from "./SyncCaption.ts";
@@ -25,7 +26,10 @@ const fragmentIndexMatch = /\.tracks.json$/i;
 const captionsMatch = /\.captions.json$/i;
 const imageMatch = /\.(png|jpe?g|gif|webp)$/i;
 
+const log = debug("ef:SubAssetSync");
+
 export const getAssetSync = (subAssetPath: string, md5: string) => {
+  log("getAssetSync", { subAssetPath, md5 });
   if (imageMatch.test(subAssetPath)) {
     return new SyncImage(subAssetPath, md5);
   }

package/src/operations/syncAssetsDirectory/SyncCaption.test.ts

@@ -1,12 +1,13 @@
+import { useMSW } from "TEST/useMSW.ts";
 import { describe, expect, test } from "vitest";
 import { fixture, withFixtures } from "../../../test-fixtures/fixture.ts";
 import {
   mockCreateCaptionFile,
+  mockLookupCaptionFileByMd5,
+  mockLookupCaptionFileByMd5NotFound,
   mockUploadCaptionFile,
-  useMSW,
 } from "../../../test-fixtures/network.ts";
 import { SyncCaption } from "./SyncCaption.ts";
-
 describe("SyncCaption", async () => {
   const server = useMSW();
   await withFixtures(
@@ -37,8 +38,25 @@ describe("SyncCaption", async () => {
       });
 
       describe(".create()", () => {
+        test("uses matching data when file matches md5", async () => {
+          server.use(
+            mockLookupCaptionFileByMd5({
+              md5: video!.md5,
+              fixture: video!,
+            }),
+          );
+          const syncCaption = new SyncCaption(
+            await generateCaptions(video!),
+            video!.md5,
+          );
+          await syncCaption.create();
+          expect(syncCaption.isComplete()).toBe(true);
+        });
         test("isComplete() returns false when not created", async () => {
           server.use(
+            mockLookupCaptionFileByMd5NotFound({
+              md5: video!.md5,
+            }),
             mockCreateCaptionFile({
               complete: false,
               id: "123",
@@ -56,6 +74,9 @@ describe("SyncCaption", async () => {
 
         test("isComplete() returns true when created", async () => {
           server.use(
+            mockLookupCaptionFileByMd5NotFound({
+              md5: video!.md5,
+            }),
             mockCreateCaptionFile({
               complete: true,
               id: "123",
@@ -83,6 +104,9 @@ describe("SyncCaption", async () => {
 
         test("uploads caption", async () => {
           server.use(
+            mockLookupCaptionFileByMd5NotFound({
+              md5: video!.md5,
+            }),
             mockCreateCaptionFile({
               complete: true,
               id: "123",
@@ -115,6 +139,9 @@ describe("SyncCaption", async () => {
 
         test("marks synced", async () => {
           server.use(
+            mockLookupCaptionFileByMd5NotFound({
+              md5: video!.md5,
+            }),
             mockCreateCaptionFile({
               complete: true,
               id: "123",
@@ -135,7 +162,6 @@ describe("SyncCaption", async () => {
             complete: true,
             id: "123",
             md5: video!.md5,
-            asset_id: `${video!.md5}:test.mp4`,
             byte_size: 35,
           });
         });

package/src/operations/syncAssetsDirectory/SyncCaption.ts

@@ -2,20 +2,24 @@ import fs from "node:fs/promises";
 
 import {
   type CreateCaptionFileResult,
+  type LookupCaptionFileByMd5Result,
   createCaptionFile,
+  lookupCaptionFileByMd5,
   uploadCaptionFile,
 } from "@editframe/api";
 
 import { Readable } from "node:stream";
 import { getClient } from "../../utils/index.ts";
+
+import { basename } from "node:path";
+import { createReadableStreamFromReadable } from "../../utils/createReadableStreamFromReadable.ts";
 import type { SubAssetSync } from "./SubAssetSync.ts";
 import { SyncStatus } from "./SyncStatus.ts";
-
 export class SyncCaption implements SubAssetSync<CreateCaptionFileResult> {
   icon = "📝";
   label = "captions";
   syncStatus: SyncStatus = new SyncStatus(this.path);
-  created: CreateCaptionFileResult | null = null;
+  created: CreateCaptionFileResult | LookupCaptionFileByMd5Result | null = null;
   constructor(
     public path: string,
     public md5: string,
@@ -30,11 +34,19 @@ export class SyncCaption implements SubAssetSync<CreateCaptionFileResult> {
   async validate() {}
 
   async create() {
-    this.created = await createCaptionFile(getClient(), {
-      md5: this.md5,
-      filename: this.path.replace(/\.captions.json$/, ""),
-      byte_size: await this.byteSize(),
-    });
+    const maybeCaptionFile = await lookupCaptionFileByMd5(
+      getClient(),
+      this.md5,
+    );
+    if (maybeCaptionFile) {
+      this.created = maybeCaptionFile;
+    } else {
+      this.created = await createCaptionFile(getClient(), {
+        md5: this.md5,
+        filename: basename(this.path).replace(/\.captions.json$/, ""),
+        byte_size: await this.byteSize(),
+      });
+    }
   }
 
   isComplete() {
@@ -52,7 +64,9 @@ export class SyncCaption implements SubAssetSync<CreateCaptionFileResult> {
       this.created.id,
       // It's not clear why we need to use Readable.from here, but it seems
       // to fix an issue where the request is closed early in tests
-      Readable.from(await fs.readFile(this.path)),
+      createReadableStreamFromReadable(
+        Readable.from(await fs.readFile(this.path)),
+      ),
       await this.byteSize(),
     );
   }
@@ -69,7 +83,6 @@ export class SyncCaption implements SubAssetSync<CreateCaptionFileResult> {
       complete: true,
       id: this.created.id,
       md5: this.md5,
-      asset_id: this.created.asset_id,
       byte_size: byteSize,
     });
   }

package/src/operations/syncAssetsDirectory/SyncFragmentIndex.test.ts

@@ -1,9 +1,11 @@
+import { useMSW } from "TEST/useMSW.ts";
 import { describe, expect, test } from "vitest";
 import { fixture, withFixtures } from "../../../test-fixtures/fixture.ts";
 import {
   mockCreateIsobmffFile,
+  mockLookupISOBMFFFileByMd5,
+  mockLookupISOBMFFFileByMd5NotFound,
   mockUploadIsobmffFileIndex,
-  useMSW,
 } from "../../../test-fixtures/network.ts";
 import { SyncFragmentIndex } from "./SyncFragmentIndex.ts";
 
@@ -34,8 +36,25 @@ describe("SyncFragmentIndex", async () => {
         await expect(syncFragmentIndex.validate()).resolves.toBeUndefined();
       });
       describe(".create()", () => {
+        test("uses matching data when file matches md5", async () => {
+          server.use(
+            mockLookupISOBMFFFileByMd5({
+              md5: video!.md5,
+              fixture: video!,
+            }),
+          );
+          const syncFragmentIndex = new SyncFragmentIndex(
+            await generateTrackFragmentIndex(video!),
+            video!.md5,
+          );
+          await syncFragmentIndex.create();
+          expect(syncFragmentIndex.isComplete()).toBe(false);
+        });
         test("isComplete() returns false when not created", async () => {
           server.use(
+            mockLookupISOBMFFFileByMd5NotFound({
+              md5: video!.md5,
+            }),
             mockCreateIsobmffFile({
               complete: false,
               id: "123",
@@ -52,6 +71,9 @@ describe("SyncFragmentIndex", async () => {
         });
         test("isComplete() returns true when created", async () => {
           server.use(
+            mockLookupISOBMFFFileByMd5NotFound({
+              md5: video!.md5,
+            }),
             mockCreateIsobmffFile({
               complete: true,
               id: "123",
@@ -75,8 +97,11 @@ describe("SyncFragmentIndex", async () => {
           );
           await expect(syncFragmentIndex.upload()).rejects.toThrow();
         });
-        test("uploads caption", async () => {
+        test("uploads index", async () => {
           server.use(
+            mockLookupISOBMFFFileByMd5NotFound({
+              md5: video!.md5,
+            }),
             mockCreateIsobmffFile({
               complete: true,
               id: "123",
@@ -105,6 +130,9 @@ describe("SyncFragmentIndex", async () => {
         });
         test("marks synced", async () => {
           server.use(
+            mockLookupISOBMFFFileByMd5NotFound({
+              md5: video!.md5,
+            }),
             mockCreateIsobmffFile({
               complete: true,
               id: "123",
@@ -130,7 +158,6 @@ describe("SyncFragmentIndex", async () => {
             complete: true,
             id: "123",
             md5: video!.md5,
-            asset_id: `${video!.md5}:test.mp4`,
             byte_size: 31,
           });
 
@@ -141,7 +168,6 @@ describe("SyncFragmentIndex", async () => {
             complete: true,
             id: "123",
             md5: video!.md5,
-            asset_id: `${video!.md5}:test.mp4`,
             byte_size: 31,
           });
         });

package/src/operations/syncAssetsDirectory/SyncFragmentIndex.ts

@@ -1,13 +1,16 @@
 import fs from "node:fs/promises";
-import { dirname, join } from "node:path";
+import { basename, dirname, join } from "node:path";
 import { Readable } from "node:stream";
 
 import {
   type CreateISOBMFFFileResult,
+  type LookupISOBMFFFileByMd5Result,
   createISOBMFFFile,
+  lookupISOBMFFFileByMd5,
   uploadFragmentIndex,
 } from "@editframe/api";
 
+import { createReadableStreamFromReadable } from "../../utils/createReadableStreamFromReadable.ts";
 import { getClient } from "../../utils/index.ts";
 import type { SubAssetSync } from "./SubAssetSync.ts";
 import { SyncStatus } from "./SyncStatus.ts";
@@ -17,11 +20,9 @@ export class SyncFragmentIndex
 {
   icon = "📋";
   label = "fragment index";
-  syncStatus: SyncStatus = new SyncStatus(this.path);
-  fileSyncStatus: SyncStatus = new SyncStatus(
-    join(dirname(this.path), "isobmff"),
-  );
-  created: CreateISOBMFFFileResult | null = null;
+  syncStatus = new SyncStatus(this.path);
+  fileSyncStatus = new SyncStatus(join(dirname(this.path), "isobmff"));
+  created: CreateISOBMFFFileResult | LookupISOBMFFFileByMd5Result | null = null;
 
   constructor(
     public path: string,
@@ -37,10 +38,18 @@ export class SyncFragmentIndex
   async validate() {}
 
   async create() {
-    this.created = await createISOBMFFFile(getClient(), {
-      md5: this.md5,
-      filename: this.path.replace(/\.tracks.json$/, ""),
-    });
+    const maybeISOBMFFFile = await lookupISOBMFFFileByMd5(
+      getClient(),
+      this.md5,
+    );
+    if (maybeISOBMFFFile) {
+      this.created = maybeISOBMFFFile;
+    } else {
+      this.created = await createISOBMFFFile(getClient(), {
+        md5: this.md5,
+        filename: basename(this.path).replace(/\.tracks.json$/, ""),
+      });
+    }
   }
 
   isComplete() {
@@ -58,7 +67,9 @@ export class SyncFragmentIndex
       this.created.id,
       // It is unclear why we need to use Readable.from here
       // Tests fail when using createReadStream
-      Readable.from(await fs.readFile(this.path)),
+      createReadableStreamFromReadable(
+        Readable.from(await fs.readFile(this.path)),
+      ),
       await this.byteSize(),
     );
   }
@@ -76,7 +87,6 @@ export class SyncFragmentIndex
         complete: true,
         id: this.created.id,
         md5: this.md5,
-        asset_id: this.created.asset_id,
         byte_size: byteSize,
       }),
       this.fileSyncStatus.markSynced({
@@ -84,7 +94,6 @@ export class SyncFragmentIndex
         complete: true,
         id: this.created.id,
         md5: this.md5,
-        asset_id: this.created.asset_id,
         byte_size: byteSize,
       }),
     ]);

package/src/operations/syncAssetsDirectory/SyncImage.test.ts

@@ -1,12 +1,13 @@
+import { useMSW } from "TEST/useMSW.ts";
 import { describe, expect, test } from "vitest";
 import { fixture, withFixtures } from "../../../test-fixtures/fixture.ts";
 import {
   mockCreateImageFile,
   mockGetUploadImageFile,
-  useMSW,
+  mockLookupImageFileByMd5,
+  mockLookupImageFileByMd5NotFound,
 } from "../../../test-fixtures/network.ts";
 import { SyncImage } from "./SyncImage.ts";
-
 describe("SyncImage", async () => {
   const server = useMSW();
   await withFixtures(
@@ -37,8 +38,23 @@ describe("SyncImage", async () => {
           const syncImage = new SyncImage(await cacheImage(image!), image!.md5);
           await expect(syncImage.create()).rejects.toThrow();
         });
+        test("Skips creation if image file matches md5", async () => {
+          server.use(
+            mockLookupImageFileByMd5({
+              md5: image!.md5,
+              fixture: image!,
+            }),
+          );
+          const syncImage = new SyncImage(await cacheImage(image!), image!.md5);
+          await syncImage.prepare();
+          await syncImage.create();
+          expect(syncImage.isComplete()).toBe(true);
+        });
         test("isComplete() returns false when not created", async () => {
           server.use(
+            mockLookupImageFileByMd5NotFound({
+              md5: image!.md5,
+            }),
             mockCreateImageFile({
               complete: false,
               id: "123",
@@ -53,6 +69,9 @@ describe("SyncImage", async () => {
         });
         test("isComplete() returns true when created", async () => {
           server.use(
+            mockLookupImageFileByMd5NotFound({
+              md5: image!.md5,
+            }),
             mockCreateImageFile({
               complete: true,
               id: "123",
@@ -74,6 +93,9 @@ describe("SyncImage", async () => {
         });
         test("uploads image", async () => {
           server.use(
+            mockLookupImageFileByMd5NotFound({
+              md5: image!.md5,
+            }),
             mockCreateImageFile({
               complete: false,
               id: "123",
@@ -98,6 +120,9 @@ describe("SyncImage", async () => {
         });
         test("marks synced", async () => {
           server.use(
+            mockLookupImageFileByMd5NotFound({
+              md5: image!.md5,
+            }),
             mockCreateImageFile({
               complete: true,
               id: "123",
@@ -119,7 +144,6 @@ describe("SyncImage", async () => {
             complete: true,
             id: "123",
             md5: image!.md5,
-            asset_id: `${image!.md5}:test.png`,
             byte_size: 276,
           });
         });

package/src/operations/syncAssetsDirectory/SyncImage.ts

@@ -4,12 +4,15 @@ import path, { basename } from "node:path";
 
 import {
   type CreateImageFileResult,
+  type LookupImageFileByMd5Result,
   createImageFile,
+  lookupImageFileByMd5,
   uploadImageFile,
 } from "@editframe/api";
 
 import { Probe } from "@editframe/assets";
 
+import { createReadableStreamFromReadable } from "../../utils/createReadableStreamFromReadable.ts";
 import { getClient } from "../../utils/index.ts";
 import type { SubAssetSync } from "./SubAssetSync.ts";
 import { SyncStatus } from "./SyncStatus.ts";
@@ -18,7 +21,7 @@ export class SyncImage implements SubAssetSync<CreateImageFileResult> {
   icon = "🖼️";
   label = "image";
   syncStatus: SyncStatus = new SyncStatus(this.path);
-  created: CreateImageFileResult | null = null;
+  created: CreateImageFileResult | LookupImageFileByMd5Result | null = null;
 
   constructor(
     public path: string,
@@ -64,18 +67,24 @@ export class SyncImage implements SubAssetSync<CreateImageFileResult> {
         "No video stream found in image. Should have been prevented by .validate()",
       );
     }
-    this.created = await createImageFile(getClient(), {
-      md5: this.md5,
-      filename: basename(this.path),
-      width: videoProbe.width,
-      height: videoProbe.height,
-      mime_type: `image/${this.extension}` as
-        | "image/jpeg"
-        | "image/png"
-        | "image/jpg"
-        | "image/webp",
-      byte_size: byteSize,
-    });
+
+    const maybeImageFile = await lookupImageFileByMd5(getClient(), this.md5);
+    if (maybeImageFile) {
+      this.created = maybeImageFile;
+    } else {
+      this.created = await createImageFile(getClient(), {
+        md5: this.md5,
+        filename: basename(this.path),
+        width: videoProbe.width,
+        height: videoProbe.height,
+        mime_type: `image/${this.extension}` as
+          | "image/jpeg"
+          | "image/png"
+          | "image/jpg"
+          | "image/webp",
+        byte_size: byteSize,
+      });
+    }
   }
   isComplete() {
     return !!this.created?.complete;
@@ -89,7 +98,7 @@ export class SyncImage implements SubAssetSync<CreateImageFileResult> {
     await uploadImageFile(
       getClient(),
       this.created.id,
-      createReadStream(this.path),
+      createReadableStreamFromReadable(createReadStream(this.path)),
       Number.parseInt(this.probeResult.format.size || "0"),
     ).whenUploaded();
   }
@@ -105,7 +114,6 @@ export class SyncImage implements SubAssetSync<CreateImageFileResult> {
       complete: true,
       id: this.created.id,
       md5: this.md5,
-      asset_id: this.created.asset_id,
       byte_size: byteSize,
     });
   }

package/src/operations/syncAssetsDirectory/SyncStatus.ts

@@ -9,22 +9,22 @@ const SyncStatusSchema = z.object({
   complete: z.boolean(),
   id: z.string(),
   md5: z.string(),
-  asset_id: z.string(),
   byte_size: z.number(),
 });
 
 export interface SyncStatusInfo extends z.infer<typeof SyncStatusSchema> {}
 
 export class SyncStatus {
-  constructor(private basePath: string) {}
-
   infoPath = `${this.basePath}.info`;
 
+  constructor(private basePath: string) {}
+
   async isSynced() {
     const syncInfo = await this.readInfo();
     if (!syncInfo) {
       return false;
     }
+    console.log("syncInfo.infoPath", this.infoPath);
     return syncInfo.version === SYNC_VERSION && syncInfo.complete;
   }