@editframe/cli 0.10.0-beta.5 → 0.10.0-beta.7

package/src/commands/render.ts

@@ -1,53 +1,47 @@
-import path from "node:path";
 import { readFile, writeFile } from "node:fs/promises";
-import { inspect } from "node:util";
+import path, { basename, join } from "node:path";
 import { PassThrough } from "node:stream";
+import { inspect } from "node:util";
 
-import * as tar from "tar";
-import { program, Option } from "commander";
+import { Option, program } from "commander";
 import { parse as parseHTML } from "node-html-parser";
+import * as tar from "tar";
 
 import { md5Directory, md5FilePath } from "@editframe/assets";
 
-import { withSpinner } from "../utils/withSpinner.ts";
-import { syncAssetDirectory } from "../operations/syncAssetsDirectory.ts";
+import { spawnSync } from "node:child_process";
 import { createRender, uploadRender } from "@editframe/api";
-import { launchBrowserAndWaitForSDK } from "../utils/launchBrowserAndWaitForSDK.ts";
-import { PreviewServer } from "../utils/startPreviewServer.ts";
-import { getClient } from "../utils/index.ts";
+import debug from "debug";
 import { RenderInfo, getRenderInfo } from "../operations/getRenderInfo.ts";
 import { processRenderInfo } from "../operations/processRenderInfo.ts";
-import { validateVideoResolution } from "../utils/validateVideoResolution.ts";
+import { syncAssetDirectory } from "../operations/syncAssetsDirectory.ts";
+import { SyncStatus } from "../operations/syncAssetsDirectory/SyncStatus.ts";
 import { getFolderSize } from "../utils/getFolderSize.ts";
-import { spawnSync } from "node:child_process";
-
-interface StrategyBuilder {
-  buildAssetId: (assetPath: string) => Promise<string>;
-}
-
-export const buildAssetId = async (assetPath: string) => {
-  const md5Sum = await md5FilePath(assetPath);
-
-  const basename = path.basename(assetPath);
-
-  return `${md5Sum}:${basename}`;
-};
-
-class V1Builder implements StrategyBuilder {
-  async buildAssetId(assetPath: string) {
-    return buildAssetId(assetPath);
-  }
-}
+import { getClient } from "../utils/index.ts";
+import { launchBrowserAndWaitForSDK } from "../utils/launchBrowserAndWaitForSDK.ts";
+import { PreviewServer } from "../utils/startPreviewServer.ts";
+import { validateVideoResolution } from "../utils/validateVideoResolution.ts";
+import { withSpinner } from "../utils/withSpinner.ts";
 
-class V2Builder implements StrategyBuilder {
-  async buildAssetId(assetPath: string) {
-    return buildAssetId(assetPath);
+const log = debug("ef:cli:render");
+
+export const buildAssetId = async (
+  srcDir: string,
+  src: string,
+  basename: string,
+) => {
+  log(`Building image asset id for ${src}\n`);
+  const assetPath = path.join(srcDir, src);
+  const assetMd5 = await md5FilePath(assetPath);
+  const syncStatus = new SyncStatus(
+    join(srcDir, "assets", ".cache", assetMd5, basename),
+  );
+  const info = await syncStatus.readInfo();
+  if (!info) {
+    throw new Error(`SyncStatus info is not found for ${syncStatus.infoPath}`);
   }
-}
 
-const strategyBuilders: Record<string, StrategyBuilder> = {
-  v1: new V1Builder(),
-  v2: new V2Builder(),
+  return info.asset_id;
 };
 
 program
@@ -57,21 +51,18 @@ program
   )
   .addOption(
     new Option("-s, --strategy <strategy>", "Render strategy")
-      .choices(["v1", "v2"])
+      .choices(["v1"])
       .default("v1"),
   )
   .action(async (directory, options) => {
     directory ??= ".";
 
-    await syncAssetDirectory(directory);
+    await syncAssetDirectory(
+      join(process.cwd(), directory, "src", "assets", ".cache"),
+    );
 
     const srcDir = path.join(directory, "src");
     const distDir = path.join(directory, "dist");
-    const builder = strategyBuilders[options.strategy];
-    if (!builder) {
-      console.error("Invalid strategy", options.strategy);
-      return;
-    }
     await withSpinner("Building\n", async () => {
       try {
         await withSpinner("Building\n", async () => {
@@ -117,23 +108,41 @@ program
         const doc = parseHTML(
           await readFile(path.join(distDir, "index.html"), "utf-8"),
         );
-        const elements = doc.querySelectorAll(
-          "ef-audio, ef-video, ef-image, ef-captions",
-        );
-        for (const element of elements) {
+
+        log("Building asset IDs");
+        for (const element of doc.querySelectorAll(
+          "ef-image, ef-audio, ef-video",
+        )) {
+          log(`Processing ${element.tagName}`);
           if (element.hasAttribute("asset-id")) {
+            log(
+              `Asset ID for ${element.tagName} ${element.getAttribute("src")} is ${element.getAttribute("asset-id")}`,
+            );
             continue;
           }
           const src = element.getAttribute("src");
           if (!src) {
+            log(`No src attribute for ${element.tagName}`);
             continue;
           }
-          const assetPath = path.join(srcDir, src);
 
-          element.setAttribute(
-            "asset-id",
-            await builder.buildAssetId(assetPath),
-          );
+          switch (element.tagName) {
+            case "EF-IMAGE":
+              element.setAttribute(
+                "asset-id",
+                await buildAssetId(srcDir, src, basename(src)),
+              );
+              break;
+            case "EF-AUDIO":
+            case "EF-VIDEO":
+              element.setAttribute(
+                "asset-id",
+                await buildAssetId(srcDir, src, "isobmff"),
+              );
+              break;
+            default:
+              log(`Unknown element type: ${element.tagName}`);
+          }
         }
 
         await writeFile(path.join(distDir, "index.html"), doc.toString());

package/src/commands/sync.ts

@@ -1,3 +1,4 @@
+import { join } from "node:path";
 import { program } from "commander";
 import { syncAssetDirectory } from "../operations/syncAssetsDirectory.ts";
 
@@ -5,6 +6,8 @@ program
   .command("sync")
   .description("Sync assets to Editframe servers for rendering")
   .argument("[directory]", "Path to project directory to sync.")
-  .action(async (projectDirectory = ".") => {
-    await syncAssetDirectory(projectDirectory);
+  .action(async (directory = ".") => {
+    await syncAssetDirectory(
+      join(process.cwd(), directory, "src", "assets", ".cache"),
+    );
   });

package/src/operations/syncAssetsDirectory/SubAssetSync.ts

@@ -0,0 +1,42 @@
+import type { SyncStatus } from "./SyncStatus.ts";
+
+import { SyncCaption } from "./SyncCaption.ts";
+import { SyncFragmentIndex } from "./SyncFragmentIndex.ts";
+import { SyncImage } from "./SyncImage.ts";
+import { SyncTrack } from "./SyncTrack.ts";
+
+export interface SubAssetSync<CreationType> {
+  icon: string;
+  label: string;
+  path: string;
+  md5: string;
+  prepare: () => Promise<void>;
+  validate: () => Promise<void>;
+  create: () => Promise<void>;
+  upload: () => Promise<void>;
+  syncStatus: SyncStatus;
+  isComplete: () => boolean;
+  markSynced: () => Promise<void>;
+  created: CreationType | null;
+}
+
+const trackMatch = /\.track-[\d]+.mp4$/i;
+const fragmentIndexMatch = /\.tracks.json$/i;
+const captionsMatch = /\.captions.json$/i;
+const imageMatch = /\.(png|jpe?g|gif|webp)$/i;
+
+export const getAssetSync = (subAssetPath: string, md5: string) => {
+  if (imageMatch.test(subAssetPath)) {
+    return new SyncImage(subAssetPath, md5);
+  }
+  if (trackMatch.test(subAssetPath)) {
+    return new SyncTrack(subAssetPath, md5);
+  }
+  if (fragmentIndexMatch.test(subAssetPath)) {
+    return new SyncFragmentIndex(subAssetPath, md5);
+  }
+  if (captionsMatch.test(subAssetPath)) {
+    return new SyncCaption(subAssetPath, md5);
+  }
+  throw new Error(`Unrecognized sub-asset type: ${subAssetPath}`);
+};

package/src/operations/syncAssetsDirectory/SyncCaption.test.ts

@@ -0,0 +1,145 @@
+import { describe, expect, test } from "vitest";
+import { fixture, withFixtures } from "../../../test-fixtures/fixture.ts";
+import {
+  mockCreateCaptionFile,
+  mockUploadCaptionFile,
+  useMSW,
+} from "../../../test-fixtures/network.ts";
+import { SyncCaption } from "./SyncCaption.ts";
+
+describe("SyncCaption", async () => {
+  const server = useMSW();
+  await withFixtures(
+    [fixture("test.mp4", "test.mp4")],
+    async ({ files: [video], generateCaptions }) => {
+      test("Reads byte size", async () => {
+        const syncCaption = new SyncCaption(
+          await generateCaptions(video!),
+          video!.md5,
+        );
+        await expect(syncCaption.byteSize()).resolves.toEqual(35);
+      });
+
+      test("prepare() is noop", async () => {
+        const syncCaption = new SyncCaption(
+          await generateCaptions(video!),
+          video!.md5,
+        );
+        await expect(syncCaption.prepare()).resolves.toBeUndefined();
+      });
+
+      test("validate() is noop", async () => {
+        const syncCaption = new SyncCaption(
+          await generateCaptions(video!),
+          video!.md5,
+        );
+        await expect(syncCaption.validate()).resolves.toBeUndefined();
+      });
+
+      describe(".create()", () => {
+        test("isComplete() returns false when not created", async () => {
+          server.use(
+            mockCreateCaptionFile({
+              complete: false,
+              id: "123",
+              filename: "test.mp4",
+              fixture: video!,
+            }),
+          );
+          const syncCaption = new SyncCaption(
+            await generateCaptions(video!),
+            video!.md5,
+          );
+          await syncCaption.create();
+          expect(syncCaption.isComplete()).toBe(false);
+        });
+
+        test("isComplete() returns true when created", async () => {
+          server.use(
+            mockCreateCaptionFile({
+              complete: true,
+              id: "123",
+              filename: "test.mp4",
+              fixture: video!,
+            }),
+          );
+          const syncCaption = new SyncCaption(
+            await generateCaptions(video!),
+            video!.md5,
+          );
+          await syncCaption.create();
+          expect(syncCaption.isComplete()).toBe(true);
+        });
+      });
+
+      describe(".upload()", () => {
+        test("throws when not created", async () => {
+          const syncCaption = new SyncCaption(
+            await generateCaptions(video!),
+            video!.md5,
+          );
+          await expect(syncCaption.upload()).rejects.toThrow();
+        });
+
+        test("uploads caption", async () => {
+          server.use(
+            mockCreateCaptionFile({
+              complete: true,
+              id: "123",
+              filename: "test.mp4",
+              fixture: video!,
+            }),
+            mockUploadCaptionFile({
+              id: "123",
+              filename: "test.mp4",
+              fixture: video!,
+            }),
+          );
+          const syncCaption = new SyncCaption(
+            await generateCaptions(video!),
+            video!.md5,
+          );
+          await syncCaption.create();
+          await expect(syncCaption.upload()).resolves.toBeUndefined();
+        });
+      });
+
+      describe(".markSynced()", () => {
+        test("throws when not created", async () => {
+          const syncCaption = new SyncCaption(
+            await generateCaptions(video!),
+            video!.md5,
+          );
+          await expect(syncCaption.markSynced()).rejects.toThrow();
+        });
+
+        test("marks synced", async () => {
+          server.use(
+            mockCreateCaptionFile({
+              complete: true,
+              id: "123",
+              filename: "test.mp4",
+              fixture: video!,
+            }),
+          );
+          const syncCaption = new SyncCaption(
+            await generateCaptions(video!),
+            video!.md5,
+          );
+          await syncCaption.create();
+          await syncCaption.markSynced();
+
+          await expect(syncCaption.syncStatus.isSynced()).resolves.toBe(true);
+          await expect(syncCaption.syncStatus.readInfo()).resolves.toEqual({
+            version: "1",
+            complete: true,
+            id: "123",
+            md5: video!.md5,
+            asset_id: `${video!.md5}:test.mp4`,
+            byte_size: 35,
+          });
+        });
+      });
+    },
+  );
+});

package/src/operations/syncAssetsDirectory/SyncCaption.ts

@@ -0,0 +1,76 @@
+import fs from "node:fs/promises";
+
+import {
+  type CreateCaptionFileResult,
+  createCaptionFile,
+  uploadCaptionFile,
+} from "@editframe/api";
+
+import { Readable } from "node:stream";
+import { getClient } from "../../utils/index.ts";
+import type { SubAssetSync } from "./SubAssetSync.ts";
+import { SyncStatus } from "./SyncStatus.ts";
+
+export class SyncCaption implements SubAssetSync<CreateCaptionFileResult> {
+  icon = "📝";
+  label = "captions";
+  syncStatus: SyncStatus = new SyncStatus(this.path);
+  created: CreateCaptionFileResult | null = null;
+  constructor(
+    public path: string,
+    public md5: string,
+  ) {}
+
+  async byteSize() {
+    return (await fs.stat(this.path)).size;
+  }
+
+  async prepare() {}
+
+  async validate() {}
+
+  async create() {
+    this.created = await createCaptionFile(getClient(), {
+      md5: this.md5,
+      filename: this.path.replace(/\.captions.json$/, ""),
+      byte_size: await this.byteSize(),
+    });
+  }
+
+  isComplete() {
+    return !!this.created?.complete;
+  }
+
+  async upload() {
+    if (!this.created) {
+      throw new Error(
+        "Caption not created. Should have been prevented by .isComplete()",
+      );
+    }
+    await uploadCaptionFile(
+      getClient(),
+      this.created.id,
+      // It's not clear why we need to use Readable.from here, but it seems
+      // to fix an issue where the request is closed early in tests
+      Readable.from(await fs.readFile(this.path)),
+      await this.byteSize(),
+    );
+  }
+
+  async markSynced() {
+    if (!this.created) {
+      throw new Error(
+        "Caption not created. Should have been prevented by .isComplete()",
+      );
+    }
+    const byteSize = await this.byteSize();
+    await this.syncStatus.markSynced({
+      version: "1",
+      complete: true,
+      id: this.created.id,
+      md5: this.md5,
+      asset_id: this.created.asset_id,
+      byte_size: byteSize,
+    });
+  }
+}

package/src/operations/syncAssetsDirectory/SyncFragmentIndex.test.ts

@@ -0,0 +1,151 @@
+import { describe, expect, test } from "vitest";
+import { fixture, withFixtures } from "../../../test-fixtures/fixture.ts";
+import {
+  mockCreateIsobmffFile,
+  mockUploadIsobmffFileIndex,
+  useMSW,
+} from "../../../test-fixtures/network.ts";
+import { SyncFragmentIndex } from "./SyncFragmentIndex.ts";
+
+describe("SyncFragmentIndex", async () => {
+  const server = useMSW();
+  await withFixtures(
+    [fixture("test.mp4", "test.mp4")],
+    async ({ files: [video], generateTrackFragmentIndex }) => {
+      test("Reads byte size", async () => {
+        const syncFragmentIndex = new SyncFragmentIndex(
+          await generateTrackFragmentIndex(video!),
+          video!.md5,
+        );
+        await expect(syncFragmentIndex.byteSize()).resolves.toEqual(31);
+      });
+      test("prepare() is noop", async () => {
+        const syncFragmentIndex = new SyncFragmentIndex(
+          await generateTrackFragmentIndex(video!),
+          video!.md5,
+        );
+        await expect(syncFragmentIndex.prepare()).resolves.toBeUndefined();
+      });
+      test("validate() is noop", async () => {
+        const syncFragmentIndex = new SyncFragmentIndex(
+          await generateTrackFragmentIndex(video!),
+          video!.md5,
+        );
+        await expect(syncFragmentIndex.validate()).resolves.toBeUndefined();
+      });
+      describe(".create()", () => {
+        test("isComplete() returns false when not created", async () => {
+          server.use(
+            mockCreateIsobmffFile({
+              complete: false,
+              id: "123",
+              filename: "test.mp4",
+              fixture: video!,
+            }),
+          );
+          const syncFragmentIndex = new SyncFragmentIndex(
+            await generateTrackFragmentIndex(video!),
+            video!.md5,
+          );
+          await syncFragmentIndex.create();
+          expect(syncFragmentIndex.isComplete()).toBe(false);
+        });
+        test("isComplete() returns true when created", async () => {
+          server.use(
+            mockCreateIsobmffFile({
+              complete: true,
+              id: "123",
+              filename: "test.mp4",
+              fixture: video!,
+            }),
+          );
+          const syncFragmentIndex = new SyncFragmentIndex(
+            await generateTrackFragmentIndex(video!),
+            video!.md5,
+          );
+          await syncFragmentIndex.create();
+          expect(syncFragmentIndex.isComplete()).toBe(true);
+        });
+      });
+      describe(".upload()", () => {
+        test("throws when not created", async () => {
+          const syncFragmentIndex = new SyncFragmentIndex(
+            await generateTrackFragmentIndex(video!),
+            video!.md5,
+          );
+          await expect(syncFragmentIndex.upload()).rejects.toThrow();
+        });
+        test("uploads caption", async () => {
+          server.use(
+            mockCreateIsobmffFile({
+              complete: true,
+              id: "123",
+              filename: "test.mp4",
+              fixture: video!,
+            }),
+            mockUploadIsobmffFileIndex({
+              id: "123",
+            }),
+          );
+          const syncFragmentIndex = new SyncFragmentIndex(
+            await generateTrackFragmentIndex(video!),
+            video!.md5,
+          );
+          await syncFragmentIndex.create();
+          await expect(syncFragmentIndex.upload()).resolves.toBeUndefined();
+        });
+      });
+      describe(".markSynced()", () => {
+        test("throws when not created", async () => {
+          const syncFragmentIndex = new SyncFragmentIndex(
+            await generateTrackFragmentIndex(video!),
+            video!.md5,
+          );
+          await expect(syncFragmentIndex.markSynced()).rejects.toThrow();
+        });
+        test("marks synced", async () => {
+          server.use(
+            mockCreateIsobmffFile({
+              complete: true,
+              id: "123",
+              fixture: video!,
+            }),
+            mockUploadIsobmffFileIndex({
+              id: "123",
+            }),
+          );
+          const syncFragmentIndex = new SyncFragmentIndex(
+            await generateTrackFragmentIndex(video!),
+            video!.md5,
+          );
+          await syncFragmentIndex.create();
+          await syncFragmentIndex.markSynced();
+          await expect(syncFragmentIndex.syncStatus.isSynced()).resolves.toBe(
+            true,
+          );
+          await expect(
+            syncFragmentIndex.syncStatus.readInfo(),
+          ).resolves.toEqual({
+            version: "1",
+            complete: true,
+            id: "123",
+            md5: video!.md5,
+            asset_id: `${video!.md5}:test.mp4`,
+            byte_size: 31,
+          });
+
+          await expect(
+            syncFragmentIndex.fileSyncStatus.readInfo(),
+          ).resolves.toEqual({
+            version: "1",
+            complete: true,
+            id: "123",
+            md5: video!.md5,
+            asset_id: `${video!.md5}:test.mp4`,
+            byte_size: 31,
+          });
+        });
+      });
+    },
+  );
+});

package/src/operations/syncAssetsDirectory/SyncFragmentIndex.ts

@@ -0,0 +1,92 @@
+import fs from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { Readable } from "node:stream";
+
+import {
+  type CreateISOBMFFFileResult,
+  createISOBMFFFile,
+  uploadFragmentIndex,
+} from "@editframe/api";
+
+import { getClient } from "../../utils/index.ts";
+import type { SubAssetSync } from "./SubAssetSync.ts";
+import { SyncStatus } from "./SyncStatus.ts";
+
+export class SyncFragmentIndex
+  implements SubAssetSync<CreateISOBMFFFileResult>
+{
+  icon = "📋";
+  label = "fragment index";
+  syncStatus: SyncStatus = new SyncStatus(this.path);
+  fileSyncStatus: SyncStatus = new SyncStatus(
+    join(dirname(this.path), "isobmff"),
+  );
+  created: CreateISOBMFFFileResult | null = null;
+
+  constructor(
+    public path: string,
+    public md5: string,
+  ) {}
+
+  async byteSize() {
+    return (await fs.stat(this.path)).size;
+  }
+
+  async prepare() {}
+
+  async validate() {}
+
+  async create() {
+    this.created = await createISOBMFFFile(getClient(), {
+      md5: this.md5,
+      filename: this.path.replace(/\.tracks.json$/, ""),
+    });
+  }
+
+  isComplete() {
+    return !!this.created?.fragment_index_complete;
+  }
+
+  async upload() {
+    if (!this.created) {
+      throw new Error(
+        "Fragment index not created. Should have been prevented by .isComplete()",
+      );
+    }
+    await uploadFragmentIndex(
+      getClient(),
+      this.created.id,
+      // It is unclear why we need to use Readable.from here
+      // Tests fail when using createReadStream
+      Readable.from(await fs.readFile(this.path)),
+      await this.byteSize(),
+    );
+  }
+
+  async markSynced() {
+    if (!this.created) {
+      throw new Error(
+        "Fragment index not created. Should have been prevented by .isComplete()",
+      );
+    }
+    const byteSize = await this.byteSize();
+    await Promise.all([
+      this.syncStatus.markSynced({
+        version: "1",
+        complete: true,
+        id: this.created.id,
+        md5: this.md5,
+        asset_id: this.created.asset_id,
+        byte_size: byteSize,
+      }),
+      this.fileSyncStatus.markSynced({
+        version: "1",
+        complete: true,
+        id: this.created.id,
+        md5: this.md5,
+        asset_id: this.created.asset_id,
+        byte_size: byteSize,
+      }),
+    ]);
+  }
+}