@editframe/api 0.9.0-beta.1 → 0.10.0-beta.2

package/src/resources/unprocessed-file.test.ts

@@ -10,6 +10,8 @@ import {
   createUnprocessedFile,
   processAVFile,
   processAVFileBuffer,
+  processImageFile,
+  processImageFileBuffer,
   updateUnprocessedFile,
   uploadUnprocessedFile,
 } from "./unprocessed-file.ts";
@@ -29,7 +31,7 @@ describe("Unprocessed File", () => {
     test("Throws when file is too large", async () => {
       await expect(
         createUnprocessedFile(client, {
-          id: "test-file",
+          md5: "test-file",
           filename: "test-file",
           processes: [],
           byte_size: 1024 * 1024 * 1025,
@@ -58,7 +60,7 @@ describe("Unprocessed File", () => {
 
       await expect(
         createUnprocessedFile(client, {
-          id: "test-file",
+          md5: "test-file",
           filename: "test-file",
           processes: [],
           byte_size: 1024 * 1024,
@@ -79,7 +81,7 @@ describe("Unprocessed File", () => {
       );
 
       const result = await createUnprocessedFile(client, {
-        id: "test-file",
+        md5: "test-file",
         filename: "test-file",
         processes: [],
         byte_size: 1024 * 1024,
@@ -182,7 +184,14 @@ describe("Unprocessed File", () => {
     test("Throws when server responds with an error when uploading file", async () => {
       server.use(
         http.post("http://localhost/api/v1/unprocessed_files", () =>
-          HttpResponse.json({}, { status: 200 }),
+          HttpResponse.json(
+            {
+              complete: false,
+              id: "098f6bcd-4621-d373-cade-4e832627b4f6",
+              processes: [],
+            },
+            { status: 200 },
+          ),
         ),
         http.post(
           "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6/upload",
@@ -200,7 +209,14 @@ describe("Unprocessed File", () => {
     test("Throws when server responds with an error when updating file", async () => {
       server.use(
         http.post("http://localhost/api/v1/unprocessed_files", () =>
-          HttpResponse.json({}, { status: 200 }),
+          HttpResponse.json(
+            {
+              complete: false,
+              id: "098f6bcd-4621-d373-cade-4e832627b4f6",
+              processes: [],
+            },
+            { status: 200 },
+          ),
         ),
         http.post(
           "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6/upload",
@@ -222,7 +238,14 @@ describe("Unprocessed File", () => {
     test("Returns json data when upload is successful", async () => {
       server.use(
         http.post("http://localhost/api/v1/unprocessed_files", () =>
-          HttpResponse.json({}, { status: 200 }),
+          HttpResponse.json(
+            {
+              complete: false,
+              id: "098f6bcd-4621-d373-cade-4e832627b4f6",
+              processes: [],
+            },
+            { status: 200 },
+          ),
         ),
         http.post(
           "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6/upload",
@@ -256,7 +279,14 @@ describe("Unprocessed File", () => {
     test("Throws when server responds with an error when uploading file", async () => {
       server.use(
         http.post("http://localhost/api/v1/unprocessed_files", () =>
-          HttpResponse.json({}, { status: 200 }),
+          HttpResponse.json(
+            {
+              complete: false,
+              id: "098f6bcd-4621-d373-cade-4e832627b4f6",
+              processes: [],
+            },
+            { status: 200 },
+          ),
         ),
         http.post(
           "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6/upload",
@@ -272,7 +302,14 @@ describe("Unprocessed File", () => {
     test("Throws when server responds with an error when updating file", async () => {
       server.use(
         http.post("http://localhost/api/v1/unprocessed_files", () =>
-          HttpResponse.json({}, { status: 200 }),
+          HttpResponse.json(
+            {
+              complete: false,
+              id: "098f6bcd-4621-d373-cade-4e832627b4f6",
+              processes: [],
+            },
+            { status: 200 },
+          ),
         ),
         http.post(
           "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6/upload",
@@ -292,7 +329,14 @@ describe("Unprocessed File", () => {
     test("Returns json data when upload is successful", async () => {
       server.use(
         http.post("http://localhost/api/v1/unprocessed_files", () =>
-          HttpResponse.json({}, { status: 200 }),
+          HttpResponse.json(
+            {
+              complete: false,
+              id: "098f6bcd-4621-d373-cade-4e832627b4f6",
+              processes: [],
+            },
+            { status: 200 },
+          ),
         ),
         http.post(
           "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6/upload",
@@ -309,4 +353,188 @@ describe("Unprocessed File", () => {
       });
     });
   });
+
+  describe("processImageFileBuffer", () => {
+    test("Throws when server responds with an error when creating file", async () => {
+      server.use(
+        http.post("http://localhost/api/v1/unprocessed_files", () =>
+          HttpResponse.text("Internal Server Error", { status: 500 }),
+        ),
+      );
+
+      await expect(
+        processImageFileBuffer(client, Buffer.from("test"), "test-file"),
+      ).rejects.toThrowError(
+        "Failed to create unprocessed file 500 Internal Server Error",
+      );
+    });
+
+    test("Throws when server responds with an error when uploading file", async () => {
+      server.use(
+        http.post("http://localhost/api/v1/unprocessed_files", () =>
+          HttpResponse.json(
+            {
+              complete: false,
+              id: "098f6bcd-4621-d373-cade-4e832627b4f6",
+              processes: [],
+            },
+            { status: 200 },
+          ),
+        ),
+        http.post(
+          "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6/upload",
+          () => HttpResponse.text("Internal Server Error", { status: 500 }),
+        ),
+      );
+
+      await expect(
+        processImageFileBuffer(client, Buffer.from("test"), "test-file"),
+      ).rejects.toThrowError(
+        "Failed to upload chunk 0 for /api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6/upload 500 Internal Server Error",
+      );
+    });
+
+    test("Throws when server responds with an error when updating file", async () => {
+      server.use(
+        http.post("http://localhost/api/v1/unprocessed_files", () =>
+          HttpResponse.json(
+            {
+              complete: false,
+              id: "098f6bcd-4621-d373-cade-4e832627b4f6",
+              processes: [],
+            },
+            { status: 200 },
+          ),
+        ),
+        http.post(
+          "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6/upload",
+          () => HttpResponse.json({ test }, { status: 201 }),
+        ),
+        http.post(
+          "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6",
+          () => HttpResponse.text("Internal Server Error", { status: 500 }),
+        ),
+      );
+
+      await expect(
+        processImageFileBuffer(client, Buffer.from("test"), "test-file"),
+      ).rejects.toThrowError(
+        "Failed to update unprocessed file 500 Internal Server Error",
+      );
+    });
+
+    test("Returns json data when upload is successful", async () => {
+      server.use(
+        http.post("http://localhost/api/v1/unprocessed_files", () =>
+          HttpResponse.json(
+            {
+              complete: false,
+              id: "098f6bcd-4621-d373-cade-4e832627b4f6",
+              processes: [],
+            },
+            { status: 200 },
+          ),
+        ),
+        http.post(
+          "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6/upload",
+          () => HttpResponse.json({}, { status: 201 }),
+        ),
+        http.post(
+          "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6",
+          () => HttpResponse.json({ test: "response" }, { status: 200 }),
+        ),
+      );
+
+      await expect(
+        processImageFileBuffer(client, Buffer.from("test"), "test-file"),
+      ).resolves.toEqual({ test: "response" });
+    });
+  });
+
+  describe("ProcessImageFile", () => {
+    test("Throws when server responds with an error when creating file", async () => {
+      server.use(
+        http.post("http://localhost/api/v1/unprocessed_files", () =>
+          HttpResponse.text("Internal Server Error", { status: 500 }),
+        ),
+      );
+
+      await expect(processImageFile(client, TEST_AV_FILE)).rejects.toThrowError(
+        "Failed to create unprocessed file 500 Internal Server Error",
+      );
+    });
+
+    test("Throws when server responds with an error when uploading file", async () => {
+      server.use(
+        http.post("http://localhost/api/v1/unprocessed_files", () =>
+          HttpResponse.json(
+            {
+              complete: false,
+              id: "098f6bcd-4621-d373-cade-4e832627b4f6",
+              processes: [],
+            },
+            { status: 200 },
+          ),
+        ),
+        http.post(
+          "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6/upload",
+          () => HttpResponse.text("Internal Server Error", { status: 500 }),
+        ),
+      );
+
+      await expect(processImageFile(client, TEST_AV_FILE)).rejects.toThrowError(
+        "Failed to upload chunk 0 for /api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6/upload 500 Internal Server Error",
+      );
+    });
+
+    test("Throws when server responds with an error when updating file", async () => {
+      server.use(
+        http.post("http://localhost/api/v1/unprocessed_files", () =>
+          HttpResponse.json(
+            {
+              complete: false,
+              id: "098f6bcd-4621-d373-cade-4e832627b4f6",
+              processes: [],
+            },
+            { status: 200 },
+          ),
+        ),
+        http.post(
+          "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6/upload",
+          () => HttpResponse.json({ test }, { status: 201 }),
+        ),
+        http.post(
+          "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6",
+          () => HttpResponse.text("Internal Server Error", { status: 500 }),
+        ),
+      );
+
+      await expect(processImageFile(client, TEST_AV_FILE)).rejects.toThrowError(
+        "Failed to update unprocessed file 500 Internal Server Error",
+      );
+    });
+
+    test("Returns json data when upload is successful", async () => {
+      server.use(
+        http.post("http://localhost/api/v1/unprocessed_files", () =>
+          HttpResponse.json(
+            {
+              complete: false,
+              id: "098f6bcd-4621-d373-cade-4e832627b4f6",
+              processes: [],
+            },
+            { status: 200 },
+          ),
+        ),
+        http.post(
+          "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6/upload",
+          () => HttpResponse.json({}, { status: 201 }),
+        ),
+        http.post(
+          "http://localhost/api/v1/unprocessed_files/098f6bcd-4621-d373-cade-4e832627b4f6",
+          () => HttpResponse.json({ test: "response" }, { status: 200 }),
+        ),
+      );
+    });
+  });
 });

package/src/resources/unprocessed-file.ts

@@ -13,21 +13,25 @@ import { uploadChunks } from "../uploadChunks.ts";
 
 const log = debug("ef:api:unprocessed-file");
 
-const FileProcessors = z
-  .array(z.union([z.literal("isobmff"), z.literal("captions")]))
-  .refine(
-    (value) => {
-      return new Set(value).size === value.length;
-    },
-    {
-      message: "Processors list must not include duplicates",
-    },
-  );
+const FileProcessor = z.union([
+  z.literal("isobmff"),
+  z.literal("image"),
+  z.literal("captions"),
+]);
+
+const FileProcessors = z.array(FileProcessor).refine(
+  (value) => {
+    return new Set(value).size === value.length;
+  },
+  {
+    message: "Processors list must not include duplicates",
+  },
+);
 
 const MAX_FILE_SIZE = 1024 * 1024 * 1024; // 1GiB
 
 export const CreateUnprocessedFilePayload = z.object({
-  id: z.string(),
+  md5: z.string(),
   filename: z.string(),
   processes: FileProcessors.optional(),
   byte_size: z.number().int().max(MAX_FILE_SIZE),
@@ -40,15 +44,21 @@ export const UpdateUnprocessedFilePayload = z.object({
 export interface CreateUnprocessedFileResult {
   byte_size: number;
   next_byte: number;
+  complete: boolean;
   id: string;
+  md5: string;
   processes: z.infer<typeof FileProcessors>;
+  asset_id: string;
 }
 
 export interface UpdateUnprocessedFileResult {
   byte_size?: number;
   next_byte: number;
+  complete: boolean;
   id: string;
+  md5: string;
   processes: z.infer<typeof FileProcessors>;
+  asset_id: string;
 }
 
 export const createUnprocessedFile = async (
@@ -124,66 +134,90 @@ export const uploadUnprocessedFile = async (
   log("Unprocessed file upload complete");
 };
 
-export const processAVFileBuffer = async (
+const processResource = async (
   client: Client,
-  buffer: Buffer,
-  filename = "buffer",
+  filename: string,
+  md5: string,
+  byteSize: number,
+  processor: z.infer<typeof FileProcessor>,
+  doUpload: (id: string) => Promise<void>,
 ) => {
-  log("Processing AV file buffer");
-  const fileId = md5Buffer(buffer);
+  log("Processing", { filename, md5, byteSize, processor });
 
-  log("File ID", fileId);
-  log(`File size: ${buffer.byteLength} bytes`);
-
-  await createUnprocessedFile(client, {
-    id: fileId,
+  const unprocessedFile = await createUnprocessedFile(client, {
+    md5: md5,
     processes: [],
     filename,
-    byte_size: buffer.byteLength,
-  });
-
-  const readStream = new Readable({
-    read() {
-      readStream.push(buffer);
-      readStream.push(null);
-    },
+    byte_size: byteSize,
   });
 
-  await uploadUnprocessedFile(client, fileId, readStream, buffer.byteLength);
+  if (unprocessedFile.complete === false) {
+    await doUpload(unprocessedFile.id);
+  }
 
-  const fileInformation = await updateUnprocessedFile(client, fileId, {
-    processes: ["isobmff"],
-  });
+  if (unprocessedFile.processes.includes(processor)) {
+    log("File already processed", unprocessedFile);
+    return unprocessedFile;
+  }
 
+  const fileInformation = await updateUnprocessedFile(
+    client,
+    unprocessedFile.id,
+    {
+      processes: [processor],
+    },
+  );
   log("File processed", fileInformation);
   return fileInformation;
 };
 
-export const processAVFile = async (client: Client, filePath: string) => {
-  log("Processing AV file", filePath);
-  const fileId = await md5FilePath(filePath);
-
-  log("File ID", fileId);
-  await createUnprocessedFile(client, {
-    id: fileId,
-    processes: [],
-    filename: basename(filePath),
-    byte_size: (await stat(filePath)).size,
-  });
-
-  const readStream = createReadStream(filePath);
+const buildBufferProcessor = (processor: z.infer<typeof FileProcessor>) => {
+  return async (client: Client, buffer: Buffer, filename = "buffer") => {
+    log(`Processing file buffer: ${processor}`, filename);
+    const md5 = md5Buffer(buffer);
+
+    return await processResource(
+      client,
+      filename,
+      md5,
+      buffer.byteLength,
+      processor,
+      async (id: string) => {
+        const readStream = new Readable({
+          read() {
+            readStream.push(buffer);
+            readStream.push(null);
+          },
+        });
+
+        await uploadUnprocessedFile(client, id, readStream, buffer.byteLength);
+      },
+    );
+  };
+};
 
-  await uploadUnprocessedFile(
-    client,
-    fileId,
-    readStream,
-    (await stat(filePath)).size,
-  );
+const buildFileProcessor = (processor: z.infer<typeof FileProcessor>) => {
+  return async (client: Client, filePath: string) => {
+    log(`Processing file ${processor}`, filePath);
+    const md5 = await md5FilePath(filePath);
+    const byteSize = (await stat(filePath)).size;
+
+    return await processResource(
+      client,
+      basename(filePath),
+      md5,
+      byteSize,
+      processor,
+      async (id: string) => {
+        const readStream = createReadStream(filePath);
+        return await uploadUnprocessedFile(client, id, readStream, byteSize);
+      },
+    );
+  };
+};
 
-  const fileInformation = await updateUnprocessedFile(client, fileId, {
-    processes: ["isobmff"],
-  });
+export const processAVFileBuffer = buildBufferProcessor("isobmff");
+export const processAVFile = buildFileProcessor("isobmff");
 
-  log("File processed", fileInformation);
-  return fileInformation;
-};
+export const processImageFileBuffer = buildBufferProcessor("image");
+export const processImageFile = buildFileProcessor("image");