@editframe/api 0.8.0-beta.4 → 0.8.0-beta.7

package/dist/resources/isobmff-track.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/resources/renders.d.ts

@@ -1,7 +1,6 @@
 import { Readable } from 'node:stream';
 import { z } from 'zod';
 import { Client } from '../client.ts';
-
 export declare const CreateRenderPayload: z.ZodObject<{
     id: z.ZodString;
     fps: z.ZodNumber;
@@ -14,22 +13,22 @@ export declare const CreateRenderPayload: z.ZodObject<{
     id: string;
     height: number;
     width: number;
+    strategy: "v1" | "v2";
     duration_ms: number;
     fps: number;
     work_slice_ms: number;
-    strategy: "v1" | "v2";
 }, {
     id: string;
     height: number;
     width: number;
+    strategy: "v1" | "v2";
     duration_ms: number;
     fps: number;
     work_slice_ms: number;
-    strategy: "v1" | "v2";
 }>;
 export interface CreateRenderResult {
     status: "complete" | "created" | "failed" | "pending" | "rendering";
     id: string;
 }
-export declare const createRender: (client: Client, payload: z.infer<typeof CreateRenderPayload>) => Promise<CreateRenderResult | undefined>;
-export declare const uploadRender: (client: Client, fileId: string, fileStream: Readable) => Promise<unknown>;
+export declare const createRender: (client: Client, payload: z.infer<typeof CreateRenderPayload>) => Promise<CreateRenderResult>;
+export declare const uploadRender: (client: Client, fileId: string, fileStream: Readable, folderSize: number) => Promise<unknown>;

package/dist/resources/renders.js

@@ -1,6 +1,7 @@
 import { z } from "zod";
 import debug from "debug";
 const log = debug("ef:api:renders");
+const FILE_SIZE_LIMIT = 1024 * 1024 * 16;
 const CreateRenderPayload = z.object({
   id: z.string().uuid(),
   fps: z.number(),
@@ -17,37 +18,32 @@ const createRender = async (client, payload) => {
     body: JSON.stringify(payload)
   });
   log("Render created", response);
-  switch (response.status) {
-    case 200: {
-      return await response.json();
-    }
-    default: {
-      console.error("Failed to create render");
-      console.error(await response.json());
-      return;
-    }
+  if (response.ok) {
+    return await response.json();
   }
+  throw new Error(
+    `Failed to create render ${response.status} ${response.statusText}`
+  );
 };
-const uploadRender = async (client, fileId, fileStream) => {
+const uploadRender = async (client, fileId, fileStream, folderSize) => {
   log("Uploading render", fileId);
-  const fileIndex = await client.authenticatedFetch(
+  log("Folder size", folderSize, "bytes");
+  if (folderSize > FILE_SIZE_LIMIT) {
+    throw new Error(`File size exceeds limit of ${FILE_SIZE_LIMIT} bytes`);
+  }
+  const response = await client.authenticatedFetch(
     `/api/video2/renders/${fileId}/upload`,
     {
       method: "POST",
       body: fileStream
     }
   );
-  log("Render uploaded", fileIndex);
-  switch (fileIndex.status) {
-    case 200: {
-      return fileIndex.json();
-    }
-    default: {
-      console.error("Failed to upload render");
-      console.error(fileIndex.status, fileIndex.statusText);
-      return;
-    }
+  if (response.ok) {
+    return response.json();
   }
+  throw new Error(
+    `Failed to upload render ${response.status} ${response.statusText}`
+  );
 };
 export {
   CreateRenderPayload,

package/dist/resources/renders.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/resources/unprocessed-file.d.ts

@@ -1,43 +1,45 @@
 import { Readable } from 'node:stream';
 import { z } from 'zod';
 import { Client } from '../client.ts';
-
-declare const FileProcessors: z.ZodEffects<z.ZodArray<z.ZodUnion<[z.ZodLiteral<"isobmff">, z.ZodLiteral<"captions">]>, "many">, ("isobmff" | "captions")[], ("isobmff" | "captions")[]>;
+declare const FileProcessors: z.ZodEffects<z.ZodArray<z.ZodUnion<[z.ZodLiteral<"isobmff">, z.ZodLiteral<"captions">]>, "many">, ("captions" | "isobmff")[], ("captions" | "isobmff")[]>;
 export declare const CreateUnprocessedFilePayload: z.ZodObject<{
     id: z.ZodString;
     filename: z.ZodString;
-    processes: z.ZodOptional<z.ZodEffects<z.ZodArray<z.ZodUnion<[z.ZodLiteral<"isobmff">, z.ZodLiteral<"captions">]>, "many">, ("isobmff" | "captions")[], ("isobmff" | "captions")[]>>;
+    processes: z.ZodOptional<z.ZodEffects<z.ZodArray<z.ZodUnion<[z.ZodLiteral<"isobmff">, z.ZodLiteral<"captions">]>, "many">, ("captions" | "isobmff")[], ("captions" | "isobmff")[]>>;
+    byte_size: z.ZodNumber;
 }, "strip", z.ZodTypeAny, {
     id: string;
     filename: string;
-    processes?: ("isobmff" | "captions")[] | undefined;
+    byte_size: number;
+    processes?: ("captions" | "isobmff")[] | undefined;
 }, {
     id: string;
     filename: string;
-    processes?: ("isobmff" | "captions")[] | undefined;
+    byte_size: number;
+    processes?: ("captions" | "isobmff")[] | undefined;
 }>;
 export declare const UpdateUnprocessedFilePayload: z.ZodObject<{
-    processes: z.ZodOptional<z.ZodEffects<z.ZodArray<z.ZodUnion<[z.ZodLiteral<"isobmff">, z.ZodLiteral<"captions">]>, "many">, ("isobmff" | "captions")[], ("isobmff" | "captions")[]>>;
+    processes: z.ZodOptional<z.ZodEffects<z.ZodArray<z.ZodUnion<[z.ZodLiteral<"isobmff">, z.ZodLiteral<"captions">]>, "many">, ("captions" | "isobmff")[], ("captions" | "isobmff")[]>>;
 }, "strip", z.ZodTypeAny, {
-    processes?: ("isobmff" | "captions")[] | undefined;
+    processes?: ("captions" | "isobmff")[] | undefined;
 }, {
-    processes?: ("isobmff" | "captions")[] | undefined;
+    processes?: ("captions" | "isobmff")[] | undefined;
 }>;
 export interface CreateUnprocessedFileResult {
     byte_size: number;
-    last_received_byte: number;
+    next_byte: number;
     id: string;
     processes: z.infer<typeof FileProcessors>;
 }
 export interface UpdateUnprocessedFileResult {
-    byte_size: number;
-    last_received_byte: number;
+    byte_size?: number;
+    next_byte: number;
     id: string;
     processes: z.infer<typeof FileProcessors>;
 }
 export declare const createUnprocessedFile: (client: Client, payload: z.infer<typeof CreateUnprocessedFilePayload>) => Promise<CreateUnprocessedFileResult>;
 export declare const updateUnprocessedFile: (client: Client, fileId: string, payload: Partial<z.infer<typeof UpdateUnprocessedFilePayload>>) => Promise<UpdateUnprocessedFileResult>;
-export declare const uploadUnprocessedFile: (client: Client, fileId: string, fileStream: Readable) => Promise<unknown>;
+export declare const uploadUnprocessedFile: (client: Client, fileId: string, fileStream: Readable, fileSize: number) => Promise<void>;
 export declare const processAVFileBuffer: (client: Client, buffer: Buffer, filename?: string) => Promise<UpdateUnprocessedFileResult>;
 export declare const processAVFile: (client: Client, filePath: string) => Promise<UpdateUnprocessedFileResult>;
 export {};

package/dist/resources/unprocessed-file.js

@@ -1,9 +1,11 @@
 import { Readable } from "node:stream";
 import { basename } from "node:path";
 import { createReadStream } from "node:fs";
+import { stat } from "node:fs/promises";
 import { z } from "zod";
 import debug from "debug";
 import { md5Buffer, md5FilePath } from "@editframe/assets";
+import { uploadChunks } from "../uploadChunks.js";
 const log = debug("ef:api:unprocessed-file");
 const FileProcessors = z.array(z.union([z.literal("isobmff"), z.literal("captions")])).refine(
   (value) => {
@@ -13,16 +15,19 @@ const FileProcessors = z.array(z.union([z.literal("isobmff"), z.literal("caption
     message: "Processors list must not include duplicates"
   }
 );
+const MAX_FILE_SIZE = 1024 * 1024 * 1024;
 const CreateUnprocessedFilePayload = z.object({
   id: z.string(),
   filename: z.string(),
-  processes: FileProcessors.optional()
+  processes: FileProcessors.optional(),
+  byte_size: z.number().int().max(MAX_FILE_SIZE)
 });
 const UpdateUnprocessedFilePayload = z.object({
   processes: FileProcessors.optional()
 });
 const createUnprocessedFile = async (client, payload) => {
   log("Creating an unprocessed file", payload);
+  CreateUnprocessedFilePayload.parse(payload);
   const response = await client.authenticatedFetch(
     "/api/v1/unprocessed_files",
     {
@@ -36,21 +41,16 @@ const createUnprocessedFile = async (client, payload) => {
     response.statusText,
     response.headers
   );
-  switch (response.status) {
-    case 200: {
-      return await response.json();
-    }
-    default: {
-      console.error(
-        `Failed to create file ${response.status} ${response.statusText}`
-      );
-      console.error(await response.text());
-      throw new Error("Failed to create unprocessed file");
-    }
+  if (response.ok) {
+    return await response.json();
   }
+  throw new Error(
+    `Failed to create unprocessed file ${response.status} ${response.statusText}`
+  );
 };
 const updateUnprocessedFile = async (client, fileId, payload) => {
   log("Updating unprocessed file", fileId, payload);
+  UpdateUnprocessedFilePayload.parse(payload);
   const response = await client.authenticatedFetch(
     `/api/v1/unprocessed_files/${fileId}`,
     {
@@ -59,47 +59,32 @@ const updateUnprocessedFile = async (client, fileId, payload) => {
     }
   );
   log("Unprocessed file updated", response);
-  switch (response.status) {
-    case 200: {
-      return await response.json();
-    }
-    default: {
-      console.error(
-        `Failed to update file ${response.status} ${response.statusText}`
-      );
-      throw new Error("Failed to update unprocessed file");
-    }
+  if (response.ok) {
+    return await response.json();
   }
+  throw new Error(
+    `Failed to update unprocessed file ${response.status} ${response.statusText}`
+  );
 };
-const uploadUnprocessedFile = async (client, fileId, fileStream) => {
+const uploadUnprocessedFile = async (client, fileId, fileStream, fileSize) => {
   log("Uploading unprocessed file", fileId);
-  const unprocessedFile = await client.authenticatedFetch(
-    `/api/v1/unprocessed_files/${fileId}/upload`,
-    {
-      method: "POST",
-      body: fileStream
-    }
-  );
-  log("Unprocessed file track uploaded", unprocessedFile);
-  switch (unprocessedFile.status) {
-    case 200: {
-      return unprocessedFile.json();
-    }
-    default: {
-      console.error("Failed to upload unprocessed file");
-      console.error(unprocessedFile.status, unprocessedFile.statusText);
-      throw new Error("Failed to upload unprocessed file");
-    }
-  }
+  await uploadChunks(client, {
+    url: `/api/v1/unprocessed_files/${fileId}/upload`,
+    fileSize,
+    fileStream
+  });
+  log("Unprocessed file upload complete");
 };
 const processAVFileBuffer = async (client, buffer, filename = "buffer") => {
   log("Processing AV file buffer");
   const fileId = md5Buffer(buffer);
   log("File ID", fileId);
+  log(`File size: ${buffer.byteLength} bytes`);
   await createUnprocessedFile(client, {
     id: fileId,
     processes: [],
-    filename
+    filename,
+    byte_size: buffer.byteLength
   });
   const readStream = new Readable({
     read() {
@@ -107,7 +92,7 @@ const processAVFileBuffer = async (client, buffer, filename = "buffer") => {
       readStream.push(null);
     }
   });
-  await uploadUnprocessedFile(client, fileId, readStream);
+  await uploadUnprocessedFile(client, fileId, readStream, buffer.byteLength);
   const fileInformation = await updateUnprocessedFile(client, fileId, {
     processes: ["isobmff"]
   });
@@ -121,10 +106,16 @@ const processAVFile = async (client, filePath) => {
   await createUnprocessedFile(client, {
     id: fileId,
     processes: [],
-    filename: basename(filePath)
+    filename: basename(filePath),
+    byte_size: (await stat(filePath)).size
   });
   const readStream = createReadStream(filePath);
-  await uploadUnprocessedFile(client, fileId, readStream);
+  await uploadUnprocessedFile(
+    client,
+    fileId,
+    readStream,
+    (await stat(filePath)).size
+  );
   const fileInformation = await updateUnprocessedFile(client, fileId, {
     processes: ["isobmff"]
   });

package/dist/resources/unprocessed-file.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/resources/url-token.d.ts

@@ -1,5 +1,4 @@
 import { Client } from '../client.ts';
-
 export interface URLTokenResult {
     token: string;
 }

package/dist/resources/url-token.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/streamChunker.d.ts

@@ -0,0 +1,2 @@
+import { Readable } from 'node:stream';
+export declare function streamChunker(readableStream: Readable, chunkSize?: number): AsyncGenerator<Buffer, void, unknown>;

package/dist/streamChunker.js

@@ -0,0 +1,17 @@
+import { CHUNK_SIZE_BYTES } from "./CHUNK_SIZE_BYTES.js";
+async function* streamChunker(readableStream, chunkSize = CHUNK_SIZE_BYTES) {
+  let buffer = Buffer.alloc(0);
+  for await (const chunk of readableStream) {
+    buffer = Buffer.concat([buffer, chunk]);
+    while (buffer.length >= chunkSize) {
+      yield buffer.slice(0, chunkSize);
+      buffer = buffer.slice(chunkSize);
+    }
+  }
+  if (buffer.length > 0) {
+    yield buffer;
+  }
+}
+export {
+  streamChunker
+};

package/dist/streamChunker.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/uploadChunks.d.ts

@@ -0,0 +1,10 @@
+import { Readable } from 'node:stream';
+import { Client } from './client.ts';
+interface UploadChunksOptions {
+    url: string;
+    fileStream: Readable;
+    fileSize: number;
+    chunkSizeBytes?: number;
+}
+export declare function uploadChunks(client: Client, { url, fileSize, fileStream, chunkSizeBytes, }: UploadChunksOptions): Promise<void>;
+export {};

package/dist/uploadChunks.js

@@ -0,0 +1,65 @@
+import debug from "debug";
+import { streamChunker } from "./streamChunker.js";
+import { CHUNK_SIZE_BYTES } from "./CHUNK_SIZE_BYTES.js";
+const log = debug("ef:api:uploadChunk");
+const uploadChunk = async (client, {
+  url,
+  chunkBuffer,
+  chunkNumber,
+  fileSize,
+  chunkSizeBytes = CHUNK_SIZE_BYTES
+}) => {
+  const startByte = chunkNumber * chunkSizeBytes;
+  const endByte = startByte + chunkBuffer.length - 1;
+  log(`Uploading chunk ${chunkNumber} for ${url}`);
+  const response = await client.authenticatedFetch(url, {
+    method: "POST",
+    headers: {
+      "Content-Range": `bytes=${startByte}-${endByte}/${fileSize}`,
+      "Content-Type": "application/octet-stream"
+    },
+    body: chunkBuffer
+  });
+  if (response.ok) {
+    if (response.status === 201) {
+      log(`File ${url} fully uploaded`);
+      return { complete: true, body: await response.json() };
+    }
+    if (response.status === 202) {
+      log(`File ${url} chunk ${chunkNumber} uploaded`);
+      return { complete: false, body: await response.json() };
+    }
+  }
+  throw new Error(
+    `Failed to upload chunk ${chunkNumber} for ${url} ${response.status} ${response.statusText}`
+  );
+};
+async function uploadChunks(client, {
+  url,
+  fileSize,
+  fileStream,
+  chunkSizeBytes = CHUNK_SIZE_BYTES
+}) {
+  let chunkNumber = 0;
+  let complete = false;
+  for await (const chunkBuffer of streamChunker(fileStream, chunkSizeBytes)) {
+    log(`Uploading chunk ${chunkNumber}`);
+    ({ complete } = await uploadChunk(client, {
+      url,
+      chunkBuffer,
+      chunkNumber,
+      fileSize,
+      chunkSizeBytes
+    }));
+    chunkNumber++;
+  }
+  if (!fileStream.readableEnded) {
+    throw new Error("Did not read entire file stream");
+  }
+  if (!complete) {
+    throw new Error("Did not complete upload");
+  }
+}
+export {
+  uploadChunks
+};

package/dist/uploadChunks.test.d.ts

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@editframe/api",
-  "version": "0.8.0-beta.4",
+  "version": "0.8.0-beta.7",
   "description": "API functions for EditFrame",
   "exports": {
     ".": {
@@ -21,13 +21,14 @@
   "devDependencies": {
     "@types/jsonwebtoken": "^9.0.6",
     "@types/node": "^20.14.13",
+    "typedoc": "^0.26.5",
     "typescript": "^5.5.4",
     "vite": "^5.2.11",
-    "vite-plugin-dts": "^3.9.1",
+    "vite-plugin-dts": "^4.0.3",
     "vite-tsconfig-paths": "^4.3.2"
   },
   "dependencies": {
-    "@editframe/assets": "0.8.0-beta.4",
+    "@editframe/assets": "0.8.0-beta.7",
     "debug": "^4.3.5",
     "jsonwebtoken": "^9.0.2",
     "node-fetch": "^3.3.2",

package/src/resources/caption-file.test.ts

@@ -0,0 +1,124 @@
+import { test, expect, beforeAll, afterEach, afterAll, describe } from "vitest";
+import { http, HttpResponse } from "msw";
+import { setupServer } from "msw/node";
+
+import { Client } from "../client.ts";
+import { createCaptionFile, uploadCaptionFile } from "./caption-file.ts";
+import { readableFromBuffers } from "../readableFromBuffers.ts";
+
+const server = setupServer();
+const client = new Client("ef_TEST_TOKEN", "http://localhost");
+
+describe("CaptionFile", () => {
+  beforeAll(() => server.listen());
+  afterEach(() => server.resetHandlers());
+  afterAll(() => server.close());
+
+  describe("createCaptionFile", () => {
+    test("Throws when file is too large", async () => {
+      await expect(
+        createCaptionFile(client, {
+          id: "test-id",
+          filename: "test",
+          byte_size: 1024 * 1024 * 3,
+        }),
+      ).rejects.toThrowError(
+        "File size 3145728 bytes exceeds limit 2097152 bytes",
+      );
+    });
+
+    test("Throws when server returns an error", async () => {
+      server.use(
+        http.post("http://localhost/api/video2/caption_files", () =>
+          HttpResponse.text("Internal Server Error", { status: 500 }),
+        ),
+      );
+
+      await expect(
+        createCaptionFile(client, {
+          id: "test-id",
+          filename: "test",
+          byte_size: 4,
+        }),
+      ).rejects.toThrowError(
+        "Failed to create caption 500 Internal Server Error",
+      );
+    });
+
+    test("Returns json data from the http response", async () => {
+      server.use(
+        http.post("http://localhost/api/video2/caption_files", () =>
+          HttpResponse.json(
+            { id: "test-id" },
+            { status: 200, statusText: "OK" },
+          ),
+        ),
+      );
+
+      const response = await createCaptionFile(client, {
+        id: "test-id",
+        filename: "test",
+        byte_size: 4,
+      });
+
+      expect(response).toEqual({ id: "test-id" });
+    });
+  });
+
+  describe("uploadCaptionFile", () => {
+    test("Throws when file is too large", async () => {
+      await expect(
+        uploadCaptionFile(
+          client,
+          "test-id",
+          readableFromBuffers(Buffer.from("test")),
+          1024 * 1024 * 3,
+        ),
+      ).rejects.toThrowError(
+        "File size 3145728 bytes exceeds limit 2097152 bytes",
+      );
+    });
+
+    test("Throws when server returns an error", async () => {
+      server.use(
+        http.post(
+          "http://localhost/api/video2/caption_files/test-id/upload",
+          () => HttpResponse.text("Internal Server Error", { status: 500 }),
+        ),
+      );
+
+      await expect(
+        uploadCaptionFile(
+          client,
+          "test-id",
+          readableFromBuffers(Buffer.from("nice")),
+          4,
+        ),
+      ).rejects.toThrowError(
+        "Failed to upload caption 500 Internal Server Error",
+      );
+    });
+
+    test("Returns json data from the http response", async () => {
+      server.use(
+        http.post(
+          "http://localhost/api/video2/caption_files/test-id/upload",
+          () =>
+            HttpResponse.json(
+              { id: "test-id" },
+              { status: 200, statusText: "OK" },
+            ),
+        ),
+      );
+
+      const response = await uploadCaptionFile(
+        client,
+        "test-id",
+        readableFromBuffers(Buffer.from("nice")),
+        4,
+      );
+
+      expect(response).toEqual({ id: "test-id" });
+    });
+  });
+});