@editframe/api 0.11.0-beta.9 → 0.12.0-beta.10

package/src/resources/unprocessed-file.ts

@@ -1,71 +1,70 @@
-import { createReadStream } from "node:fs";
-import { stat } from "node:fs/promises";
-import { basename } from "node:path";
-import { Readable } from "node:stream";
-
 import debug from "debug";
 import { z } from "zod";
 
-import { md5Buffer, md5FilePath } from "@editframe/assets";
-
 import type { Client } from "../client.ts";
-import {
-  type IteratorWithPromise,
-  type UploadChunkEvent,
-  uploadChunks,
-} from "../uploadChunks.ts";
+import { uploadChunks } from "../uploadChunks.ts";
 
 const log = debug("ef:api:unprocessed-file");
 
-const FileProcessor = z.union([
-  z.literal("isobmff"),
-  z.literal("image"),
-  z.literal("captions"),
-  z.string(),
-]);
-
-const FileProcessors = z.array(FileProcessor).refine(
-  (value) => {
-    return new Set(value).size === value.length;
-  },
-  {
-    message: "Processors list must not include duplicates",
-  },
-);
-
 const MAX_FILE_SIZE = 1024 * 1024 * 1024; // 1GiB
 
 export const CreateUnprocessedFilePayload = z.object({
   md5: z.string(),
   filename: z.string(),
-  processes: FileProcessors.optional(),
   byte_size: z.number().int().max(MAX_FILE_SIZE),
 });
 
-export const UpdateUnprocessedFilePayload = z.object({
-  processes: FileProcessors.optional(),
-});
+export const UpdateUnprocessedFilePayload = z.object({});
 
-export interface CreateUnprocessedFileResult {
+interface UnprocessedFile {
   byte_size: number;
   next_byte: number;
   complete: boolean;
   id: string;
   md5: string;
-  processes: z.infer<typeof FileProcessors> & string[];
-  asset_id: string;
 }
 
-export interface UpdateUnprocessedFileResult {
-  byte_size?: number;
-  next_byte: number;
-  complete: boolean;
+type UnprocessedFileUploadDetails = Pick<UnprocessedFile, "id" | "byte_size">;
+
+export interface CreateUnprocessedFileResult extends UnprocessedFile {}
+
+export interface LookupUnprocessedFileByMd5Result extends UnprocessedFile {}
+
+export interface UpdateUnprocessedFileResult extends UnprocessedFile {}
+
+export interface ProcessIsobmffFileResult {
   id: string;
-  md5: string;
-  processes: z.infer<typeof FileProcessors>;
-  asset_id: string;
 }
 
+export const createUnprocessedFileFromPath = async (
+  client: Client,
+  path: string,
+) => {
+  const [{ stat }, { basename }, { md5FilePath }] = await Promise.all([
+    import("node:fs/promises"),
+    import("node:path"),
+    import("@editframe/assets"),
+  ]).catch((error) => {
+    console.error("Error importing modules", error);
+    console.error(
+      "This is likely because you are bundling for the browser. createUnprocessedFileFromPath can only be run in environments that support importing `node:path`",
+    );
+    throw error;
+  });
+
+  const fileInfo = await stat(path);
+
+  const byte_size = fileInfo.size;
+
+  const md5 = await md5FilePath(path);
+
+  return createUnprocessedFile(client, {
+    md5,
+    filename: basename(path),
+    byte_size,
+  });
+};
+
 export const createUnprocessedFile = async (
   client: Client,
   payload: z.infer<typeof CreateUnprocessedFilePayload>,
@@ -96,146 +95,77 @@ export const createUnprocessedFile = async (
   );
 };
 
-export const updateUnprocessedFile = async (
+export const uploadUnprocessedFile = async (
   client: Client,
-  fileId: string,
-  payload: Partial<z.infer<typeof UpdateUnprocessedFilePayload>>,
+  uploadDetails: UnprocessedFileUploadDetails,
+  path: string,
 ) => {
-  log("Updating unprocessed file", fileId, payload);
-  UpdateUnprocessedFilePayload.parse(payload);
-  const response = await client.authenticatedFetch(
-    `/api/v1/unprocessed_files/${fileId}`,
-    {
-      method: "POST",
-      body: JSON.stringify(payload),
-    },
-  );
-
-  log("Unprocessed file updated", response);
+  const { createReadStream } = await import("node:fs");
+  const readStream = createReadStream(path);
 
-  if (response.ok) {
-    return (await response.json()) as UpdateUnprocessedFileResult;
-  }
+  const { createReadableStreamFromReadable } = await import(
+    "packages/cli/src/utils/createReadableStreamFromReadable.ts"
+  );
 
-  throw new Error(
-    `Failed to update unprocessed file ${response.status} ${response.statusText}`,
+  return uploadUnprocessedReadableStream(
+    client,
+    uploadDetails,
+    createReadableStreamFromReadable(readStream),
   );
 };
 
-export const uploadUnprocessedFile = (
+export const uploadUnprocessedReadableStream = (
   client: Client,
-  fileId: string,
-  fileStream: Readable,
-  fileSize: number,
+  uploadDetails: UnprocessedFileUploadDetails,
+  fileStream: ReadableStream,
 ) => {
-  log("Uploading unprocessed file", fileId);
+  log("Uploading unprocessed file", uploadDetails.id);
 
   return uploadChunks(client, {
-    url: `/api/v1/unprocessed_files/${fileId}/upload`,
-    fileSize,
+    url: `/api/v1/unprocessed_files/${uploadDetails.id}/upload`,
+    fileSize: uploadDetails.byte_size,
     fileStream,
     maxSize: MAX_FILE_SIZE,
   });
 };
 
-const processResource = (
+export const lookupUnprocessedFileByMd5 = async (
   client: Client,
-  filename: string,
   md5: string,
-  byteSize: number,
-  processor: z.infer<typeof FileProcessor>,
-  doUpload: (id: string) => IteratorWithPromise<UploadChunkEvent>,
-) => {
-  log("Processing", { filename, md5, byteSize, processor });
+): Promise<LookupUnprocessedFileByMd5Result | null> => {
+  const response = await client.authenticatedFetch(
+    `/api/v1/unprocessed_files/md5/${md5}`,
+    {
+      method: "GET",
+    },
+  );
 
-  const createFilePromise = createUnprocessedFile(client, {
-    md5: md5,
-    processes: [],
-    filename,
-    byte_size: byteSize,
-  });
+  if (response.ok) {
+    return (await response.json()) as LookupUnprocessedFileByMd5Result;
+  }
 
-  return {
-    async *progress() {
-      const unprocessedFile = await createFilePromise;
-      if (unprocessedFile.complete) {
-        yield { type: "progress", progress: 0 };
-        yield { type: "progress", progress: 1 };
-      }
-      yield* doUpload(unprocessedFile.id);
-    },
-    file: async () => {
-      const unprocessedFile = await createFilePromise;
-      if (unprocessedFile.complete) {
-        return unprocessedFile;
-      }
-      await doUpload(unprocessedFile.id).whenUploaded();
-      const fileInformation = await updateUnprocessedFile(
-        client,
-        unprocessedFile.id,
-        {
-          processes: [processor],
-        },
-      );
-      log("File processed", fileInformation);
-      return fileInformation;
-    },
-  };
-};
+  if (response.status === 404) {
+    return null;
+  }
 
-const buildBufferProcessor = (processor: z.infer<typeof FileProcessor>) => {
-  return (client: Client, buffer: Buffer, filename = "buffer") => {
-    log(`Processing file buffer: ${processor}`, filename);
-    const md5 = md5Buffer(buffer);
-
-    return processResource(
-      client,
-      filename,
-      md5,
-      buffer.byteLength,
-      processor,
-      (id: string) => {
-        const readStream = new Readable({
-          read() {
-            readStream.push(buffer);
-            readStream.push(null);
-          },
-        });
-
-        return uploadUnprocessedFile(client, id, readStream, buffer.byteLength);
-      },
-    );
-  };
+  throw new Error(
+    `Failed to lookup unprocessed file by md5 ${md5} ${response.status} ${response.statusText}`,
+  );
 };
 
-const buildFileProcessor = (processor: z.infer<typeof FileProcessor>) => {
-  return (client: Client, filePath: string) => {
-    const processPromise = async () => {
-      const [md5, { size: byteSize }] = await Promise.all([
-        md5FilePath(filePath),
-        stat(filePath),
-      ]);
-      return processResource(
-        client,
-        basename(filePath),
-        md5,
-        byteSize,
-        processor,
-        (id: string) => {
-          const readStream = createReadStream(filePath);
-          return uploadUnprocessedFile(client, id, readStream, byteSize);
-        },
-      );
-    };
-    return {
-      progress: async () => (await processPromise()).progress(),
-      file: async () => (await processPromise()).file(),
-    };
-  };
-};
+export const processIsobmffFile = async (client: Client, id: string) => {
+  const response = await client.authenticatedFetch(
+    `/api/v1/unprocessed_files/${id}/isobmff`,
+    {
+      method: "POST",
+    },
+  );
 
-export const processAVFileBuffer = buildBufferProcessor("isobmff");
-export const processAVFile = buildFileProcessor("isobmff");
+  if (response.ok) {
+    return (await response.json()) as ProcessIsobmffFileResult;
+  }
 
-export const processImageFileBuffer = buildBufferProcessor("image");
-export const processImageFile = buildFileProcessor("image");
+  throw new Error(
+    `Failed to process isobmff file ${id} ${response.status} ${response.statusText}`,
+  );
+};

package/dist/client.js

@@ -1,35 +0,0 @@
-import debug from "debug";
-import fetch from "node-fetch";
-const log = debug("ef:api:client");
-class Client {
-  constructor(token, efHost = "https://editframe.dev") {
-    this.authenticatedFetch = async (path, init = {}) => {
-      init.headers ||= {};
-      const url = new URL(path, this.#efHost);
-      log(
-        "Authenticated fetch",
-        { url: url.toString(), init },
-        "(Token will be added as Bearer token)"
-      );
-      Object.assign(init.headers, {
-        Authorization: `Bearer ${this.#token}`,
-        "Content-Type": "application/json"
-      });
-      const response = await fetch(url, init);
-      log("Authenticated fetch response", response.status, response.statusText);
-      return response;
-    };
-    log("Creating client with efHost", { efHost, tokenIsSet: !!token });
-    this.#token = token;
-    this.#efHost = efHost;
-    const { apiKey, apiSecret } = token.match(/^(?<apiSecret>ef_[^_]+)_(?<apiKey>.+)$/)?.groups ?? {};
-    if (!apiKey || !apiSecret) {
-      throw new Error("Invalid token format. Must look like: ef_{}_{}");
-    }
-  }
-  #token;
-  #efHost;
-}
-export {
-  Client
-};

package/dist/index.js

@@ -1,36 +0,0 @@
-import { CreateCaptionFilePayload, createCaptionFile, uploadCaptionFile } from "./resources/caption-file.js";
-import { CreateImageFilePayload, createImageFile, uploadImageFile } from "./resources/image-file.js";
-import { CreateISOBMFFFilePayload, createISOBMFFFile, uploadFragmentIndex } from "./resources/isobmff-file.js";
-import { CreateISOBMFFTrackPayload, createISOBMFFTrack, uploadISOBMFFTrack } from "./resources/isobmff-track.js";
-import { CreateRenderPayload, createRender, uploadRender } from "./resources/renders.js";
-import { createURLToken } from "./resources/url-token.js";
-import { CreateUnprocessedFilePayload, UpdateUnprocessedFilePayload, createUnprocessedFile, processAVFile, processAVFileBuffer, processImageFile, processImageFileBuffer, updateUnprocessedFile, uploadUnprocessedFile } from "./resources/unprocessed-file.js";
-import { Client } from "./client.js";
-export {
-  Client,
-  CreateCaptionFilePayload,
-  CreateISOBMFFFilePayload,
-  CreateISOBMFFTrackPayload,
-  CreateImageFilePayload,
-  CreateRenderPayload,
-  CreateUnprocessedFilePayload,
-  UpdateUnprocessedFilePayload,
-  createCaptionFile,
-  createISOBMFFFile,
-  createISOBMFFTrack,
-  createImageFile,
-  createRender,
-  createURLToken,
-  createUnprocessedFile,
-  processAVFile,
-  processAVFileBuffer,
-  processImageFile,
-  processImageFileBuffer,
-  updateUnprocessedFile,
-  uploadCaptionFile,
-  uploadFragmentIndex,
-  uploadISOBMFFTrack,
-  uploadImageFile,
-  uploadRender,
-  uploadUnprocessedFile
-};

package/dist/resources/isobmff-file.js

@@ -1,47 +0,0 @@
-import { z } from "zod";
-import debug from "debug";
-const log = debug("ef:api:isobmff-file");
-const FILE_SIZE_LIMIT = 1024 * 1024 * 2;
-const CreateISOBMFFFilePayload = z.object({
-  md5: z.string(),
-  filename: z.string()
-});
-const createISOBMFFFile = async (client, payload) => {
-  log("Creating isobmff file", payload);
-  const response = await client.authenticatedFetch("/api/v1/isobmff_files", {
-    method: "POST",
-    body: JSON.stringify(payload)
-  });
-  log("ISOBMFF file created", response);
-  if (response.ok) {
-    return await response.json();
-  }
-  throw new Error(
-    `Failed to create isobmff file ${response.status} ${response.statusText}`
-  );
-};
-const uploadFragmentIndex = async (client, fileId, fileStream, fileSize) => {
-  log("Uploading fragment index", fileId);
-  if (fileSize > FILE_SIZE_LIMIT) {
-    throw new Error(`File size exceeds limit of ${FILE_SIZE_LIMIT} bytes`);
-  }
-  const response = await client.authenticatedFetch(
-    `/api/v1/isobmff_files/${fileId}/index/upload`,
-    {
-      method: "POST",
-      body: fileStream
-    }
-  );
-  log("Fragment index uploaded", response);
-  if (response.ok) {
-    return response.json();
-  }
-  throw new Error(
-    `Failed to create fragment index ${response.status} ${response.statusText}`
-  );
-};
-export {
-  CreateISOBMFFFilePayload,
-  createISOBMFFFile,
-  uploadFragmentIndex
-};

package/dist/resources/unprocessed-file.js

@@ -1,180 +0,0 @@
-import { createReadStream } from "node:fs";
-import { stat } from "node:fs/promises";
-import { basename } from "node:path";
-import { Readable } from "node:stream";
-import debug from "debug";
-import { z } from "zod";
-import { md5Buffer, md5FilePath } from "@editframe/assets";
-import { uploadChunks } from "../uploadChunks.js";
-const log = debug("ef:api:unprocessed-file");
-const FileProcessor = z.union([
-  z.literal("isobmff"),
-  z.literal("image"),
-  z.literal("captions"),
-  z.string()
-]);
-const FileProcessors = z.array(FileProcessor).refine(
-  (value) => {
-    return new Set(value).size === value.length;
-  },
-  {
-    message: "Processors list must not include duplicates"
-  }
-);
-const MAX_FILE_SIZE = 1024 * 1024 * 1024;
-const CreateUnprocessedFilePayload = z.object({
-  md5: z.string(),
-  filename: z.string(),
-  processes: FileProcessors.optional(),
-  byte_size: z.number().int().max(MAX_FILE_SIZE)
-});
-const UpdateUnprocessedFilePayload = z.object({
-  processes: FileProcessors.optional()
-});
-const createUnprocessedFile = async (client, payload) => {
-  log("Creating an unprocessed file", payload);
-  CreateUnprocessedFilePayload.parse(payload);
-  const response = await client.authenticatedFetch(
-    "/api/v1/unprocessed_files",
-    {
-      method: "POST",
-      body: JSON.stringify(payload)
-    }
-  );
-  log(
-    "Unprocessed file created",
-    response.status,
-    response.statusText,
-    response.headers
-  );
-  if (response.ok) {
-    return await response.json();
-  }
-  throw new Error(
-    `Failed to create unprocessed file ${response.status} ${response.statusText}`
-  );
-};
-const updateUnprocessedFile = async (client, fileId, payload) => {
-  log("Updating unprocessed file", fileId, payload);
-  UpdateUnprocessedFilePayload.parse(payload);
-  const response = await client.authenticatedFetch(
-    `/api/v1/unprocessed_files/${fileId}`,
-    {
-      method: "POST",
-      body: JSON.stringify(payload)
-    }
-  );
-  log("Unprocessed file updated", response);
-  if (response.ok) {
-    return await response.json();
-  }
-  throw new Error(
-    `Failed to update unprocessed file ${response.status} ${response.statusText}`
-  );
-};
-const uploadUnprocessedFile = (client, fileId, fileStream, fileSize) => {
-  log("Uploading unprocessed file", fileId);
-  return uploadChunks(client, {
-    url: `/api/v1/unprocessed_files/${fileId}/upload`,
-    fileSize,
-    fileStream,
-    maxSize: MAX_FILE_SIZE
-  });
-};
-const processResource = (client, filename, md5, byteSize, processor, doUpload) => {
-  log("Processing", { filename, md5, byteSize, processor });
-  const createFilePromise = createUnprocessedFile(client, {
-    md5,
-    processes: [],
-    filename,
-    byte_size: byteSize
-  });
-  return {
-    async *progress() {
-      const unprocessedFile = await createFilePromise;
-      if (unprocessedFile.complete) {
-        yield { type: "progress", progress: 0 };
-        yield { type: "progress", progress: 1 };
-      }
-      yield* doUpload(unprocessedFile.id);
-    },
-    file: async () => {
-      const unprocessedFile = await createFilePromise;
-      if (unprocessedFile.complete) {
-        return unprocessedFile;
-      }
-      await doUpload(unprocessedFile.id).whenUploaded();
-      const fileInformation = await updateUnprocessedFile(
-        client,
-        unprocessedFile.id,
-        {
-          processes: [processor]
-        }
-      );
-      log("File processed", fileInformation);
-      return fileInformation;
-    }
-  };
-};
-const buildBufferProcessor = (processor) => {
-  return (client, buffer, filename = "buffer") => {
-    log(`Processing file buffer: ${processor}`, filename);
-    const md5 = md5Buffer(buffer);
-    return processResource(
-      client,
-      filename,
-      md5,
-      buffer.byteLength,
-      processor,
-      (id) => {
-        const readStream = new Readable({
-          read() {
-            readStream.push(buffer);
-            readStream.push(null);
-          }
-        });
-        return uploadUnprocessedFile(client, id, readStream, buffer.byteLength);
-      }
-    );
-  };
-};
-const buildFileProcessor = (processor) => {
-  return (client, filePath) => {
-    const processPromise = async () => {
-      const [md5, { size: byteSize }] = await Promise.all([
-        md5FilePath(filePath),
-        stat(filePath)
-      ]);
-      return processResource(
-        client,
-        basename(filePath),
-        md5,
-        byteSize,
-        processor,
-        (id) => {
-          const readStream = createReadStream(filePath);
-          return uploadUnprocessedFile(client, id, readStream, byteSize);
-        }
-      );
-    };
-    return {
-      progress: async () => (await processPromise()).progress(),
-      file: async () => (await processPromise()).file()
-    };
-  };
-};
-const processAVFileBuffer = buildBufferProcessor("isobmff");
-const processAVFile = buildFileProcessor("isobmff");
-const processImageFileBuffer = buildBufferProcessor("image");
-const processImageFile = buildFileProcessor("image");
-export {
-  CreateUnprocessedFilePayload,
-  UpdateUnprocessedFilePayload,
-  createUnprocessedFile,
-  processAVFile,
-  processAVFileBuffer,
-  processImageFile,
-  processImageFileBuffer,
-  updateUnprocessedFile,
-  uploadUnprocessedFile
-};

package/dist/streamChunker.js

@@ -1,17 +0,0 @@
-import { CHUNK_SIZE_BYTES } from "./CHUNK_SIZE_BYTES.js";
-async function* streamChunker(readableStream, chunkSize = CHUNK_SIZE_BYTES) {
-  let buffer = Buffer.alloc(0);
-  for await (const chunk of readableStream) {
-    buffer = Buffer.concat([buffer, chunk]);
-    while (buffer.length >= chunkSize) {
-      yield buffer.slice(0, chunkSize);
-      buffer = buffer.slice(chunkSize);
-    }
-  }
-  if (buffer.length > 0) {
-    yield buffer;
-  }
-}
-export {
-  streamChunker
-};