@autonomys/auto-drive 1.0.3 → 1.0.5

package/src/api/models/uploads.ts

@@ -1,4 +1,4 @@
-import { FileUploadOptions } from '@autonomys/auto-dag-data'
+import { CID, FileUploadOptions } from '@autonomys/auto-dag-data'
 import { z } from 'zod'
 import { FolderTreeFolderSchema } from './folderTree.js'
 
@@ -51,3 +51,19 @@ export type FolderUpload = z.infer<typeof folderUploadSchema>
 export type CompleteUploadResponse = {
   cid: string
 }
+
+export type UploadFileStatus = {
+  type: 'file'
+  progress: number
+  cid?: CID
+}
+
+export type UploadFolderStatus = {
+  type: 'folder'
+  progress: number
+  cid?: CID
+}
+
+export type UploadChunksStatus = {
+  uploadBytes: number
+}

package/src/api/wrappers.ts

@@ -10,8 +10,10 @@ import {
 } from '@autonomys/auto-dag-data'
 import fs from 'fs'
 import mime from 'mime-types'
-import { asyncByChunk, asyncFromStream } from '../utils/async.js'
+import { asyncByChunk, asyncFromStream, fileToIterable } from '../utils/async.js'
 import { getFiles } from '../utils/folder.js'
+import { progressToPercentage } from '../utils/misc.js'
+import { PromisedObservable } from '../utils/observable.js'
 import {
   completeUpload,
   createFileUpload,
@@ -22,7 +24,13 @@ import {
   uploadFileChunk,
 } from './calls/index.js'
 import { AutoDriveApi } from './connection.js'
-import { constructFromFileSystemEntries } from './models/folderTree.js'
+import { GenericFile } from './models/file.js'
+import {
+  constructFromFileSystemEntries,
+  constructFromInput,
+  constructZipBlobFromTreeAndPaths,
+} from './models/folderTree.js'
+import { UploadChunksStatus, UploadFileStatus, UploadFolderStatus } from './models/uploads.js'
 
 type UploadFileOptions = {
   password?: string
@@ -31,80 +39,135 @@ type UploadFileOptions = {
 
 const UPLOAD_FILE_CHUNK_SIZE = 1024 * 1024
 
-const uploadFileChunks = async (
+const uploadFileChunks = (
   api: AutoDriveApi,
   fileUploadId: string,
   asyncIterable: AsyncIterable<Buffer>,
   uploadChunkSize: number = UPLOAD_FILE_CHUNK_SIZE,
-) => {
-  let index = 0
-  for await (const chunk of asyncByChunk(asyncIterable, uploadChunkSize)) {
-    await uploadFileChunk(api, { uploadId: fileUploadId, chunk, index })
-    index++
-  }
+): PromisedObservable<UploadChunksStatus> => {
+  return new PromisedObservable<UploadChunksStatus>(async (subscriber) => {
+    let index = 0
+    let uploadBytes = 0
+    for await (const chunk of asyncByChunk(asyncIterable, uploadChunkSize)) {
+      await uploadFileChunk(api, { uploadId: fileUploadId, chunk, index })
+      uploadBytes += chunk.length
+      subscriber.next({ uploadBytes })
+      index++
+    }
+    subscriber.complete()
+  })
 }
 
 /**
  * Uploads a file to the server with optional encryption and compression.
  *
  * This function reads a file from the specified file path, optionally encrypts it
- * using the provided password, and compresses it using the ZLIB algorithm if specified.
+ * using the provided password, and compresses it using the specified algorithm if requested.
  * It then uploads the file in chunks to the server, creating an upload session and
- * completing it once all chunks have been uploaded.
+ * completing it once all chunks have been successfully uploaded.
  *
  * @param {AutoDriveApi} api - The API instance used to send requests.
  * @param {string} filePath - The path to the file to be uploaded.
  * @param {UploadFileOptions} options - Options for the upload process.
  * @param {string} [options.password] - The password for encryption (optional).
  * @param {boolean} [options.compression=true] - Whether to compress the file (optional).
- * @returns {Promise<void>} - A promise that resolves when the upload is complete.
+ * @param {number} [uploadChunkSize] - The size of each chunk to upload (optional).
+ * @returns {PromisedObservable<UploadFileStatus>} - An observable that emits the upload status.
  * @throws {Error} - Throws an error if the upload fails at any stage.
  */
-export const uploadFile = async (
+export const uploadFileFromFilepath = (
   api: AutoDriveApi,
   filePath: string,
   { password, compression = true }: UploadFileOptions,
   uploadChunkSize?: number,
-): Promise<CID> => {
-  let asyncIterable: AsyncIterable<Buffer> = fs.createReadStream(filePath)
+): PromisedObservable<UploadFileStatus> => {
+  const name = filePath.split('/').pop()!
 
-  if (compression) {
-    asyncIterable = compressFile(asyncIterable, {
-      level: 9,
-      algorithm: CompressionAlgorithm.ZLIB,
-    })
-  }
+  return uploadFileFromInput(
+    api,
+    {
+      read: () => fs.createReadStream(filePath),
+      name,
+      mimeType: mime.lookup(name) || undefined,
+      size: fs.statSync(filePath).size,
+      path: filePath,
+    },
+    {
+      password,
+      compression,
+    },
+    uploadChunkSize,
+  )
+}
 
-  if (password) {
-    asyncIterable = encryptFile(asyncIterable, password, {
-      algorithm: EncryptionAlgorithm.AES_256_GCM,
-    })
-  }
+/**
+ * Uploads a file to the server with optional encryption and compression.
+ *
+ * This function reads a file from the provided input, optionally encrypts it
+ * using the specified password, and compresses it using the specified algorithm if requested.
+ * It uploads the file in chunks to the server, creating an upload session and
+ * completing it once all chunks have been successfully uploaded.
+ *
+ * @param {AutoDriveApi} api - The API instance used to send requests.
+ * @param {File | GenericFile} file - The file to be uploaded, which can be a File or a GenericFile.
+ * @param {UploadFileOptions} options - Options for the upload process.
+ * @param {string} [options.password] - The password for encryption (optional).
+ * @param {boolean} [options.compression=true] - Whether to compress the file (optional).
+ * @param {number} [uploadChunkSize] - The size of each chunk to upload (optional).
+ * @returns {Promise<CID>} - A promise that resolves with the CID of the uploaded file.
+ * @throws {Error} - Throws an error if the upload fails at any stage.
+ */
+export const uploadFileFromInput = (
+  api: AutoDriveApi,
+  file: File | GenericFile,
+  { password, compression = true }: UploadFileOptions,
+  uploadChunkSize?: number,
+): PromisedObservable<UploadFileStatus> => {
+  return new PromisedObservable<UploadFileStatus>(async (subscriber) => {
+    let asyncIterable: AsyncIterable<Buffer> =
+      file instanceof File ? fileToIterable(file) : file.read()
 
-  const uploadOptions = {
-    compression: compression
-      ? {
-          level: 9,
-          algorithm: CompressionAlgorithm.ZLIB,
-        }
-      : undefined,
-    encryption: password
-      ? {
-          algorithm: EncryptionAlgorithm.AES_256_GCM,
-        }
-      : undefined,
-  }
-  const fileUpload = await createFileUpload(api, {
-    mimeType: mime.lookup(filePath) || undefined,
-    filename: filePath.split('/').pop()!,
-    uploadOptions,
-  })
+    if (compression) {
+      asyncIterable = compressFile(asyncIterable, {
+        level: 9,
+        algorithm: CompressionAlgorithm.ZLIB,
+      })
+    }
 
-  await uploadFileChunks(api, fileUpload.id, asyncIterable, uploadChunkSize)
+    if (password) {
+      asyncIterable = encryptFile(asyncIterable, password, {
+        algorithm: EncryptionAlgorithm.AES_256_GCM,
+      })
+    }
 
-  const result = await completeUpload(api, { uploadId: fileUpload.id })
+    const uploadOptions = {
+      compression: compression
+        ? {
+            level: 9,
+            algorithm: CompressionAlgorithm.ZLIB,
+          }
+        : undefined,
+      encryption: password
+        ? {
+            algorithm: EncryptionAlgorithm.AES_256_GCM,
+          }
+        : undefined,
+    }
+    const fileUpload = await createFileUpload(api, {
+      mimeType: mime.lookup(file.name) || undefined,
+      filename: file.name,
+      uploadOptions,
+    })
 
-  return stringToCid(result.cid)
+    await uploadFileChunks(api, fileUpload.id, asyncIterable, uploadChunkSize).forEach((e) =>
+      subscriber.next({ type: 'file', progress: progressToPercentage(e.uploadBytes, file.size) }),
+    )
+
+    const result = await completeUpload(api, { uploadId: fileUpload.id })
+
+    subscriber.next({ type: 'file', progress: 100, cid: stringToCid(result.cid) })
+    subscriber.complete()
+  })
 }
 
 /**
@@ -112,40 +175,170 @@ export const uploadFile = async (
  *
  * This function retrieves all files within the specified folder,
  * constructs a file tree representation, and initiates the upload
- * process. It also handles optional compression of the files during
+ * process. It also handles optional compression and encryption of the files during
  * the upload.
  *
+ * If a password is provided, the files will be zipped before uploading.
+ *
  * @param {AutoDriveApi} api - The API instance used to send requests.
  * @param {string} folderPath - The path of the folder to be uploaded.
- * @returns {Promise<void>} - A promise that resolves when the folder upload is complete.
+ * @param {Object} options - Optional parameters for the upload.
+ * @param {number} [options.uploadChunkSize] - The size of each chunk to be uploaded.
+ * @param {string} [options.password] - An optional password for encrypting the files.
+ * @returns {Promise<PromisedObservable<UploadFileStatus | UploadFolderStatus>>} - A promise that resolves to an observable that tracks the upload progress.
  * @throws {Error} - Throws an error if the upload fails at any stage.
  */
-export const uploadFolder = async (
+export const uploadFolderFromFolderPath = async (
   api: AutoDriveApi,
   folderPath: string,
-  uploadChunkSize?: number,
-): Promise<CID> => {
+  { uploadChunkSize, password }: { uploadChunkSize?: number; password?: string } = {},
+): Promise<PromisedObservable<UploadFileStatus | UploadFolderStatus>> => {
   const files = await getFiles(folderPath)
-
   const fileTree = constructFromFileSystemEntries(files)
 
-  const folderUpload = await createFolderUpload(api, {
-    fileTree,
-    uploadOptions: {
-      compression: {
-        algorithm: CompressionAlgorithm.ZLIB,
-        level: 9,
+  if (password) {
+    const filesMap = Object.fromEntries(files.map((file) => [file, file]))
+    const zipBlob = await constructZipBlobFromTreeAndPaths(fileTree, filesMap)
+    const name = folderPath.split('/').pop()!
+    return uploadFileFromInput(
+      api,
+      {
+        read: () => fileToIterable(zipBlob),
+        name,
+        mimeType: 'application/zip',
+        size: zipBlob.size,
+        path: name,
       },
-    },
+      {
+        password,
+        compression: true,
+      },
+    )
+  }
+
+  return new PromisedObservable<UploadFolderStatus>(async (subscriber) => {
+    const folderUpload = await createFolderUpload(api, {
+      fileTree,
+      uploadOptions: {
+        compression: {
+          algorithm: CompressionAlgorithm.ZLIB,
+          level: 9,
+        },
+      },
+    })
+
+    const genericFiles: GenericFile[] = files.map((file) => ({
+      read: () => fs.createReadStream(file),
+      name: file.split('/').pop()!,
+      mimeType: mime.lookup(file.split('/').pop()!) || undefined,
+      size: fs.statSync(file).size,
+      path: file,
+    }))
+
+    const totalSize = genericFiles.reduce((acc, file) => acc + file.size, 0)
+
+    let progress = 0
+    for (const file of genericFiles) {
+      await uploadFileWithinFolderUpload(api, folderUpload.id, file, uploadChunkSize).forEach((e) =>
+        subscriber.next({
+          type: 'folder',
+          progress: progressToPercentage(progress + e.uploadBytes, totalSize),
+        }),
+      )
+      progress += file.size
+    }
+
+    const result = await completeUpload(api, { uploadId: folderUpload.id })
+
+    subscriber.next({ type: 'folder', progress: 100, cid: stringToCid(result.cid) })
+    subscriber.complete()
   })
+}
 
-  for (const file of files) {
-    await uploadFileWithinFolderUpload(api, folderUpload.id, file, uploadChunkSize)
+/**
+ * Uploads an entire folder to the server.
+ *
+ * This function retrieves all files within the specified folder,
+ * constructs a file tree representation, and initiates the upload
+ * process. It also handles optional compression of the files during
+ * the upload. If a password is provided, the files will be zipped
+ * before uploading.
+ *
+ * @param {AutoDriveApi} api - The API instance used to send requests.
+ * @param {FileList | File[]} fileList - The list of files to be uploaded.
+ * @param {Object} options - Options for the upload process.
+ * @param {number} [options.uploadChunkSize] - The size of each chunk to upload (optional).
+ * @param {string} [options.password] - The password for encryption (optional).
+ * @returns {PromisedObservable<UploadFileStatus | UploadFolderStatus>} - An observable that emits the upload status.
+ * @throws {Error} - Throws an error if the upload fails at any stage.
+ */
+export const uploadFolderFromInput = async (
+  api: AutoDriveApi,
+  fileList: FileList | File[],
+  { uploadChunkSize, password }: { uploadChunkSize?: number; password?: string } = {},
+): Promise<PromisedObservable<UploadFileStatus | UploadFolderStatus>> => {
+  const files = fileList instanceof FileList ? Array.from(fileList) : fileList
+  const fileTree = constructFromInput(files)
+
+  // If password is provided, we zip the files and upload the zip file
+  if (password) {
+    const filesMap: Record<string, File> = Object.fromEntries(
+      files.map((file) => [file.webkitRelativePath, file]),
+    )
+    const zipBlob = await constructZipBlobFromTreeAndPaths(fileTree, filesMap)
+    const name = fileList[0].webkitRelativePath.split('/').filter(Boolean)[0]!
+
+    return uploadFileFromInput(
+      api,
+      {
+        read: () => fileToIterable(zipBlob),
+        name: `${name}.zip`,
+        mimeType: 'application/zip',
+        size: zipBlob.size,
+        path: name,
+      },
+      {
+        password,
+        compression: true,
+      },
+    )
   }
 
-  const result = await completeUpload(api, { uploadId: folderUpload.id })
+  return new PromisedObservable<UploadFolderStatus>(async (subscriber) => {
+    // Otherwise, we upload the files as a folder w/o compression or encryption
+    const folderUpload = await createFolderUpload(api, {
+      fileTree,
+    })
+
+    let currentBytesUploaded = 0
+    const totalSize = files.reduce((acc, file) => acc + file.size, 0)
+    for (const file of files) {
+      await uploadFileWithinFolderUpload(
+        api,
+        folderUpload.id,
+        {
+          read: () => fileToIterable(file),
+          name: file.name,
+          mimeType: mime.lookup(file.name) || undefined,
+          size: file.size,
+          path: file.webkitRelativePath,
+        },
+        uploadChunkSize,
+      ).forEach((e) => {
+        subscriber.next({
+          type: 'folder',
+          progress: progressToPercentage(currentBytesUploaded + e.uploadBytes, totalSize),
+        })
+      })
+
+      currentBytesUploaded += file.size
+    }
+
+    const result = await completeUpload(api, { uploadId: folderUpload.id })
 
-  return stringToCid(result.cid)
+    subscriber.next({ type: 'folder', progress: 100, cid: stringToCid(result.cid) })
+    subscriber.complete()
+  })
 }
 
 /**
@@ -157,26 +350,29 @@ export const uploadFolder = async (
  *
  * @returns {Promise<void>} A promise that resolves when the file upload is complete.
  */
-export const uploadFileWithinFolderUpload = async (
+export const uploadFileWithinFolderUpload = (
   api: AutoDriveApi,
   uploadId: string,
-  filepath: string,
+  file: GenericFile,
   uploadChunkSize?: number,
-): Promise<CID> => {
-  const name = filepath.split('/').pop()!
-  const fileUpload = await createFileUploadWithinFolderUpload(api, {
-    uploadId,
-    name,
-    mimeType: mime.lookup(name) || undefined,
-    relativeId: filepath,
-    uploadOptions: {},
-  })
+): PromisedObservable<UploadChunksStatus> => {
+  return new PromisedObservable<UploadChunksStatus>(async (subscriber) => {
+    const fileUpload = await createFileUploadWithinFolderUpload(api, {
+      uploadId,
+      name: file.name,
+      mimeType: file.mimeType,
+      relativeId: file.path,
+      uploadOptions: {},
+    })
 
-  await uploadFileChunks(api, fileUpload.id, fs.createReadStream(filepath), uploadChunkSize)
+    await uploadFileChunks(api, fileUpload.id, file.read(), uploadChunkSize).forEach((e) =>
+      subscriber.next({ uploadBytes: e.uploadBytes }),
+    )
 
-  const result = await completeUpload(api, { uploadId: fileUpload.id })
+    await completeUpload(api, { uploadId: fileUpload.id })
 
-  return stringToCid(result.cid)
+    subscriber.complete()
+  })
 }
 
 /**
@@ -211,27 +407,3 @@ export const downloadFile = async (
 
   return iterable
 }
-
-/**
- * Downloads a folder from the AutoDrive service without encryption.
- *
- * @param {AutoDriveApi} api - The API instance to interact with the AutoDrive service.
- * @param {string} cid - The CID of the folder to be downloaded.
- *
- * @returns {Promise<ReadableStream<Uint8Array>>} A promise that resolves to a ReadableStream of the downloaded folder.
- *
- * @warning If the folder is encrypted, a warning will be logged, but the download will proceed without decryption.
- */
-export const downloadFolderWithoutEncryption = async (
-  api: AutoDriveApi,
-  cid: string,
-): Promise<ReadableStream<Uint8Array>> => {
-  const metadata = await getObjectMetadata(api, { cid })
-  if (metadata.uploadOptions?.encryption) {
-    console.warn(
-      'Downloading encrypted folder. Folder support encryption but it is not recommended.',
-    )
-  }
-
-  return downloadObject(api, { cid })
-}

package/src/utils/async.ts

@@ -27,3 +27,10 @@ export const asyncFromStream = async function* (
     result = await reader.read()
   }
 }
+
+export const fileToIterable = async function* (file: File | Blob): AsyncIterable<Buffer> {
+  const chunkSize = 1024 * 1024
+  for (let i = 0; i < file.size; i += chunkSize) {
+    yield Buffer.from(await file.slice(i, i + chunkSize).arrayBuffer())
+  }
+}

package/src/utils/misc.ts

@@ -0,0 +1 @@
+export const progressToPercentage = (current: number, target: number) => (current / target) * 100

package/src/utils/observable.ts

@@ -0,0 +1,19 @@
+import { firstValueFrom, lastValueFrom, Observable, Subscriber } from 'rxjs'
+
+const asyncCallback = <T, O>(callback: (t: T) => O) => {
+  return (t: T) => {
+    callback(t)
+  }
+}
+
+export class PromisedObservable<T> extends Observable<T> {
+  constructor(subscribe?: (this: Observable<T>, subscriber: Subscriber<T>) => void) {
+    super(subscribe && asyncCallback(subscribe))
+  }
+
+  get promise(): Promise<T> {
+    return lastValueFrom(this)
+  }
+}
+
+export { firstValueFrom, lastValueFrom }

package/dist/cjs/api/connection.d.ts

@@ -1,7 +0,0 @@
-export interface AutoDriveApi {
-    sendRequest: (relativeUrl: string, request: Partial<Request>, body?: BodyInit) => Promise<Response>;
-}
-export declare const createAutoDriveApi: ({ apiKey, url, }: {
-    apiKey: string;
-    url?: string;
-}) => AutoDriveApi;

package/dist/cjs/api/models/folderTree.d.ts

@@ -1,29 +0,0 @@
-import { z } from 'zod';
-export type FolderTreeFolder = {
-    name: string;
-    type: 'folder';
-    children: FolderTree[];
-    id: string;
-};
-export type FolderTreeFile = {
-    name: string;
-    type: 'file';
-    id: string;
-};
-export type FolderTree = FolderTreeFolder | FolderTreeFile;
-export declare const FolderTreeFolderSchema: any;
-export declare const FolderTreeFileSchema: z.ZodObject<{
-    name: z.ZodString;
-    type: z.ZodLiteral<"file">;
-    id: z.ZodString;
-}, "strip", z.ZodTypeAny, {
-    name?: string;
-    type?: "file";
-    id?: string;
-}, {
-    name?: string;
-    type?: "file";
-    id?: string;
-}>;
-export declare const FolderTreeSchema: any;
-export declare const constructFromFileSystemEntries: (entries: string[]) => FolderTree;

package/dist/cjs/api/models/uploads.d.ts

@@ -1,113 +0,0 @@
-import { FileUploadOptions } from '@autonomys/auto-dag-data';
-import { z } from 'zod';
-export declare enum UploadType {
-    FILE = "file",
-    FOLDER = "folder"
-}
-export declare enum UploadStatus {
-    PENDING = "pending",
-    COMPLETED = "completed",
-    MIGRATING = "migrating",
-    CANCELLED = "cancelled",
-    FAILED = "failed"
-}
-export declare const fileUploadSchema: z.ZodObject<{
-    id: z.ZodString;
-    rootId: z.ZodString;
-    relativeId: z.ZodNullable<z.ZodString>;
-    type: z.ZodNativeEnum<typeof UploadType>;
-    status: z.ZodNativeEnum<typeof UploadStatus>;
-    fileTree: z.ZodNull;
-    name: z.ZodString;
-    mimeType: z.ZodNullable<z.ZodString>;
-    oauthProvider: z.ZodString;
-    oauthUserId: z.ZodString;
-}, "strip", z.ZodTypeAny, {
-    id?: string;
-    rootId?: string;
-    relativeId?: string;
-    type?: UploadType;
-    status?: UploadStatus;
-    fileTree?: null;
-    name?: string;
-    mimeType?: string;
-    oauthProvider?: string;
-    oauthUserId?: string;
-}, {
-    id?: string;
-    rootId?: string;
-    relativeId?: string;
-    type?: UploadType;
-    status?: UploadStatus;
-    fileTree?: null;
-    name?: string;
-    mimeType?: string;
-    oauthProvider?: string;
-    oauthUserId?: string;
-}>;
-export type FileUpload = z.infer<typeof fileUploadSchema> & {
-    uploadOptions: FileUploadOptions | null;
-};
-export declare const folderUploadSchema: z.ZodObject<{
-    id: z.ZodString;
-    rootId: z.ZodString;
-    relativeId: z.ZodNullable<z.ZodString>;
-    type: z.ZodNativeEnum<typeof UploadType>;
-    status: z.ZodNativeEnum<typeof UploadStatus>;
-    fileTree: z.ZodObject<{
-        name: z.ZodString;
-        type: z.ZodLiteral<"folder">;
-        children: z.ZodArray<z.ZodLazy<any>, "many">;
-        id: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
-        id?: string;
-        type?: "folder";
-        name?: string;
-        children?: any[];
-    }, {
-        id?: string;
-        type?: "folder";
-        name?: string;
-        children?: any[];
-    }>;
-    name: z.ZodString;
-    mimeType: z.ZodNull;
-    oauthProvider: z.ZodString;
-    oauthUserId: z.ZodString;
-    uploadOptions: z.ZodNull;
-}, "strip", z.ZodTypeAny, {
-    id?: string;
-    rootId?: string;
-    relativeId?: string;
-    type?: UploadType;
-    status?: UploadStatus;
-    fileTree?: {
-        id?: string;
-        type?: "folder";
-        name?: string;
-        children?: any[];
-    };
-    name?: string;
-    mimeType?: null;
-    oauthProvider?: string;
-    oauthUserId?: string;
-    uploadOptions?: null;
-}, {
-    id?: string;
-    rootId?: string;
-    relativeId?: string;
-    type?: UploadType;
-    status?: UploadStatus;
-    fileTree?: {
-        id?: string;
-        type?: "folder";
-        name?: string;
-        children?: any[];
-    };
-    name?: string;
-    mimeType?: null;
-    oauthProvider?: string;
-    oauthUserId?: string;
-    uploadOptions?: null;
-}>;
-export type FolderUpload = z.infer<typeof folderUploadSchema>;