d-drive-cli 1.1.3 → 1.3.0

package/README.md

@@ -40,6 +40,8 @@ Upload a single file:
 d-drive upload ./myfile.txt /backups/
 ```
 
+Note: For very large files the server exposes a streaming upload endpoint (`POST /api/files/upload/stream`) that accepts multipart uploads and streams chunks directly to the storage backend without full buffering. Use the API streaming endpoint (see `docs/API.md`) for multi-GB uploads or when you need more robust handling for long uploads.
+
 Upload a directory recursively:
 
 ```bash

package/dist/commands/upload.js

@@ -6,12 +6,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.uploadCommand = uploadCommand;
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const path_1 = __importDefault(require("path"));
+const stream_1 = require("stream");
 const chalk_1 = __importDefault(require("chalk"));
 const ora_1 = __importDefault(require("ora"));
 const form_data_1 = __importDefault(require("form-data"));
 const progress_1 = __importDefault(require("progress"));
 const api_1 = require("../api");
 const glob_1 = require("glob");
+const axios_1 = __importDefault(require("axios"));
 async function uploadCommand(source, destination = '/', options) {
     const spinner = (0, ora_1.default)('Preparing upload...').start();
     try {
@@ -61,27 +63,114 @@ async function uploadSingleFile(api, filePath, destination, showProgress) {
     console.log(chalk_1.default.cyan(`\nUploading: ${fileName}`));
     console.log(chalk_1.default.gray(`Size: ${formatFileSize(fileSize)}`));
     const formData = new form_data_1.default();
-    formData.append('file', fs_extra_1.default.createReadStream(filePath));
-    formData.append('path', destination);
+    // Create a passthrough so we can monitor bytes read from disk
+    const fileStream = fs_extra_1.default.createReadStream(filePath);
+    const pass = new stream_1.PassThrough();
+    // Pipe file stream into pass-through which is appended to form-data
+    fileStream.pipe(pass);
+    formData.append('file', pass, {
+        filename: fileName,
+        knownLength: fileSize,
+    });
+    // Resolve destination directory to a parentId and send parentId (server-authoritative)
+    const parentDir = path_1.default.posix.dirname(destination || '/');
+    let parentId = null;
+    if (parentDir && parentDir !== '/' && parentDir !== '.') {
+        parentId = await ensureFolderExists(api, parentDir);
+    }
+    if (parentId) {
+        formData.append('parentId', parentId);
+    }
     // Ensure CLI uploads follow frontend behavior and request server-side encryption by default
     formData.append('encrypt', 'true');
     let progressBar = null;
     if (showProgress) {
+        // Use a byte-counting progress bar so we can display an explicit percent.
         progressBar = new progress_1.default('[:bar] :percent :etas', {
             complete: '█',
             incomplete: '░',
             width: 40,
             total: fileSize,
         });
-    }
-    await api.post('/files/upload', formData, {
-        headers: formData.getHeaders(),
-        onUploadProgress: (progressEvent) => {
-            if (progressBar && progressEvent.loaded) {
-                progressBar.update(progressEvent.loaded / fileSize);
+        // Track bytes read from disk and update progress bar by bytes.
+        fileStream.on('data', (chunk) => {
+            const len = typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length;
+            if (progressBar)
+                progressBar.tick(len);
+        });
+        // When local read finishes, indicate we're waiting for the server to finish
+        fileStream.on('end', () => {
+            if (progressBar && !progressBar.complete) {
+                // ensure the bar shows very near completion but leave finalizing to server response
+                try {
+                    progressBar.update(Math.min(1, (progressBar.curr || 0) / (progressBar.total || 1)));
+                }
+                catch (_) { }
             }
-        },
+            console.log(chalk_1.default.gray('\nLocal file read complete — waiting for server to finish...'));
+        });
+    }
+    // Use streaming upload endpoint. Do not force Content-Length so the request
+    // can stream large files without buffering the whole body in memory.
+    const headers = formData.getHeaders();
+    // axios in Node needs the adapter to handle stream form-data; use api (axios instance)
+    await api.post('/files/upload/stream', formData, {
+        headers,
+        maxContentLength: Infinity,
+        maxBodyLength: Infinity,
+        // Do not set a timeout for potentially long uploads
+        timeout: 0,
+        // Allow axios to stream the form-data
+        transitional: { forcedJSONParsing: false },
     });
+    // Upload complete (server has processed). If progress bar exists, ensure it shows 100%.
+    if (progressBar && !progressBar.complete) {
+        try {
+            progressBar.update(progressBar.total || 1);
+        }
+        catch (_) { }
+    }
+    console.log(chalk_1.default.green('\nUpload finished'));
+}
+// Ensure the directory at `dirPath` exists. Returns the `id` of the directory or null for root.
+async function ensureFolderExists(api, dirPath) {
+    // Normalize and split
+    const normalized = path_1.default.posix.normalize(dirPath);
+    if (normalized === '/' || normalized === '.' || normalized === '')
+        return null;
+    const segments = normalized.split('/').filter(Boolean);
+    let currentPath = '';
+    let parentId = null;
+    for (const seg of segments) {
+        currentPath = `${currentPath}/${seg}`;
+        try {
+            const resp = await api.get('/files', { params: { path: currentPath } });
+            const items = resp.data;
+            const dir = items.find(i => i.type === 'DIRECTORY');
+            if (dir) {
+                parentId = dir.id;
+                continue;
+            }
+            // Not found — create it
+            const createResp = await api.post('/files/directory', { name: seg, parentId: parentId || null, path: currentPath });
+            parentId = createResp.data.id;
+        }
+        catch (err) {
+            // If a 409 or other error occurs, try to re-query; otherwise rethrow
+            if (axios_1.default.isAxiosError(err) && err.response) {
+                // Retry by querying again in case of race
+                const retry = await api.get('/files', { params: { path: currentPath } });
+                const items = retry.data;
+                const dir = items.find(i => i.type === 'DIRECTORY');
+                if (dir) {
+                    parentId = dir.id;
+                    continue;
+                }
+            }
+            throw err;
+        }
+    }
+    return parentId;
 }
 function formatFileSize(bytes) {
     if (bytes === 0)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "d-drive-cli",
-  "version": "1.1.3",
+  "version": "1.3.0",
   "description": "D-Drive CLI tool for developers",
   "main": "dist/index.js",
   "bin": {

package/src/commands/upload.ts

@@ -1,11 +1,13 @@
 import fs from 'fs-extra';
 import path from 'path';
+import { PassThrough } from 'stream';
 import chalk from 'chalk';
 import ora from 'ora';
 import FormData from 'form-data';
 import ProgressBar from 'progress';
 import { createApiClient } from '../api';
 import { glob } from 'glob';
+import axios from 'axios';
 
 interface UploadOptions {
   recursive?: boolean;
@@ -85,30 +87,113 @@ async function uploadSingleFile(
   console.log(chalk.gray(`Size: ${formatFileSize(fileSize)}`));
 
   const formData = new FormData();
-  formData.append('file', fs.createReadStream(filePath));
-  formData.append('path', destination);
+  // Create a passthrough so we can monitor bytes read from disk
+  const fileStream = fs.createReadStream(filePath);
+  const pass = new PassThrough();
+  // Pipe file stream into pass-through which is appended to form-data
+  fileStream.pipe(pass);
+  formData.append('file', pass, {
+    filename: fileName,
+    knownLength: fileSize,
+  });
+  // Resolve destination directory to a parentId and send parentId (server-authoritative)
+  const parentDir = path.posix.dirname(destination || '/');
+  let parentId: string | null = null;
+  if (parentDir && parentDir !== '/' && parentDir !== '.') {
+    parentId = await ensureFolderExists(api, parentDir);
+  }
+  if (parentId) {
+    formData.append('parentId', parentId);
+  }
   // Ensure CLI uploads follow frontend behavior and request server-side encryption by default
   formData.append('encrypt', 'true');
 
   let progressBar: ProgressBar | null = null;
 
   if (showProgress) {
+    // Use a byte-counting progress bar so we can display an explicit percent.
     progressBar = new ProgressBar('[:bar] :percent :etas', {
       complete: '█',
       incomplete: '░',
       width: 40,
       total: fileSize,
     });
-  }
 
-  await api.post('/files/upload', formData, {
-    headers: formData.getHeaders(),
-    onUploadProgress: (progressEvent: any) => {
-      if (progressBar && progressEvent.loaded) {
-        progressBar.update(progressEvent.loaded / fileSize);
+    // Track bytes read from disk and update progress bar by bytes.
+    fileStream.on('data', (chunk: Buffer | string) => {
+      const len = typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length;
+      if (progressBar) progressBar.tick(len);
+    });
+    // When local read finishes, indicate we're waiting for the server to finish
+    fileStream.on('end', () => {
+      if (progressBar && !progressBar.complete) {
+        // ensure the bar shows very near completion but leave finalizing to server response
+        try { progressBar.update(Math.min(1, (progressBar.curr || 0) / (progressBar.total || 1))); } catch (_) {}
       }
-    },
+      console.log(chalk.gray('\nLocal file read complete — waiting for server to finish...'));
+    });
+  }
+
+  // Use streaming upload endpoint. Do not force Content-Length so the request
+  // can stream large files without buffering the whole body in memory.
+  const headers = formData.getHeaders();
+  // axios in Node needs the adapter to handle stream form-data; use api (axios instance)
+  await api.post('/files/upload/stream', formData, {
+    headers,
+    maxContentLength: Infinity,
+    maxBodyLength: Infinity,
+    // Do not set a timeout for potentially long uploads
+    timeout: 0,
+    // Allow axios to stream the form-data
+    transitional: { forcedJSONParsing: false },
   });
+  // Upload complete (server has processed). If progress bar exists, ensure it shows 100%.
+  if (progressBar && !progressBar.complete) {
+    try { progressBar.update(progressBar.total || 1); } catch (_) {}
+  }
+  console.log(chalk.green('\nUpload finished'));
+}
+
+// Ensure the directory at `dirPath` exists. Returns the `id` of the directory or null for root.
+async function ensureFolderExists(api: any, dirPath: string): Promise<string | null> {
+  // Normalize and split
+  const normalized = path.posix.normalize(dirPath);
+  if (normalized === '/' || normalized === '.' || normalized === '') return null;
+
+  const segments = normalized.split('/').filter(Boolean);
+  let currentPath = '';
+  let parentId: string | null = null;
+
+  for (const seg of segments) {
+    currentPath = `${currentPath}/${seg}`;
+    try {
+      const resp = await api.get('/files', { params: { path: currentPath } });
+      const items = resp.data as any[];
+      const dir = items.find(i => i.type === 'DIRECTORY');
+      if (dir) {
+        parentId = dir.id;
+        continue;
+      }
+      // Not found — create it
+      const createResp = await api.post('/files/directory', { name: seg, parentId: parentId || null, path: currentPath });
+      parentId = createResp.data.id;
+    } catch (err: any) {
+      // If a 409 or other error occurs, try to re-query; otherwise rethrow
+      if (axios.isAxiosError(err) && err.response) {
+        // Retry by querying again in case of race
+        const retry = await api.get('/files', { params: { path: currentPath } });
+        const items = retry.data as any[];
+        const dir = items.find(i => i.type === 'DIRECTORY');
+        if (dir) {
+          parentId = dir.id;
+          continue;
+        }
+      }
+      throw err;
+    }
+  }
+
+  return parentId;
 }
 
 function formatFileSize(bytes: number): string {