@rowengine/common 1.1.7 → 1.1.9

package/build/utils/gc.d.ts

@@ -1,13 +1,3 @@
-interface UploadFileToGCS {
-    filePath: string;
-    gcsFilePath: string;
-}
-interface UploadFileToGCSResponse {
-    filename: string;
-    size: number;
-}
-export declare const uploadFileToGCS: ({ filePath, gcsFilePath, }: UploadFileToGCS) => Promise<UploadFileToGCSResponse>;
-export declare const uploadFolderToGCS: (folderPath: string, gcsPath: string) => Promise<void>;
+export declare function uploadFolderToGCS(folderPath: string, gcsPath: string): Promise<void>;
 export declare const getFilesFromGCSFolder: (gcsFolderPath: string) => Promise<string[]>;
 export declare const downloadProjectFileFromGCS: (projectId: string, fileName: string, destinationPath: string) => Promise<string | null>;
-export {};

package/build/utils/gc.js

@@ -12,54 +12,58 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.downloadProjectFileFromGCS = exports.getFilesFromGCSFolder = exports.uploadFolderToGCS = exports.uploadFileToGCS = void 0;
-const gc_storage_1 = __importDefault(require("./gc-storage"));
-const path_1 = __importDefault(require("path"));
+exports.downloadProjectFileFromGCS = exports.getFilesFromGCSFolder = void 0;
+exports.uploadFolderToGCS = uploadFolderToGCS;
+const promises_1 = __importDefault(require("fs/promises"));
 const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
 const p_limit_1 = __importDefault(require("p-limit"));
-const uploadFileToGCS = ({ filePath, gcsFilePath, }) => {
-    return new Promise((resolve, reject) => {
-        if (!fs_1.default.existsSync(filePath))
-            throw new Error('File not found');
-        const { size } = fs_1.default.statSync(filePath);
-        const blob = gc_storage_1.default.file(gcsFilePath);
-        const blobStream = blob.createWriteStream({
-            resumable: false,
-        });
-        const readStream = fs_1.default.createReadStream(filePath);
-        readStream.pipe(blobStream);
-        blobStream.on('error', (err) => {
-            reject(err);
-        });
-        blobStream.on('finish', () => {
-            resolve({
-                filename: path_1.default.basename(filePath),
-                size: size / 1024 / 1024,
-            });
-        });
+const promises_2 = require("stream/promises");
+const gc_storage_1 = __importDefault(require("./gc-storage"));
+// Tune this based on VM size
+const UPLOAD_CONCURRENCY = 24;
+// ---- 1️⃣ Walk directory once (no recursion during upload)
+function walkDir(dir_1) {
+    return __awaiter(this, arguments, void 0, function* (dir, baseDir = dir) {
+        const entries = yield promises_1.default.readdir(dir, { withFileTypes: true });
+        const results = yield Promise.all(entries.map((entry) => __awaiter(this, void 0, void 0, function* () {
+            const abs = path_1.default.join(dir, entry.name);
+            if (entry.isDirectory()) {
+                return walkDir(abs, baseDir);
+            }
+            const stat = yield promises_1.default.stat(abs);
+            return [
+                {
+                    relativePath: path_1.default.relative(baseDir, abs),
+                    absolutePath: abs,
+                    size: stat.size,
+                },
+            ];
+        })));
+        return results.flat();
     });
-};
-exports.uploadFileToGCS = uploadFileToGCS;
-const uploadFolderToGCS = (folderPath, gcsPath) => __awaiter(void 0, void 0, void 0, function* () {
-    const limit = (0, p_limit_1.default)(8);
-    const files = fs_1.default.readdirSync(folderPath);
-    let promises = [];
-    for (const file of files) {
-        // If file is directory, we need to upload it recursively.
-        if (fs_1.default.lstatSync(path_1.default.join(folderPath, file)).isDirectory()) {
-            yield (0, exports.uploadFolderToGCS)(path_1.default.join(folderPath, file), path_1.default.join(gcsPath, file));
-        }
-        else {
-            const filePath = path_1.default.join(folderPath, file);
-            const gcsFilePath = path_1.default.join(gcsPath, file);
-            promises.push(limit(() => {
-                return (0, exports.uploadFileToGCS)({ filePath, gcsFilePath });
-            }));
-        }
-    }
-    const res = yield Promise.all(promises);
-});
-exports.uploadFolderToGCS = uploadFolderToGCS;
+}
+function uploadFileToGCS(absolutePath, gcsPath, size) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const file = gc_storage_1.default.file(gcsPath);
+        const resumable = size > 5 * 1024 * 1024; // 5MB threshold
+        yield (0, promises_2.pipeline)(fs_1.default.createReadStream(absolutePath), file.createWriteStream({
+            resumable,
+            validation: false,
+            gzip: false,
+            metadata: {
+                cacheControl: 'public, max-age=31536000',
+            },
+        }));
+    });
+}
+function uploadFolderToGCS(folderPath, gcsPath) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const limit = (0, p_limit_1.default)(UPLOAD_CONCURRENCY);
+        const files = yield walkDir(folderPath);
+        yield Promise.all(files.map(({ relativePath, absolutePath, size }) => limit(() => uploadFileToGCS(absolutePath, path_1.default.posix.join(gcsPath, relativePath), size))));
+    });
+}
 const getFilesFromGCSFolder = (gcsFolderPath) => __awaiter(void 0, void 0, void 0, function* () {
     const options = {
         prefix: gcsFolderPath.endsWith('/')

package/build/utils/index.d.ts

@@ -1,2 +1,3 @@
 export * from './grades';
 export * from './gc';
+export { default as gcStorage } from './gc-storage';

package/build/utils/index.js

@@ -13,6 +13,12 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.gcStorage = void 0;
 __exportStar(require("./grades"), exports);
 __exportStar(require("./gc"), exports);
+var gc_storage_1 = require("./gc-storage");
+Object.defineProperty(exports, "gcStorage", { enumerable: true, get: function () { return __importDefault(gc_storage_1).default; } });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@rowengine/common",
-    "version": "1.1.7",
+    "version": "1.1.9",
     "description": "",
     "main": "./build/index.js",
     "private": false,