@e-mc/cloud 0.7.1 → 0.8.1

package/index.js

@@ -210,53 +210,52 @@ class Cloud extends core_1.ClientDb {
                     let pending = (downloadMap || (downloadMap = {}))[location];
                     if (pending) {
                         pending.add(downloadUri);
+                        continue;
                     }
-                    else {
-                        pending = new Set([downloadUri]);
-                        download.admin = data.admin;
-                        const task = instance.downloadObject(data.service, instance.getCredential(data), data.bucket, download, (value) => {
-                            let result;
-                            if (value && !instance.aborted) {
-                                const items = Array.from(pending);
-                                for (let i = 0, length = items.length, size, copy; i < length; ++i) {
-                                    const destUri = items[i];
-                                    try {
-                                        if (typeof value === 'string') {
-                                            if (!copy && i === length - 1) {
-                                                fs.renameSync(value, destUri);
-                                            }
-                                            else {
-                                                fs.copyFileSync(value, destUri);
-                                            }
-                                            size = this.addDownload(destUri);
+                    pending = new Set([downloadUri]);
+                    download.admin = data.admin;
+                    const task = instance.downloadObject(data.service, instance.getCredential(data), data.bucket, download, async (value) => {
+                        let result;
+                        if (value && !instance.aborted) {
+                            const items = Array.from(pending);
+                            for (let i = 0, length = items.length, size, copy; i < length; ++i) {
+                                const destUri = items[i];
+                                try {
+                                    if (typeof value === 'string') {
+                                        if (!copy && i === length - 1) {
+                                            fs.renameSync(value, destUri);
                                         }
                                         else {
-                                            fs.writeFileSync(destUri, value);
-                                            this.addDownload(size = value.length);
+                                            fs.copyFileSync(value, destUri);
                                         }
-                                        this.add(destUri);
-                                        this.formatMessage(64 /* LOG_TYPE.CLOUD */, data.service, ["Download success" /* VAL_CLOUD.DOWNLOAD_FILE */, (0, types_1.formatSize)(size)], destUri, { ...Cloud.LOG_CLOUD_DOWNLOAD });
-                                        result || (result = destUri);
+                                        size = this.addDownload(destUri);
                                     }
-                                    catch (err) {
-                                        if (!copy && core_1.ClientDb.isErrorCode(err, 'EXDEV')) {
-                                            copy = true;
-                                            --i;
-                                        }
-                                        else {
-                                            instance.writeFail(["Unable to write file" /* ERR_MESSAGE.WRITE_FILE */, path.basename(destUri)], err, { type: 32 /* LOG_TYPE.FILE */, fatal: !!active, startTime });
-                                        }
+                                    else {
+                                        fs.writeFileSync(destUri, value);
+                                        this.addDownload(size = value.length);
+                                    }
+                                    this.add(destUri);
+                                    this.formatMessage(64 /* LOG_TYPE.CLOUD */, data.service, ["Download success" /* VAL_CLOUD.DOWNLOAD_FILE */, (0, types_1.formatSize)(size)], destUri, { ...Cloud.LOG_CLOUD_DOWNLOAD });
+                                    result || (result = destUri);
+                                }
+                                catch (err) {
+                                    if (!copy && core_1.ClientDb.isErrorCode(err, 'EXDEV')) {
+                                        copy = true;
+                                        --i;
+                                    }
+                                    else {
+                                        instance.writeFail(["Unable to write file" /* ERR_MESSAGE.WRITE_FILE */, path.basename(destUri)], err, { type: 32 /* LOG_TYPE.FILE */, fatal: !!active, startTime });
                                     }
                                 }
                             }
-                            return Promise.resolve(result);
-                        })
-                            .catch(err => instance.writeFail(["Download failed" /* ERR_CLOUD.DOWNLOAD_FAIL */, path.basename(downloadUri)], err, { type: 64 /* LOG_TYPE.CLOUD */, startTime }));
-                        if (active || waitStatus || this.incremental === 'staging') {
-                            tasks.push(task);
                         }
-                        downloadMap[location] = pending;
+                        return result;
+                    })
+                        .catch(err => instance.writeFail(["Download failed" /* ERR_CLOUD.DOWNLOAD_FAIL */, path.basename(downloadUri)], err, { type: 64 /* LOG_TYPE.CLOUD */, startTime }));
+                    if (active || waitStatus || this.incremental === 'staging') {
+                        tasks.push(task);
                     }
+                    downloadMap[location] = pending;
                 }
             }
         }
@@ -284,20 +283,23 @@ class Cloud extends core_1.ClientDb {
                 state.localStorage.set(file, upload);
             }
             const callback = async (value) => {
-                if (ignoreProcess || !(0, types_1.isString)(value)) {
-                    return;
-                }
-                for (const { instance: document } of host.Document) {
-                    if (document.cloudUpload && await document.cloudUpload(state, file, value, active)) {
-                        return;
+                if ((0, types_1.isString)(value) && !ignoreProcess) {
+                    for (const { instance: document } of host.Document) {
+                        if (document.cloudUpload && await document.cloudUpload(state, file, value, active)) {
+                            return;
+                        }
                     }
                 }
             };
             const task = new Promise(resolve => {
                 const { service, bucket = state.bucketGroup, admin } = storage;
+                let minStreamSize = upload.minStreamSize;
+                if ((0, types_1.isString)(minStreamSize)) {
+                    minStreamSize = (0, types_1.formatSize)(minStreamSize);
+                }
                 const credential = instance.getCredential(storage, true);
                 const uploading = [];
-                getFiles(file, upload).forEach((group, index) => {
+                getFiles(file, upload).forEach(async (group, index) => {
                     let fileGroup;
                     if (index === 0 && group.length > 1) {
                         if (SERVICE_CLIENT.get(service)?.CLOUD_UPLOAD_FROMDISK) {
@@ -309,7 +311,7 @@ class Cloud extends core_1.ClientDb {
                                 const value = group[i];
                                 try {
                                     if (fs.existsSync(value)) {
-                                        fileGroup.push([fs.readFileSync(value), path.extname(value), value]);
+                                        fileGroup.push([typeof minStreamSize === 'number' ? await this.streamFile(value, { minStreamSize, cache: false, signal: instance.signal }) : fs.readFileSync(value), path.extname(value), value]);
                                     }
                                 }
                                 catch (err) {
@@ -336,7 +338,7 @@ class Cloud extends core_1.ClientDb {
                             else if (upload.overwrite) {
                                 filename = path.basename(localUri);
                             }
-                            buffer = file.sourceUTF8 ? Buffer.from(file.sourceUTF8, file.encoding) : host.getBuffer(file);
+                            buffer = file.sourceUTF8 ? Buffer.from(file.sourceUTF8, file.encoding) : typeof minStreamSize === 'number' ? await host.getBuffer(file, minStreamSize).catch(() => host.getBuffer(file)) : host.getBuffer(file);
                         }
                         else {
                             contentType = this.lookupMime(path.basename(localUri)) || file.mimeType;
@@ -435,7 +437,7 @@ class Cloud extends core_1.ClientDb {
         }
         return super.setQueryResult(service, credential, queryString, result, sessionKey);
     }
-    createBucket(service, credential, bucket, publicRead, options) {
+    async createBucket(service, credential, bucket, publicRead, options) {
         if (this.aborted) {
             return Promise.reject((0, types_1.createAbortError)());
         }
@@ -465,7 +467,7 @@ class Cloud extends core_1.ClientDb {
             return Promise.reject(err);
         }
     }
-    setBucketPolicy(service, credential, bucket, options) {
+    async setBucketPolicy(service, credential, bucket, options) {
         if (this.aborted) {
             return Promise.reject((0, types_1.createAbortError)());
         }
@@ -486,7 +488,7 @@ class Cloud extends core_1.ClientDb {
             return Promise.reject(err);
         }
     }
-    setBucketWebsite(service, credential, bucket, options) {
+    async setBucketWebsite(service, credential, bucket, options) {
         if (this.aborted) {
             return Promise.reject((0, types_1.createAbortError)());
         }
@@ -507,7 +509,7 @@ class Cloud extends core_1.ClientDb {
             return Promise.reject(err);
         }
     }
-    deleteObjects(service, credential, bucket, recursive = true) {
+    async deleteObjects(service, credential, bucket, recursive = true) {
         if (this.aborted) {
             return Promise.reject((0, types_1.createAbortError)());
         }
@@ -528,7 +530,7 @@ class Cloud extends core_1.ClientDb {
             return Promise.reject(err);
         }
     }
-    uploadObject(service, credential, bucket, upload, localUri, beforeResolve) {
+    async uploadObject(service, credential, bucket, upload, localUri, beforeResolve) {
         if (this.aborted) {
             return Promise.reject((0, types_1.createAbortError)());
         }
@@ -563,7 +565,7 @@ class Cloud extends core_1.ClientDb {
             }
         });
     }
-    downloadObject(service, credential, bucket, download, beforeResolve) {
+    async downloadObject(service, credential, bucket, download, beforeResolve) {
         if (this.aborted) {
             return Promise.reject((0, types_1.createAbortError)());
         }
@@ -740,7 +742,7 @@ class Cloud extends core_1.ClientDb {
     }
     resolveService(service, folder) {
         let result;
-        if (service[0] !== '@') {
+        if (!service.startsWith('@')) {
             result = this.settings.imports?.[service] || util_1.IMPORTS[service];
         }
         else if (!folder && !service.startsWith("@squared-functions/" /* NAMESPACE.SQUARED_FUNCTIONS */)) {
@@ -752,15 +754,15 @@ class Cloud extends core_1.ClientDb {
         const result = this.settings[service]?.[name];
         return component ? (0, types_1.isObject)(result) ? result[component] : undefined : result;
     }
-    commit() {
+    async commit() {
         if (this.aborted) {
             return Promise.reject((0, types_1.createAbortError)());
         }
-        const items = this.pending.filter(item => !item.document).map(data => {
+        const items = this.pending.filter(item => !item.document).map(async (data) => {
             data.ignoreCache ?? (data.ignoreCache = true);
             return this.getDatabaseRows(data, true);
         });
-        return items.length === 0 ? Promise.resolve(false) : this.allSettled(items, ["Execute unassigned queries" /* VAL_DB.EXEC_QUERYUNASSIGNED */, this.moduleName]).then(result => result.length > 0).catch(() => false);
+        return items.length === 0 ? false : this.allSettled(items, ["Execute unassigned queries" /* VAL_DB.EXEC_QUERYUNASSIGNED */, this.moduleName]).then(result => result.length > 0).catch(() => false);
     }
     getClient(service) {
         let client = SERVICE_CLIENT.get(service);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@e-mc/cloud",
-  "version": "0.7.1",
+  "version": "0.8.1",
   "description": "Cloud constructor for E-mc.",
   "main": "index.js",
   "types": "index.d.ts",
@@ -20,8 +20,9 @@
   "license": "BSD 3-Clause",
   "homepage": "https://github.com/anpham6/e-mc#readme",
   "dependencies": {
-    "@e-mc/core": "0.7.1",
-    "@e-mc/db": "0.7.1",
-    "@e-mc/types": "0.7.1"
+    "@e-mc/core": "0.8.1",
+    "@e-mc/db": "0.8.1",
+    "@e-mc/types": "0.8.1",
+    "mime-types": "^2.1.35"
   }
 }

package/util.d.ts

@@ -1,3 +1,5 @@
+import type { UploadContent } from '../types/lib/cloud';
+
 import type { Readable } from 'stream';
 
 interface AuthValue {
@@ -8,6 +10,7 @@ interface AuthValue {
 declare namespace util {
     const IMPORTS: Record<string, string | undefined>;
     function readableAsBuffer(stream: Readable): Promise<Buffer | null>;
+    function createKeyAndBody(filename: string, items: UploadContent[], mimeType?: string | FunctionType<void>, errorCallback?: FunctionType<void>): [string[], Buffer[], string[]];
     function generateFilename(filename: string): (i: number) => [string, boolean];
     function getBasicAuth(auth: AuthValue): string;
     function getBasicAuth(username: unknown, password?: unknown): string;

package/util.js

@@ -1,7 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.hasBasicAuth = exports.getBasicAuth = exports.formatError = exports.generateFilename = exports.readableAsBuffer = exports.IMPORTS = void 0;
+exports.hasBasicAuth = exports.getBasicAuth = exports.formatError = exports.generateFilename = exports.createKeyAndBody = exports.readableAsBuffer = exports.IMPORTS = void 0;
 const path = require("path");
+const fs = require("fs");
+const mime = require("mime-types");
 const types_1 = require("../types");
 const util_1 = require("../db/util");
 Object.defineProperty(exports, "getBasicAuth", { enumerable: true, get: function () { return util_1.getBasicAuth; } });
@@ -18,7 +20,7 @@ exports.IMPORTS = {
     "minio": "@pi-r/minio",
     "oci": "@pi-r/oci"
 };
-function readableAsBuffer(stream) {
+async function readableAsBuffer(stream) {
     return new Promise((resolve, reject) => {
         let result = null;
         stream
@@ -31,6 +33,45 @@ function readableAsBuffer(stream) {
     });
 }
 exports.readableAsBuffer = readableAsBuffer;
+function createKeyAndBody(filename, items, mimeType, errorCallback) {
+    if (typeof mimeType === 'function') {
+        errorCallback = mimeType;
+        mimeType = undefined;
+    }
+    const key = [];
+    const body = [];
+    const type = [];
+    for (let [content, ext, localFile] of items) {
+        try {
+            switch (typeof content) {
+                case 'string':
+                    if (fs.existsSync(content)) {
+                        content = fs.readFileSync(content);
+                    }
+                    else if (path.isAbsolute(content)) {
+                        continue;
+                    }
+                    else {
+                        content = Buffer.from(content);
+                    }
+                    break;
+                case 'object':
+                    break;
+                default:
+                    continue;
+            }
+            const output = filename + ext;
+            key.push(ext === '.map' && localFile ? path.basename(localFile) : output);
+            body.push(content);
+            type.push(ext !== '.map' && mime.lookup(output) || mimeType || 'application/octet-stream');
+        }
+        catch (err) {
+            errorCallback?.(err);
+        }
+    }
+    return [key, body, type];
+}
+exports.createKeyAndBody = createKeyAndBody;
 function generateFilename(filename) {
     let basename, suffix;
     return (i) => {
@@ -45,7 +86,7 @@ function generateFilename(filename) {
                 suffix = filename.substring(j);
             }
         }
-        return basename && suffix ? [basename + `_${i}` + suffix, true] : [(0, types_1.generateUUID)() + path.extname(filename), false];
+        return basename && suffix ? [basename + `_${i}` + suffix, true] : [(0, types_1.incrementUUID)() + path.extname(filename), false];
     };
 }
 exports.generateFilename = generateFilename;