@pi-r/gcp 0.6.5 → 0.7.0

package/README.md

@@ -1,10 +1,7 @@
-# @pi-r/gcp
+### @pi-r/gcp
 
-## Documentation
+https://e-mc.readthedocs.io/en/latest/cloud/gcp.html
 
-- [E-mc](https://e-mc.readthedocs.io/en/latest/cloud/gcp.html)
-- [squared](https://squared.readthedocs.io)
-
-## LICENSE
+### LICENSE
 
 MIT

package/client/index.js

@@ -1,19 +1,16 @@
 "use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getProjectId = exports.createBucketRequest = exports.setPredefinedAcl = exports.setPublicRead = exports.executeBatchQuery = exports.executeQuery = exports.deleteObjectsV2 = exports.deleteObjects = exports.setBucketWebsite = exports.setBucketPolicy = exports.createBucketV2 = exports.createBucket = exports.createDatabaseClient = exports.createStorageClient = exports.validateDatabase = exports.validateStorage = exports.isFirebaseApp = void 0;
+exports.CLOUD_DOWNLOAD_CHUNK = exports.CLOUD_UPLOAD_CHUNK = exports.CLOUD_UPLOAD_STREAM = exports.getProjectId = exports.createBucketRequest = exports.setPredefinedAcl = exports.setPublicRead = exports.executeBatchQuery = exports.executeQuery = exports.deleteObjectsV2 = exports.deleteObjects = exports.setBucketWebsite = exports.setBucketTagging = exports.setBucketPolicy = exports.createBucketV2 = exports.createBucket = exports.createDatabaseClient = exports.createStorageClient = exports.validateDatabase = exports.validateStorage = exports.isFirebaseApp = void 0;
 const path = require("path");
 const storage = require("@google-cloud/storage");
 const firestore_1 = require("@google-cloud/firestore");
 const util_1 = require("@e-mc/cloud/util");
 const types_1 = require("@e-mc/types");
-const Module = require("@e-mc/module");
 const Cloud = require("@e-mc/cloud");
 const FIREBASE_DB = {};
 const FIREBASE_STORAGE = {};
 const ADMIN_DB = {};
 const ADMIN_STORAGE = {};
 function getApp(credential, isDB) {
-    var _a, _b;
     const { projectId, databaseURL, storageBucket, apiKey, authDomain } = credential;
     if (apiKey && authDomain) {
         const { getApps, initializeApp } = require('@firebase/app');
@@ -27,7 +24,7 @@ function getApp(credential, isDB) {
         }
         if (!app) {
             const firebaseKey = (projectId || '') + '_' + apiKey + '_' + authDomain + (isDB ? '_' + databaseURL : '');
-            app = isDB ? FIREBASE_DB[firebaseKey] || (FIREBASE_DB[firebaseKey] = initializeApp({ ...credential, databaseURL }, firebaseKey)) : FIREBASE_STORAGE[firebaseKey] || (FIREBASE_STORAGE[firebaseKey] = initializeApp({ ...credential, storageBucket }, firebaseKey));
+            app = isDB ? FIREBASE_DB[firebaseKey] ||= initializeApp({ ...credential, databaseURL }, firebaseKey) : FIREBASE_STORAGE[firebaseKey] ||= initializeApp({ ...credential, storageBucket }, firebaseKey);
         }
         if (isDB) {
             const { getDatabase, goOnline } = require('@firebase/database');
@@ -48,7 +45,7 @@ function getApp(credential, isDB) {
             }
         }
         if (!app) {
-            app = isDB ? ADMIN_DB[_a = databaseURL || '[DEFAULT]'] || (ADMIN_DB[_a] = initializeApp({ credential: applicationDefault(), databaseURL }, databaseURL)) : ADMIN_STORAGE[_b = storageBucket || '[DEFAULT]'] || (ADMIN_STORAGE[_b] = initializeApp({ credential: applicationDefault(), storageBucket }, storageBucket));
+            app = isDB ? ADMIN_DB[databaseURL || "[DEFAULT]"] ||= initializeApp({ credential: applicationDefault(), databaseURL }, databaseURL) : ADMIN_STORAGE[storageBucket || "[DEFAULT]"] ||= initializeApp({ credential: applicationDefault(), storageBucket }, storageBucket);
         }
         credential.admin = true;
         if (isDB) {
@@ -60,7 +57,7 @@ function getApp(credential, isDB) {
     }
 }
 async function hasBucket(client) {
-    const [exists] = await client.exists().catch(err => isErrorConflict(err) ? [true] : [false]);
+    const [exists] = await client.exists().catch((err) => isErrorConflict(err) ? [true] : [false]);
     return exists;
 }
 async function updatePredefinedAcl(acl, values, metadata = {}) {
@@ -84,7 +81,7 @@ async function updatePredefinedAcl(acl, values, metadata = {}) {
             }
             if ((0, types_1.isString)(value.role)) {
                 if (projectNumber !== undefined) {
-                    value.userProject || (value.userProject = projectNumber.toString());
+                    value.userProject ||= projectNumber.toString();
                 }
                 return target.add(value);
             }
@@ -100,24 +97,24 @@ async function updatePredefinedAcl(acl, values, metadata = {}) {
         }
         return errors;
     })
-        .catch(err => [err]);
+        .catch((err) => [err]);
 }
 async function setUniformBucketLevel(bucket, enabled) {
     return bucket.setMetadata({ iamConfiguration: { uniformBucketLevelAccess: { enabled } } });
 }
 const getFirebasePackageName = (credential) => credential.apiKey && credential.authDomain ? 'firebase' : 'firebase-admin';
-const hasKeyFile = (credential) => (0, types_1.isString)(credential.keyFilename || credential.keyFile || process.env.GOOGLE_APPLICATION_CREDENTIALS) || (0, types_1.isObject)(credential.credentials);
 const isErrorConflict = (err) => (err instanceof Error) && err.code === 409;
+const hasGoogleAuth = (credential) => (0, types_1.isString)(credential.keyFilename || credential.keyFile || process.env.GOOGLE_APPLICATION_CREDENTIALS) || (0, types_1.isObject)(credential.credentials) || !!credential.authClient;
 function isFirebaseApp(credential, store, admin) {
-    return !!(credential.apiKey && credential.authDomain && (store || credential.databaseURL) || credential.product === 'firebase' && process.env.GOOGLE_APPLICATION_CREDENTIALS) && (!admin || !credential.admin);
+    return !!(credential.apiKey && credential.authDomain && (store || credential.databaseURL) || credential.product === "firebase" && process.env.GOOGLE_APPLICATION_CREDENTIALS) && (!admin || !credential.admin);
 }
 exports.isFirebaseApp = isFirebaseApp;
 function validateStorage(credential) {
-    return hasKeyFile(credential) || !!credential.authClient || isFirebaseApp(credential, true);
+    return hasGoogleAuth(credential) || isFirebaseApp(credential, true);
 }
 exports.validateStorage = validateStorage;
 function validateDatabase(credential, data) {
-    return (hasKeyFile(credential) || !!credential.authClient) && !!(data.product || data.database || data.table || data.query || data.id || data.keys || data.kind || data.options?.query) || isFirebaseApp(credential);
+    return hasGoogleAuth(credential) && !!(data.product || data.database || data.table || data.query || data.id || data.keys || data.kind || data.options?.query) || isFirebaseApp(credential);
 }
 exports.validateDatabase = validateDatabase;
 function createStorageClient(credential, bucket) {
@@ -142,29 +139,26 @@ exports.createStorageClient = createStorageClient;
 function createDatabaseClient(credential, data) {
     let packageName = '';
     try {
-        let product, name, database, table, query;
-        if (data) {
-            ({ product, name, table, query, database } = data);
-        }
-        if (product === 'firebase' || isFirebaseApp(credential)) {
+        if (data?.product === "firebase" || isFirebaseApp(credential)) {
             packageName = getFirebasePackageName(credential);
             return getApp(credential, true);
         }
         getProjectId(credential, true);
         if (data) {
-            if (product === 'spanner' || !product && name && database && (query || table)) {
+            const { product, name, table, query, database } = data;
+            if (product === "spanner" || !product && name && database && (query || table)) {
                 const { Spanner } = require(packageName = '@google-cloud/spanner');
                 return new Spanner(credential);
             }
-            if (product === 'bigquery' || !product && (!name && table && database || (0, types_1.isString)(query))) {
+            if (product === "bigquery" || !product && (!name && table && database || (0, types_1.isString)(query))) {
                 const { BigQuery } = require(packageName = '@google-cloud/bigquery');
                 return new BigQuery(credential);
             }
-            if (product === 'bigtable' || !product && name && table && !database) {
+            if (product === "bigtable" || !product && name && table && !database) {
                 const { Bigtable } = require(packageName = '@google-cloud/bigtable');
                 return new Bigtable(credential);
             }
-            if (product === 'datastore' || !product && (data.keys || data.kind || query && !table && !data.id)) {
+            if (product === "datastore" || !product && (data.keys || data.kind || query && !table && !data.id)) {
                 const { Datastore } = require(packageName = '@google-cloud/datastore');
                 return new Datastore(credential);
             }
@@ -223,32 +217,32 @@ async function setBucketPolicy(credential, bucket, options, service = 'gcp') {
     if (!Array.isArray(options.acl)) {
         options.acl = undefined;
         switch (acl) {
-            case 'private':
-            case 'projectPrivate':
-                await client.makePrivate(options).catch(err => !isErrorConflict(err) && errors.push(err));
-                await setPredefinedAcl.call(this, client.acl, acl, client, '', options.bucket?.metadata).catch(err => errors.push(err));
+            case "private":
+            case "projectPrivate":
+                await client.makePrivate(options).catch((err) => !isErrorConflict(err) && errors.push(err));
+                await setPredefinedAcl.call(this, client.acl, acl, client, '', options.bucket?.metadata).catch((err) => errors.push(err));
                 break;
-            case 'authenticatedRead':
-                await client.makePublic(options).catch(err => !isErrorConflict(err) && errors.push(err));
+            case "authenticatedRead":
+                await client.makePublic(options).catch((err) => !isErrorConflict(err) && errors.push(err));
                 options = { ...options };
                 options.acl = [{ entity: 'allUsers' }, { entity: 'allAuthenticatedUsers', role: 'READER' }];
                 break;
-            case 'publicReadWrite':
-                await client.makePublic(options).catch(err => !isErrorConflict(err) && errors.push(err));
+            case "publicReadWrite":
+                await client.makePublic(options).catch((err) => !isErrorConflict(err) && errors.push(err));
                 options = { ...options };
                 options.acl = [{ entity: 'allUsers', role: 'WRITER' }];
                 break;
-            case 'publicRead':
-                await client.makePublic(options).catch(err => !isErrorConflict(err) && errors.push(err));
+            case "publicRead":
+                await client.makePublic(options).catch((err) => !isErrorConflict(err) && errors.push(err));
                 break;
             case 'bucketAccessUniform':
-                await setUniformBucketLevel(client, true).catch(err => errors.push(err));
+                await setUniformBucketLevel(client, true).catch((err) => errors.push(err));
                 break;
             case 'bucketAccessACL':
-                await setUniformBucketLevel(client, false).catch(err => errors.push(err));
+                await setUniformBucketLevel(client, false).catch((err) => errors.push(err));
                 break;
             default:
-                errors.push((0, types_1.errorValue)('Invalid ACL', Module.asString(acl) || 'empty'));
+                errors.push((0, types_1.errorValue)('Invalid ACL', Cloud.asString(acl) || 'empty'));
                 break;
         }
     }
@@ -263,6 +257,33 @@ async function setBucketPolicy(credential, bucket, options, service = 'gcp') {
     return true;
 }
 exports.setBucketPolicy = setBucketPolicy;
+async function setBucketTagging(credential, bucket, labels, service = 'gcp') {
+    if (isFirebaseApp(credential, true, true) || !(0, types_1.isPlainObject)(labels)) {
+        return false;
+    }
+    const client = createStorageClient.call(this, credential, bucket).bucket(bucket);
+    const deleting = Object.keys(labels).length === 0;
+    if (deleting) {
+        const [metadata] = await client.getMetadata();
+        if (!(0, types_1.isPlainObject)(metadata.labels)) {
+            return true;
+        }
+        labels = {};
+        for (const key in metadata.labels) {
+            labels[key] = null;
+        }
+    }
+    return client.setMetadata({ labels })
+        .then(([metadata]) => {
+        this.formatMessage(64, service, [deleting ? "Tags deleted" : "Tags created", bucket], ((0, types_1.isPlainObject)(metadata.labels) ? Object.keys(metadata.labels).length : 0) + ' labels (current)', { ...Cloud.LOG_CLOUD_COMMAND });
+        return true;
+    })
+        .catch((err) => {
+        this.formatFail(64, service, ["Unable to update bucket tagging", bucket], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: false });
+        return false;
+    });
+}
+exports.setBucketTagging = setBucketTagging;
 async function setBucketWebsite(credential, bucket, options, service = 'gcp') {
     if (isFirebaseApp(credential, true, true)) {
         return false;
@@ -277,11 +298,11 @@ async function setBucketWebsite(credential, bucket, options, service = 'gcp') {
     }
     return createStorageClient.call(this, credential, bucket).bucket(bucket).setMetadata({ website })
         .then(() => {
-        this.formatMessage(64, service, ["Bucket configured", bucket], website, { ...Cloud.LOG_CLOUD_COMMAND });
+        this.formatMessage(64, service, ["Bucket website configured", bucket], website, { ...Cloud.LOG_CLOUD_COMMAND });
         return true;
     })
-        .catch(err => {
-        this.formatFail(64, service, ["Unable to configure bucket", bucket], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: false });
+        .catch((err) => {
+        this.formatFail(64, service, ["Unable to set bucket website", bucket], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: false });
         return false;
     });
 }
@@ -305,7 +326,7 @@ async function deleteObjectsV2(credential, bucket, recursive = true, service = '
                 }
                 if (recursive && prefixes.length) {
                     for (const { name } of prefixes) {
-                        await recurse(Module.joinPath(prefix, name));
+                        await recurse(Cloud.joinPath(prefix, name));
                     }
                 }
             }
@@ -328,7 +349,7 @@ async function deleteObjectsV2(credential, bucket, recursive = true, service = '
                 const result = await client.getFiles();
                 for (const { name } of result[0]) {
                     if (!name.includes('/')) {
-                        tasks.push(client.file(name).delete({ ignoreNotFound: true }).catch(err => this.formatFail(64, service, ["Delete failed", name], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: false })));
+                        tasks.push(client.file(name).delete({ ignoreNotFound: true }).catch((err) => this.formatFail(64, service, ["Delete failed", name], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: false })));
                     }
                 }
             }
@@ -355,7 +376,7 @@ async function deleteObjectsV2(credential, bucket, recursive = true, service = '
                 this.formatMessage(64, service, ["Bucket emptied" + ` (${recursive ? 'recursive' : files})`, bucket], recursive ? files : '', { ...Cloud.LOG_CLOUD_COMMAND });
             }
         })
-            .catch(err => {
+            .catch((err) => {
             errors.push(err);
         });
     }
@@ -369,7 +390,7 @@ exports.executeQuery = executeQuery;
 async function executeBatchQuery(credential, batch, sessionKey) {
     const length = batch.length;
     const result = new Array(length);
-    const clients = [];
+    const clients = {};
     const caching = length > 0 && this.hasCache(batch[0].service, sessionKey);
     const cacheValue = { value: this.valueOfKey(credential, 'cache'), sessionKey };
     const authItems = [];
@@ -384,7 +405,14 @@ async function executeBatchQuery(credential, batch, sessionKey) {
     };
     const closeClient = () => {
         const fs = clients[3];
-        fs?.terminate();
+        if (fs) {
+            fs.terminate();
+            return;
+        }
+        const sp = clients[5];
+        if (sp) {
+            sp.close();
+        }
     };
     const getFamily = (data, row, family) => {
         const familyData = data[family];
@@ -417,14 +445,14 @@ async function executeBatchQuery(credential, batch, sessionKey) {
         };
         cacheValue.exclusiveOf = Array.isArray(ignoreCache) ? ignoreCache : undefined;
         let rows, queryString = '';
-        if (product === 'firebase' || isFirebaseApp(credential)) {
+        if (product === "firebase" || isFirebaseApp(credential)) {
             if (!(0, types_1.isString)(query)) {
                 closeClient();
                 throw (0, util_1.formatError)(item, "Missing database query");
             }
             const orderBy = item.orderBy;
             if (useCache) {
-                queryString = query + Module.asString(orderBy, true);
+                queryString = query + Cloud.asString(orderBy, true);
                 if (!update && (rows = getCache(queryString))) {
                     result[i] = rows;
                     continue;
@@ -473,7 +501,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                 rows = [group && (0, types_1.isObject)(val) && (0, types_1.isObject)(val[group[0]]) ? val[group[0]] : val];
             }
         }
-        else if (product === 'spanner' || !product && name && database && (query || table)) {
+        else if (product === "spanner" || !product && name && database && (query || table)) {
             const columns = (0, types_1.isPlainObject)(query) && query.columns || item.columns;
             if (!(query || table && columns)) {
                 closeClient();
@@ -485,7 +513,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
             }
             const { updateType, options = {} } = item;
             if (useCache) {
-                queryString = name + '_' + database + '_' + (table ? table + (columns ? Module.asString(columns, true) : '') : '') + Module.asString(query, true) + (limit > 0 ? limit : '');
+                queryString = name + '_' + database + '_' + (table ? table + (columns ? Cloud.asString(columns, true) : '') : '') + Cloud.asString(query, true) + (limit > 0 ? limit : '');
                 if (!update && (rows = getCache(queryString))) {
                     result[i] = rows;
                     continue;
@@ -516,8 +544,12 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                             }
                             else {
                                 transaction.runUpdate(update)
-                                    .then(async () => transaction.commit().then(() => resolve()).catch(error => reject(error)))
-                                    .catch(error => reject(error));
+                                    .then(async () => {
+                                    transaction.commit()
+                                        .then(() => resolve())
+                                        .catch(reject);
+                                })
+                                    .catch(reject);
                             }
                         });
                     });
@@ -525,12 +557,12 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                 const request = (0, types_1.isPlainObject)(query) ? query : { sql: query };
                 request.json = true;
                 if (limit > 0) {
-                    (request.gaxOptions || (request.gaxOptions = {})).maxResults = limit;
+                    (request.gaxOptions ||= {}).maxResults = limit;
                 }
                 [rows] = await db.run(request);
             }
         }
-        else if (product === 'bigtable' || !product && name && table && !database) {
+        else if (product === "bigtable" || !product && name && table && !database) {
             if (!name || !table) {
                 closeClient();
                 throw (0, util_1.formatError)(item, !name ? "Missing database name" : "Missing database table");
@@ -540,10 +572,10 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                 options.filter = query;
             }
             if (!id && limit > 0) {
-                (options.gaxOptions || (options.gaxOptions = {})).maxResults = limit;
+                (options.gaxOptions ||= {}).maxResults = limit;
             }
             if (useCache) {
-                queryString = name + '_' + table + '_' + ((0, types_1.isString)(id) ? id + ((0, types_1.isArray)(columns) ? '_' + Module.asString(columns, true) : '') : '') + Module.asString(options, true);
+                queryString = name + '_' + table + '_' + ((0, types_1.isString)(id) ? id + ((0, types_1.isArray)(columns) ? '_' + Cloud.asString(columns, true) : '') : '') + Cloud.asString(options, true);
                 if (!(update && id) && (rows = getCache(queryString))) {
                     result[i] = rows;
                     continue;
@@ -574,7 +606,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                 });
             }
         }
-        else if (product === 'bigquery' || !product && (!name && database && table || (0, types_1.isString)(query) || (0, types_1.isString)(item.options?.query))) {
+        else if (product === "bigquery" || !product && (!name && database && table || (0, types_1.isString)(query) || (0, types_1.isString)(item.options?.query))) {
             const { params, options = {} } = item;
             if ((0, types_1.isString)(query)) {
                 options.query = query;
@@ -586,7 +618,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                 options.maxResults = limit;
             }
             if (useCache) {
-                queryString = query + (database ? '_' + database + (table ? '_' + table : '') : '') + Module.asString(options, true);
+                queryString = query + (database ? '_' + database + (table ? '_' + table : '') : '') + Cloud.asString(options, true);
                 if (!update && (rows = getCache(queryString))) {
                     result[i] = rows;
                     continue;
@@ -613,7 +645,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
             }
             [rows] = await job.getQueryResults();
         }
-        else if (product === 'datastore' || !product && (item.keys || item.kind || query && !table && !id)) {
+        else if (product === "datastore" || !product && (item.keys || item.kind || query && !table && !id)) {
             const { keys, kind } = item;
             if (!keys && !kind && !Array.isArray(query)) {
                 closeClient();
@@ -669,19 +701,19 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                         db.runQuery(task.select('__key__'), item.options).then(entities => {
                             if ((0, types_1.isArray)(entities)) {
                                 const items = entities.map((entity) => db.key(entity[db.KEY]));
-                                db.save({ key: items, data: update }).then(() => resolve()).catch(err => reject(err));
+                                db.save({ key: items, data: update }).then(() => resolve()).catch((err) => reject(err));
                             }
                             else {
                                 resolve();
                             }
                         })
-                            .catch(err => reject(err));
+                            .catch((err) => reject(err));
                     });
                 }
                 [rows] = await db.runQuery(task, item.options);
             }
         }
-        else if (product === 'firestore' || !product && table) {
+        else if (product === "firestore" || !product && table) {
             if (!table) {
                 closeClient();
                 throw (0, util_1.formatError)(item, "Missing database table");
@@ -705,7 +737,34 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                 else {
                     const doc = db.collection(table).doc(id);
                     if ((0, types_1.isPlainObject)(update)) {
-                        await doc.update(update, { exists: true });
+                        const pattern = /^__(delete|serverTimestamp|increment)(?:<(.+)>)?__$/;
+                        for (const attr in update) {
+                            const match = pattern.exec(attr);
+                            if (match) {
+                                const method = match[1];
+                                switch (method) {
+                                    case 'delete':
+                                    case 'serverTimestamp':
+                                        update[attr] = firestore_1.FieldValue[method]();
+                                        break;
+                                    case 'increment':
+                                        update[attr] = firestore_1.FieldValue.increment(match[2] ? +match[2] : 1);
+                                        break;
+                                }
+                            }
+                        }
+                        switch (item.updateType) {
+                            case 1:
+                                await doc.create(update);
+                                break;
+                            case 2:
+                            case 3:
+                                await doc.set(update, { merge: item.updateType === 3 });
+                                break;
+                            default:
+                                await doc.update(update);
+                                break;
+                        }
                     }
                     const data = (await doc.get()).data();
                     if (data) {
@@ -715,7 +774,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
             }
             else if (Array.isArray(query)) {
                 const orderBy = item.orderBy;
-                if (useCache && (rows = getCache(queryString = table + '_' + Module.asString(query, true) + Module.asString(orderBy, true) + (limit > 0 ? limit : '')))) {
+                if (useCache && (rows = getCache(queryString = table + '_' + Cloud.asString(query, true) + Cloud.asString(orderBy, true) + (limit > 0 ? limit : '')))) {
                     result[i] = rows;
                     continue;
                 }
@@ -727,7 +786,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                         switch (method) {
                             case 'whereAnd':
                             case 'whereOr':
-                                if (params.every(p => Array.isArray(p))) {
+                                if (params.every(p => (0, types_1.isArray)(p))) {
                                     db = db.where(firestore_1.Filter[method === 'whereAnd' ? 'and' : 'or'](...params.map((p) => firestore_1.Filter.where(p[0], p[1], p[2]))));
                                 }
                                 else {
@@ -784,7 +843,7 @@ function setPublicRead(acl, pathname, requested, service = 'gcp') {
             this.formatMessage(64, service, "Grant public-read", pathname, { ...Cloud.LOG_CLOUD_COMMAND });
         }
     })
-        .catch(err => {
+        .catch((err) => {
         if (requested) {
             this.formatMessage(64, service, ["Unable to grant public-read", pathname], err, { ...Cloud.LOG_CLOUD_WARN });
         }
@@ -816,7 +875,7 @@ async function setPredefinedAcl(target, value, bucket, pathname = '', metadata =
                 items = [{ entity: 'allUsers' }, { entity: 'allAuthenticatedUsers' }];
             }
             switch (value) {
-                case 'private': {
+                case "private": {
                     const owners = new Set(acl.filter(item => item.role === 'OWNER' && item.entity).map(item => item.entity));
                     if (owner) {
                         owners.add(owner);
@@ -831,7 +890,7 @@ async function setPredefinedAcl(target, value, bucket, pathname = '', metadata =
                     }
                     break;
                 }
-                case 'projectPrivate':
+                case "projectPrivate":
                     for (const user of acl) {
                         const entity = user.entity;
                         if (entity) {
@@ -840,7 +899,7 @@ async function setPredefinedAcl(target, value, bucket, pathname = '', metadata =
                         }
                     }
                     break;
-                case 'bucketOwnerRead':
+                case "bucketOwnerRead":
                     if (bucket && owner) {
                         for (const user of acl) {
                             const entity = user.entity;
@@ -849,7 +908,7 @@ async function setPredefinedAcl(target, value, bucket, pathname = '', metadata =
                             }
                         }
                     }
-                case 'bucketOwnerFullControl':
+                case "bucketOwnerFullControl":
                     if (bucket && owner) {
                         items.push({ entity: owner, role: 'OWNER' });
                     }
@@ -916,13 +975,16 @@ function getProjectId(credential, cache) {
                 const id = require(pathname).project_id;
                 if ((0, types_1.isString)(id)) {
                     projectId = id;
-                    credential.projectId = projectId;
                 }
             }
             catch {
             }
+            credential.projectId = projectId || (projectId = process.env.GCLOUD_PROJECT);
         }
     }
     return projectId;
 }
 exports.getProjectId = getProjectId;
+exports.CLOUD_UPLOAD_STREAM = true;
+exports.CLOUD_UPLOAD_CHUNK = true;
+exports.CLOUD_DOWNLOAD_CHUNK = true;

package/download/index.js

@@ -3,20 +3,18 @@ const path = require("path");
 const fs = require("fs");
 const https = require("https");
 const types_1 = require("@e-mc/types");
-const Module = require("@e-mc/module");
 const Cloud = require("@e-mc/cloud");
 const client_1 = require("@pi-r/gcp");
-function download(credential, service = 'gcp') {
+module.exports = function download(credential, service = 'gcp') {
     const storage = client_1.createStorageClient.call(this, credential);
     return (data, callback) => {
         const { bucket: Bucket, download: target } = data;
-        const Key = target.filename;
-        const firebase = (0, client_1.isFirebaseApp)(credential, true, true);
+        const Key = target.keyname || target.filename;
         let tempDir;
-        const downloadResponse = (err, url) => {
+        const downloadResponse = (err, url = null) => {
             callback(err, url);
             if (tempDir) {
-                queueMicrotask(() => Module.removeDir(tempDir));
+                queueMicrotask(() => Cloud.removeDir(tempDir));
             }
         };
         if (!Bucket || !Key) {
@@ -24,13 +22,13 @@ function download(credential, service = 'gcp') {
             return;
         }
         if (!(tempDir = this.getTempDir({ uuidDir: true }))) {
-            downloadResponse(new Error("Unable to create temp directory"));
+            downloadResponse((0, types_1.errorValue)("Unable to create temp directory", service));
             return;
         }
-        const location = Module.joinPath(Bucket, Key);
-        const destination = path.join(tempDir, path.basename(Key));
+        const location = Cloud.joinPath(Bucket, Key);
+        const destination = path.join(tempDir, path.basename(target.filename || Key));
         const deleteMessage = () => this.formatMessage(64, service, "Delete success", location, { ...Cloud.LOG_CLOUD_DELETE });
-        if (firebase) {
+        if ((0, client_1.isFirebaseApp)(credential, true, true)) {
             const { ref, getDownloadURL, deleteObject } = require('@firebase/storage');
             const objectRef = ref(storage, Key);
             getDownloadURL(objectRef).then(url => {
@@ -44,7 +42,7 @@ function download(credential, service = 'gcp') {
                             if (target.deleteObject) {
                                 deleteObject(objectRef)
                                     .then(() => deleteMessage())
-                                    .catch(err => this.formatFail(64, service, ["Delete failed", location], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: !!target.active }));
+                                    .catch((err) => this.formatFail(64, service, ["Delete failed", location], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: !!target.active }));
                             }
                         })
                             .on('error', err => downloadResponse(err)));
@@ -54,28 +52,36 @@ function download(credential, service = 'gcp') {
                     }
                 });
             });
+            return;
         }
-        else {
-            const bucket = storage.bucket(Bucket);
-            const file = bucket.file(Key, { generation: target.versionId });
-            file.download({ destination })
-                .then(() => {
-                downloadResponse(null, destination);
-                const deleteObject = target.deleteObject;
-                if (deleteObject) {
-                    file.delete((0, types_1.isPlainObject)(deleteObject) ? deleteObject : { ignoreNotFound: true }, err => {
-                        if (!err) {
-                            deleteMessage();
-                        }
-                        else {
-                            this.formatFail(64, service, ["Delete failed", location], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: !!target.active });
-                        }
-                    });
-                }
-            })
-                .catch(err => downloadResponse(err));
+        const { flags = 0 } = target;
+        const bucket = storage.bucket(Bucket);
+        const file = bucket.file(Key, { generation: target.versionId });
+        const chunkSize = flags & 4 ? (0, types_1.alignSize)(target.chunkSize, 1024) : 0;
+        let manager;
+        if (chunkSize > 0) {
+            const { TransferManager } = require('@google-cloud/storage');
+            let concurrencyLimit = target.chunkLimit;
+            if (!concurrencyLimit && target.options) {
+                ({ concurrencyLimit } = target.options);
+            }
+            manager = new TransferManager(bucket).downloadFileInChunks(file, { destination, chunkSizeBytes: chunkSize, concurrencyLimit });
         }
+        (manager || file.download({ destination }))
+            .then(() => {
+            downloadResponse(null, destination);
+            const deleteObject = target.deleteObject;
+            if (deleteObject) {
+                file.delete((0, types_1.isObject)(deleteObject) ? deleteObject : { ignoreNotFound: true }, err => {
+                    if (!err) {
+                        deleteMessage();
+                    }
+                    else {
+                        this.formatFail(64, service, ["Delete failed", location], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: !!target.active });
+                    }
+                });
+            }
+        })
+            .catch((err) => downloadResponse(err));
     };
-}
-
-module.exports = download;
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-r/gcp",
-  "version": "0.6.5",
+  "version": "0.7.0",
   "description": "Google (GCP) cloud functions for E-mc.",
   "main": "client/index.js",
   "publishConfig": {
@@ -20,10 +20,10 @@
   "license": "MIT",
   "homepage": "https://github.com/anpham6/pi-r#readme",
   "dependencies": {
-    "@e-mc/cloud": "^0.8.6",
-    "@e-mc/module": "^0.8.6",
-    "@e-mc/types": "^0.8.6",
-    "@google-cloud/firestore": "^7.3.1",
-    "@google-cloud/storage": "^7.7.0"
+    "@e-mc/cloud": "^0.9.0",
+    "@e-mc/module": "^0.9.0",
+    "@e-mc/types": "^0.9.0",
+    "@google-cloud/firestore": "^7.6.0",
+    "@google-cloud/storage": "^7.10.1"
   }
 }

package/types/index.d.ts

@@ -0,0 +1,36 @@
+import type { CloudDatabase, CloudStorage } from '@e-mc/types/lib/cloud';
+
+import type { StorageOptions } from '@google-cloud/storage';
+import type { PathType } from '@google-cloud/datastore';
+import type { entity } from '@google-cloud/datastore/build/src/entity';
+import type { FirebaseOptions as IFirebaseOptions } from '@firebase/app';
+
+// @ts-ignore
+import type { GoogleAuthOptions } from 'google-auth-library';
+
+export type GCPStorage = CloudStorage<GCPStorageCredential, "gcp | gcloud">;
+
+export interface GCPStorageCredential extends StorageOptions, IFirebaseOptions {}
+
+export interface GCPDatabaseQuery<T = PlainObject> extends Omit<CloudDatabase<unknown, T, unknown, unknown[] | StandardMap, GCPDatabaseCredential>, "id"> {
+    source: "cloud";
+    service: "gcp" | "gcloud";
+    product?: "firestore" | "firebase" | "bigquery" | "bigtable" | "datastore" | "spanner";
+    database?: string;
+    id?: ArrayOf<string>;
+    updateType?: 0 | 1 | 2 | 3;
+    columns?: string[];
+    keys?: ArrayOf<DatastoreKey>;
+    kind?: ArrayOf<string>;
+    orderBy?: unknown[][];
+}
+
+export type GCPDatabaseCredential = GoogleAuthOptions;
+
+export interface FirebaseOptions extends IFirebaseOptions {
+    product?: "firebase";
+    admin?: boolean;
+}
+
+export type DatastoreKey = string | PathType[] | entity.KeyOptions;
+export type BucketAcl = "authenticatedRead" | "private" | "projectPrivate" | "publicRead" | "publicReadWrite" | "bucketAccessUniform" | "bucketAccessACL";

package/upload/index.js

@@ -1,21 +1,21 @@
 "use strict";
 const path = require("path");
+const fs = require("fs");
+const stream = require("stream");
 const util_1 = require("@e-mc/cloud/util");
 const types_1 = require("@e-mc/types");
-const Module = require("@e-mc/module");
 const Cloud = require("@e-mc/cloud");
 const client_1 = require("@pi-r/gcp");
 const BUCKET_SESSION = new Set();
 const BUCKET_RESPONSE = {};
-const getBucketKey = (credential, bucket, acl = '') => Module.asString(credential, true) + bucket + '_' + acl;
-function upload(credential, service = 'gcp') {
+const getBucketKey = (credential, bucket, acl = '') => Cloud.asString(credential, true) + bucket + '_' + acl;
+module.exports = function upload(credential, service = 'gcp') {
     const storage = client_1.createStorageClient.call(this, credential);
     return async (data, callback) => {
-        var _a;
         const firebase = (0, client_1.isFirebaseApp)(credential, true, true);
         const { bucket, localUri } = data;
-        const { pathname = '', fileGroup, contentType, metadata, endpoint, active, publicRead, acl, admin = {}, overwrite, options } = data.upload;
-        let filename = data.upload.filename || path.basename(localUri), bucketClient, bucketKey, listFiles;
+        const { pathname = '', flags = 0, fileGroup, contentType, metadata, endpoint, active, publicRead, acl, admin = {}, overwrite, options } = data.upload;
+        let filename = data.upload.filename || path.basename(localUri), chunkSizeBytes = 0, bucketClient, bucketKey, listFiles;
         const uploadResponse = (err, url) => {
             if (!firebase) {
                 BUCKET_SESSION.delete(bucket);
@@ -42,8 +42,8 @@ function upload(credential, service = 'gcp') {
         else {
             const configBucket = admin.configBucket;
             if (!BUCKET_SESSION.has(bucket)) {
-                const bucketAcl = admin.publicRead ? 'publicRead' : admin.acl;
-                const response = BUCKET_RESPONSE[_a = bucketKey = getBucketKey(credential, bucket, bucketAcl)] || (BUCKET_RESPONSE[_a] = client_1.createBucketV2.call(this, credential, bucket, bucketAcl, configBucket?.create || (0, client_1.createBucketRequest)(credential)));
+                const bucketAcl = admin.publicRead ? "publicRead" : admin.acl;
+                const response = BUCKET_RESPONSE[bucketKey = getBucketKey(credential, bucket, bucketAcl)] ||= client_1.createBucketV2.call(this, credential, bucket, bucketAcl, configBucket?.create);
                 if (!await response) {
                     uploadResponse(null);
                     return;
@@ -51,13 +51,49 @@ function upload(credential, service = 'gcp') {
                 BUCKET_SESSION.add(bucket);
             }
             bucketClient = storage.bucket(bucket);
-            const retentionPolicy = configBucket?.retentionPolicy;
-            if (typeof retentionPolicy === 'number') {
-                bucketClient.setRetentionPeriod(retentionPolicy)
-                    .then(([response]) => {
-                    this.formatMessage(64, service, ["Bucket configured" + ' (retention policy)', bucket], response.retentionPolicy?.effectiveTime, { ...Cloud.LOG_CLOUD_COMMAND });
-                })
-                    .catch(err => addLog(err));
+            if (configBucket) {
+                let { cors, lifecycle, retentionPolicy } = configBucket;
+                const commandMessage = (feature, message) => this.formatMessage(64, service, ["Bucket configured" + ` (${feature})`, bucket], message, { ...Cloud.LOG_CLOUD_COMMAND });
+                if (typeof retentionPolicy === 'number') {
+                    bucketClient.setRetentionPeriod(retentionPolicy)
+                        .then(([response]) => commandMessage('Retention Policy', response.retentionPolicy?.effectiveTime))
+                        .catch((err) => addLog(err));
+                }
+                if ((0, types_1.isObject)(cors)) {
+                    bucketClient.setCorsConfiguration(Array.isArray(cors) ? cors : [cors])
+                        .then(([response]) => {
+                        const length = response.cors?.length;
+                        commandMessage('CORS', length ? length + ' rules' : null);
+                    })
+                        .catch((err) => addLog(err));
+                }
+                if ((0, types_1.isObject)(lifecycle)) {
+                    let append;
+                    if (Array.isArray(lifecycle)) {
+                        if (lifecycle.length === 0) {
+                            append = false;
+                        }
+                        else {
+                            const arg = lifecycle[lifecycle.length - 1];
+                            if (typeof arg === 'boolean') {
+                                append = arg;
+                                lifecycle = lifecycle.slice(0, lifecycle.length - 1);
+                            }
+                        }
+                    }
+                    bucketClient.addLifecycleRule(lifecycle, { append })
+                        .then(([response]) => {
+                        const length = response.lifecycle?.rule?.length;
+                        commandMessage('Lifecycle', length ? length + ' rules' : null);
+                    })
+                        .catch((err) => addLog(err));
+                }
+            }
+            if (flags & 4) {
+                const chunkSize = (0, types_1.alignSize)(data.upload.chunkSize, 1024);
+                if (chunkSize > 0) {
+                    chunkSizeBytes = Math.max(chunkSize, 8388608);
+                }
             }
         }
         if (!overwrite && (!listFiles || listFiles.length)) {
@@ -71,7 +107,7 @@ function upload(credential, service = 'gcp') {
                         break;
                     }
                 }
-                const name = pathname ? Module.joinPath(pathname, filename) : filename;
+                const name = pathname ? Cloud.joinPath(pathname, filename) : filename;
                 if (firebase) {
                     exists = listFiles.includes(name);
                 }
@@ -83,16 +119,94 @@ function upload(credential, service = 'gcp') {
                 this.formatMessage(64, service, ["File renamed", current], filename, { ...Cloud.LOG_CLOUD_WARN });
             }
         }
-        const getURL = (value) => endpoint ? Module.joinPath(endpoint, value) : Module.joinPath("https://storage.googleapis.com", bucket, value);
+        const getURL = (value) => endpoint ? Cloud.joinPath(endpoint, value) : Cloud.joinPath("https://storage.googleapis.com", bucket, value);
         const Key = [filename];
         const Body = [data.buffer];
+        const Path = [];
+        const Stream = [];
         const ContentType = [contentType];
-        if (fileGroup) {
-            const [key, body, type] = (0, util_1.createKeyAndBody)(filename, fileGroup, addLog);
+        if (chunkSizeBytes > 0) {
+            const main = Body[0];
+            if (main.length > chunkSizeBytes) {
+                try {
+                    fs.writeFileSync(localUri, main);
+                    Path[0] = localUri;
+                }
+                catch {
+                    if (flags & 2) {
+                        Stream[0] = stream.Readable.from(main, { highWaterMark: chunkSizeBytes });
+                    }
+                }
+            }
+            else if (main.length === 0) {
+                try {
+                    if (fs.statSync(localUri).size > chunkSizeBytes) {
+                        Path[0] = localUri;
+                    }
+                    else if (flags & 2) {
+                        Stream[0] = fs.createReadStream(localUri, { highWaterMark: chunkSizeBytes, signal: this.signal });
+                    }
+                    else {
+                        Body[0] = fs.readFileSync(localUri);
+                    }
+                }
+                catch (err) {
+                    uploadResponse(err);
+                    return;
+                }
+            }
+            if (fileGroup) {
+                const [key, body, type] = (0, util_1.createKeyAndBody)(filename, fileGroup, chunkSizeBytes, addLog, flags);
+                Key.push(...key);
+                ContentType.push(...type);
+                for (let i = 0; i < body.length; ++i) {
+                    const item = body[i];
+                    if (typeof item === 'string') {
+                        Path[i + 1] = item;
+                    }
+                    else if (Buffer.isBuffer(item)) {
+                        Body[i + 1] = item;
+                    }
+                    else {
+                        Stream[i + 1] = item;
+                    }
+                }
+            }
+        }
+        else if ((flags & 2) && !firebase) {
+            try {
+                Stream.push(data.buffer.length ? stream.Readable.from(data.buffer) : fs.createReadStream(localUri, { signal: this.signal }));
+                if (fileGroup) {
+                    const [key, body, type] = (0, util_1.createKeyAndBody)(filename, fileGroup, 0, addLog, 2);
+                    Key.push(...key);
+                    Stream.push(...body);
+                    ContentType.push(...type);
+                }
+            }
+            catch (err) {
+                uploadResponse(err);
+                return;
+            }
+        }
+        else if (fileGroup) {
+            const [key, body, type] = (0, util_1.createKeyAndBody)(filename, fileGroup, 0, addLog);
             Key.push(...key);
             Body.push(...body);
             ContentType.push(...type);
         }
+        let headers, concurrencyLimit, maxQueueSize;
+        if (options) {
+            ({ headers, concurrencyLimit, maxQueueSize } = options);
+            if (headers !== undefined) {
+                delete options.headers;
+            }
+            if (concurrencyLimit !== undefined) {
+                delete options.concurrencyLimit;
+            }
+            if (maxQueueSize !== undefined) {
+                delete options.maxQueueSize;
+            }
+        }
         for (let i = 0; i < Key.length; ++i) {
             const first = i === 0;
             if (this.aborted) {
@@ -101,12 +215,21 @@ function upload(credential, service = 'gcp') {
                 }
                 return;
             }
-            const destination = pathname ? Module.joinPath(pathname, Key[i]) : Key[i];
+            const destination = pathname ? Cloud.joinPath(pathname, Key[i]) : Key[i];
             if (firebase) {
                 const { ref, uploadBytes } = require('@firebase/storage');
                 const params = { ...options };
                 if (first) {
-                    params.contentType || (params.contentType = ContentType[i]);
+                    if (Body[0].length === 0) {
+                        try {
+                            Body[0] = fs.readFileSync(localUri);
+                        }
+                        catch (err) {
+                            uploadResponse(err);
+                            return;
+                        }
+                    }
+                    params.contentType ||= ContentType[i];
                     if (metadata) {
                         params.customMetadata = metadata;
                     }
@@ -122,7 +245,7 @@ function upload(credential, service = 'gcp') {
                         uploadResponse(null, url);
                     }
                 })
-                    .catch(err => {
+                    .catch((err) => {
                     if (first) {
                         uploadResponse(err);
                     }
@@ -136,14 +259,14 @@ function upload(credential, service = 'gcp') {
                 const readable = publicRead || active && publicRead !== false && publicRead !== 0 && !acl;
                 let ACL, revoke, predefined;
                 if (first) {
-                    params.contentType || (params.contentType = ContentType[i]);
+                    params.contentType ||= ContentType[i];
                     if (readable) {
-                        ACL = 'publicRead';
+                        ACL = "publicRead";
                     }
                     else {
                         predefined = true;
                     }
-                    if (typeof metadata === 'object') {
+                    if ((0, types_1.isObject)(metadata)) {
                         params.metadata = metadata;
                     }
                 }
@@ -151,7 +274,7 @@ function upload(credential, service = 'gcp') {
                     params.contentType = ContentType[i];
                     if (!params.predefinedAcl) {
                         if (readable) {
-                            ACL = 'publicRead';
+                            ACL = "publicRead";
                         }
                         else {
                             predefined = true;
@@ -160,26 +283,47 @@ function upload(credential, service = 'gcp') {
                 }
                 if (predefined) {
                     if (publicRead === 0) {
-                        ACL = 'publicRead';
+                        ACL = "publicRead";
                         revoke = true;
                     }
                     if (acl && (!revoke || acl !== ACL)) {
                         params.predefinedAcl = acl;
                     }
                 }
-                params.resumable ?? (params.resumable = false);
                 const file = bucketClient.file(destination);
-                file.save(Body[i], params).then(async () => {
+                let uploadClient;
+                if (Path[i]) {
+                    const { TransferManager } = require('@google-cloud/storage');
+                    uploadClient = new TransferManager(bucketClient).uploadFileInChunks(Path[i], { uploadName: destination, chunkSizeBytes, headers, concurrencyLimit: data.upload.chunkLimit || concurrencyLimit, maxQueueSize });
+                }
+                else {
+                    params.resumable ??= false;
+                    if (Stream[i]) {
+                        if (chunkSizeBytes > 0) {
+                            params.chunkSize = chunkSizeBytes;
+                        }
+                        uploadClient = new Promise((resolve, reject) => {
+                            Stream[i].pipe(file.createWriteStream(params))
+                                .on('finish', resolve)
+                                .on('error', reject);
+                        });
+                    }
+                }
+                (uploadClient || file.save(Body[i], params))
+                    .then(async () => {
                     const url = getURL(file.name);
                     this.formatMessage(64, service, "Upload success", url, { ...Cloud.LOG_CLOUD_UPLOAD });
                     if (first) {
                         uploadResponse(null, url);
                     }
+                    if (Path[i]) {
+                        file.setMetadata(params).catch((err) => addLog(err));
+                    }
                     if (ACL) {
                         client_1.setPredefinedAcl.call(this, file.acl, ACL, file.bucket, file.name, revoke || (await file.getMetadata())[0], true);
                     }
                 })
-                    .catch(err => {
+                    .catch((err) => {
                     if (first) {
                         uploadResponse(err);
                     }
@@ -190,6 +334,4 @@ function upload(credential, service = 'gcp') {
             }
         }
     };
-}
-
-module.exports = upload;
+};