@pi-r/gcp 0.3.6 → 0.3.7

package/client/index.js

@@ -1,7 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getProjectId = exports.createBucketRequest = exports.setPredefinedAcl = exports.setPublicRead = exports.executeBatchQuery = exports.executeQuery = exports.deleteObjectsV2 = exports.deleteObjects = exports.setBucketWebsite = exports.setBucketPolicy = exports.createBucketV2 = exports.createBucket = exports.createDatabaseClient = exports.createStorageClient = exports.validateDatabase = exports.validateStorage = exports.isFirebaseApp = void 0;
+exports.CLOUD_UPLOAD_FROMDISK = exports.getProjectId = exports.createBucketRequest = exports.setPredefinedAcl = exports.setPublicRead = exports.executeBatchQuery = exports.executeQuery = exports.deleteObjectsV2 = exports.deleteObjects = exports.setBucketWebsite = exports.setBucketPolicy = exports.createBucketV2 = exports.createBucket = exports.createDatabaseClient = exports.createStorageClient = exports.validateDatabase = exports.validateStorage = exports.isFirebaseApp = void 0;
 const path = require("path");
+const storage = require("@google-cloud/storage");
+const firestore = require("@google-cloud/firestore");
 const util_1 = require("@e-mc/cloud/util");
 const types_1 = require("@e-mc/types");
 const Module = require("@e-mc/module");
@@ -61,23 +63,29 @@ async function hasBucket(client) {
     const [exists] = await client.exists().catch(err => isErrorConflict(err) ? [true] : [false]);
     return exists;
 }
-function updatePredefinedAcl(acl, values, metadata) {
-    const { generation, userProject } = metadata;
+function updatePredefinedAcl(acl, values, metadata = {}) {
     const items = [];
+    let projectNumber;
+    if ('projectNumber' in metadata || 'defaultObjectAcl' in metadata) {
+        let defaultObjectAcl;
+        ({ projectNumber, defaultObjectAcl } = metadata);
+        if (projectNumber === undefined && (0, types_1.isArray)(defaultObjectAcl)) {
+            const target = defaultObjectAcl.find(item => { var _a; return (_a = item.projectTeam) === null || _a === void 0 ? void 0 : _a.projectNumber; });
+            if (target !== undefined) {
+                projectNumber = target.projectTeam.projectNumber;
+            }
+        }
+    }
     for (const target of acl.default ? [acl, acl.default] : [acl]) {
         items.push(...values.map(async (value) => {
-            var _a;
-            if (generation !== undefined) {
-                (_a = value.generation) !== null && _a !== void 0 ? _a : (value.generation = +generation);
-            }
-            if (userProject) {
-                value.userProject || (value.userProject = userProject);
-            }
             const entity = await target.get({ entity: value.entity }).catch(() => null);
             if (entity) {
                 return (0, types_1.isString)(value.role) ? target.update(value) : target.delete(value);
             }
             if ((0, types_1.isString)(value.role)) {
+                if (projectNumber !== undefined) {
+                    value.userProject || (value.userProject = projectNumber.toString());
+                }
                 return target.add(value);
             }
         }));
@@ -100,8 +108,8 @@ function setUniformBucketLevel(bucket, enabled) {
 const getFirebasePackageName = (credential) => credential.apiKey && credential.authDomain ? 'firebase' : 'firebase-admin';
 const hasKeyFile = (credential) => !!(credential.keyFile || credential.keyFilename || process.env.GOOGLE_APPLICATION_CREDENTIALS);
 const isErrorConflict = (err) => (err instanceof Error) && err.code === 409;
-function isFirebaseApp(credential, storage, admin) {
-    return !!(credential.apiKey && credential.authDomain && (storage || credential.databaseURL) || credential.product === 'firebase' && process.env.GOOGLE_APPLICATION_CREDENTIALS) && (!admin || !credential.admin);
+function isFirebaseApp(credential, store, admin) {
+    return !!(credential.apiKey && credential.authDomain && (store || credential.databaseURL) || credential.product === 'firebase' && process.env.GOOGLE_APPLICATION_CREDENTIALS) && (!admin || !credential.admin);
 }
 exports.isFirebaseApp = isFirebaseApp;
 function validateStorage(credential) {
@@ -113,59 +121,60 @@ function validateDatabase(credential, data) {
 }
 exports.validateDatabase = validateDatabase;
 function createStorageClient(credential, bucket) {
-    let packageName = '';
-    try {
-        if (isFirebaseApp(credential, true)) {
+    if (isFirebaseApp(credential, true)) {
+        let packageName = '';
+        try {
             if (bucket) {
                 credential.storageBucket = bucket;
             }
             packageName = getFirebasePackageName(credential);
             return getApp(credential, false);
         }
-        credential.projectId || (credential.projectId = getProjectId(credential));
-        const { Storage } = require(packageName = '@google-cloud/storage');
-        return new Storage(credential);
-    }
-    catch (err) {
-        this.checkPackage(err, packageName, { passThrough: true });
-        throw err;
+        catch (err) {
+            this.checkPackage(err, packageName, { passThrough: true });
+            throw err;
+        }
     }
+    credential.projectId || (credential.projectId = getProjectId(credential));
+    return new storage.Storage(credential);
 }
 exports.createStorageClient = createStorageClient;
 function createDatabaseClient(credential, data) {
     let packageName = '';
     try {
-        const product = data === null || data === void 0 ? void 0 : data.product;
+        let product, name, table;
+        if (data) {
+            ({ product, name, table } = data);
+        }
         if (product === 'firebase' || isFirebaseApp(credential)) {
             packageName = getFirebasePackageName(credential);
             return getApp(credential, true);
         }
         credential.projectId || (credential.projectId = getProjectId(credential));
         if (data) {
-            if (product === 'spanner' || !product && data.name && data.database) {
+            if (product === 'spanner' || !product && name && data.database) {
                 const { Spanner } = require(packageName = '@google-cloud/spanner');
                 return new Spanner(credential);
             }
-            if (product === 'bigtable' || !product && data.name && data.table) {
+            if (product === 'bigtable' || !product && name && table) {
                 const { Bigtable } = require(packageName = '@google-cloud/bigtable');
                 return new Bigtable(credential);
             }
-            if (product === 'bigquery' || !product && !data.table && typeof data.query === 'string') {
+            if (product === 'bigquery' || !product && !table && typeof data.query === 'string') {
                 const { BigQuery } = require(packageName = '@google-cloud/bigquery');
                 return new BigQuery(credential);
             }
-            if (product === 'datastore' || !product && !data.table && (data.keys || data.query)) {
+            if (product === 'datastore' || !product && !table && (data.keys || data.query)) {
                 const { Datastore } = require(packageName = '@google-cloud/datastore');
                 return new Datastore(credential);
             }
         }
-        const { Firestore } = require(packageName = '@google-cloud/firestore');
-        return new Firestore(credential);
     }
     catch (err) {
         this.checkPackage(err, packageName, { passThrough: true });
         throw err;
     }
+    return new firestore.Firestore(credential);
 }
 exports.createDatabaseClient = createDatabaseClient;
 async function createBucket(credential, bucket, publicRead, service = 'gcp') {
@@ -176,14 +185,14 @@ async function createBucketV2(credential, bucket, acl, options, service = 'gcp')
     if (isFirebaseApp(credential, true, true)) {
         return false;
     }
-    const storage = createStorageClient.call(this, credential, bucket);
-    if (await hasBucket(storage.bucket(bucket))) {
+    const instance = createStorageClient.call(this, credential, bucket);
+    if (await hasBucket(instance.bucket(bucket))) {
         if (acl) {
             await setBucketPolicy.call(this, credential, bucket, { acl }, service);
         }
         return true;
     }
-    return storage.createBucket(bucket, options)
+    return instance.createBucket(bucket, options)
         .then(async ([target]) => {
         this.formatMessage(64, service, ["Bucket created", bucket], '', { ...Cloud.LOG_CLOUD_COMMAND });
         if (acl) {
@@ -207,7 +216,7 @@ async function setBucketPolicy(credential, bucket, options, service = 'gcp') {
     }
     const client = createStorageClient.call(this, credential, bucket).bucket(bucket);
     if (!await hasBucket(client)) {
-        this.formatFail(64, service, ["Bucket does not exist", bucket], (0, util_1.formatError)(service, "Bucket does not exist"), { ...Cloud.LOG_CLOUD_FAIL, fatal: false });
+        this.formatFail(64, service, ["Bucket does not exist", bucket], (0, types_1.errorMessage)(service, "Bucket does not exist"), { ...Cloud.LOG_CLOUD_FAIL, fatal: false });
         return false;
     }
     const errors = [];
@@ -218,7 +227,7 @@ async function setBucketPolicy(credential, bucket, options, service = 'gcp') {
             case 'private':
             case 'projectPrivate':
                 await client.makePrivate(options).catch(err => !isErrorConflict(err) && errors.push(err));
-                await setPredefinedAcl.call(this, client.acl, acl, client, '', (_a = options.bucket) === null || _a === void 0 ? void 0 : _a.metadata);
+                await setPredefinedAcl.call(this, client.acl, acl, client, '', (_a = options.bucket) === null || _a === void 0 ? void 0 : _a.metadata).catch(err => errors.push(err));
                 break;
             case 'authenticatedRead':
                 await client.makePublic(options).catch(err => !isErrorConflict(err) && errors.push(err));
@@ -245,7 +254,7 @@ async function setBucketPolicy(credential, bucket, options, service = 'gcp') {
         }
     }
     if (options.acl && (options.acl = options.acl.filter(user => (0, types_1.isString)(user.entity))).length) {
-        errors.push(...await updatePredefinedAcl(client.acl, options.acl, options));
+        errors.push(...await updatePredefinedAcl(client.acl, options.acl, options.metadata));
     }
     if (errors.length) {
         this.formatFail(64, service, ["Unable to update bucket policy", bucket], errors[0], { ...Cloud.LOG_CLOUD_FAIL, fatal: false });
@@ -260,11 +269,12 @@ function setBucketWebsite(credential, bucket, options, service = 'gcp') {
         return Promise.resolve(false);
     }
     const website = {};
-    if ((0, types_1.isString)(options.indexPage)) {
-        website.mainPageSuffix = options.indexPage;
+    const { indexPage, errorPage } = options;
+    if ((0, types_1.isString)(indexPage)) {
+        website.mainPageSuffix = indexPage;
     }
-    if ((0, types_1.isString)(options.errorPage)) {
-        website.notFoundPage = options.errorPage;
+    if ((0, types_1.isString)(errorPage)) {
+        website.notFoundPage = errorPage;
     }
     return createStorageClient.call(this, credential, bucket).bucket(bucket).setMetadata({ website })
         .then(() => {
@@ -282,7 +292,7 @@ function deleteObjects(credential, Bucket, service) {
 }
 exports.deleteObjects = deleteObjects;
 async function deleteObjectsV2(credential, bucket, recursive = true, service = 'gcp') {
-    const storage = createStorageClient.call(this, credential, bucket);
+    const instance = createStorageClient.call(this, credential, bucket);
     const errors = [];
     let promise;
     if (isFirebaseApp(credential, true, true)) {
@@ -290,7 +300,7 @@ async function deleteObjectsV2(credential, bucket, recursive = true, service = '
         const files = [];
         await (async function recurse(prefix) {
             try {
-                const { items, prefixes } = await listAll(ref(storage, prefix));
+                const { items, prefixes } = await listAll(ref(instance, prefix));
                 if (items.length) {
                     files.push(...items.map(value => value.fullPath));
                 }
@@ -305,11 +315,11 @@ async function deleteObjectsV2(credential, bucket, recursive = true, service = '
             }
         })('');
         if (files.length) {
-            promise = Promise.allSettled(files.map(value => deleteObject(ref(storage, value))));
+            promise = Promise.allSettled(files.map(value => deleteObject(ref(instance, value))));
         }
     }
     else {
-        const client = storage.bucket(bucket);
+        const client = instance.bucket(bucket);
         if (await hasBucket(client)) {
             if (recursive) {
                 return client.deleteFiles({ force: true }).then(() => this.formatMessage(64, service, "Bucket emptied" + ' (recursive)', bucket, { ...Cloud.LOG_CLOUD_COMMAND }));
@@ -375,9 +385,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
     };
     const closeClient = () => {
         const fs = clients[3];
-        if (fs) {
-            fs.terminate();
-        }
+        fs === null || fs === void 0 ? void 0 : fs.terminate();
     };
     const getFamily = (data, row, family) => {
         const familyData = data[family];
@@ -520,29 +528,30 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                 if ((0, types_1.isArray)(orderBy)) {
                     const sort = require('@firebase/database');
                     for (const items of orderBy) {
-                        if ((0, types_1.isArray)(items)) {
-                            const method = items[0];
-                            switch (method) {
-                                case 'orderByChild':
-                                    constraints.push(sort.orderByChild(items[1]));
-                                    break;
-                                case 'orderByKey':
-                                case 'orderByValue':
-                                case 'orderByPriority':
-                                    constraints.push(sort[method]());
-                                    break;
-                                case 'limitToFirst':
-                                case 'limitToLast':
-                                    constraints.push(sort[method](limiter = +items[1]));
-                                    break;
-                                case 'startAt':
-                                case 'startAfter':
-                                case 'endAt':
-                                case 'endBefore':
-                                case 'equalTo':
-                                    constraints.push(sort[method](items[1], items[2]));
-                                    break;
-                            }
+                        if (!(0, types_1.isArray)(items)) {
+                            continue;
+                        }
+                        const method = items[0];
+                        switch (method) {
+                            case 'orderByChild':
+                                constraints.push(sort.orderByChild(items[1]));
+                                break;
+                            case 'orderByKey':
+                            case 'orderByValue':
+                            case 'orderByPriority':
+                                constraints.push(sort[method]());
+                                break;
+                            case 'limitToFirst':
+                            case 'limitToLast':
+                                constraints.push(sort[method](limiter = +items[1]));
+                                break;
+                            case 'startAt':
+                            case 'startAfter':
+                            case 'endAt':
+                            case 'endBefore':
+                            case 'equalTo':
+                                constraints.push(sort[method](items[1], items[2]));
+                                break;
                         }
                     }
                 }
@@ -588,21 +597,22 @@ async function executeBatchQuery(credential, batch, sessionKey) {
             else {
                 const task = db.createQuery(item.kind);
                 for (const args of query) {
-                    if ((0, types_1.isArray)(args)) {
-                        const method = args[0];
-                        switch (method) {
-                            case 'end':
-                            case 'filter':
-                            case 'groupBy':
-                            case 'hasAncestor':
-                            case 'limit':
-                            case 'offset':
-                            case 'order':
-                            case 'select':
-                            case 'start':
-                                task[method].apply(task, args.slice(1));
-                                break;
-                        }
+                    if (!(0, types_1.isArray)(args)) {
+                        continue;
+                    }
+                    const method = args[0];
+                    switch (method) {
+                        case 'end':
+                        case 'filter':
+                        case 'groupBy':
+                        case 'hasAncestor':
+                        case 'limit':
+                        case 'offset':
+                        case 'order':
+                        case 'select':
+                        case 'start':
+                            task[method].apply(task, args.slice(1));
+                            break;
                     }
                 }
                 if (limit > 0) {
@@ -643,23 +653,24 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                 }
                 let db = createClient(item, 3).collection(table);
                 for (const args of query) {
-                    if ((0, types_1.isArray)(args)) {
-                        const method = args[0];
-                        switch (method) {
-                            case 'endAt':
-                            case 'endBefore':
-                            case 'limit':
-                            case 'limitToLast':
-                            case 'offset':
-                            case 'orderBy':
-                            case 'select':
-                            case 'startAfter':
-                            case 'startAt':
-                            case 'where':
-                            case 'withConverter':
-                                db = db[method].apply(db, args.slice(1));
-                                break;
-                        }
+                    if (!(0, types_1.isArray)(args)) {
+                        continue;
+                    }
+                    const method = args[0];
+                    switch (method) {
+                        case 'endAt':
+                        case 'endBefore':
+                        case 'limit':
+                        case 'limitToLast':
+                        case 'offset':
+                        case 'orderBy':
+                        case 'select':
+                        case 'startAfter':
+                        case 'startAt':
+                        case 'where':
+                        case 'withConverter':
+                            db = db[method].apply(db, args.slice(1));
+                            break;
                     }
                 }
                 if ((0, types_1.isArray)(orderBy)) {
@@ -725,25 +736,19 @@ async function setPredefinedAcl(target, value, bucket, pathname = '', metadata =
                 items = [];
             }
             else {
-                if (bucket) {
-                    const [users] = await bucket.acl.get().catch(() => [[]]);
-                    acl = Array.isArray(users) ? users : [users];
-                }
-                else {
-                    acl = [];
-                }
+                acl = bucket ? [(await bucket.acl.get())[1]] : [];
                 items = [{ entity: 'allUsers' }, { entity: 'allAuthenticatedUsers' }];
             }
             switch (value) {
                 case 'private': {
-                    const owners = new Set(acl.filter(item => item.role === 'OWNER').map(item => item.entity));
+                    const owners = new Set(acl.filter(item => item.role === 'OWNER' && item.entity).map(item => item.entity));
                     if (owner) {
                         owners.add(owner);
                     }
                     if (owners.size) {
                         for (const user of acl) {
                             const entity = user.entity;
-                            if (!owners.has(entity)) {
+                            if (entity && !owners.has(entity)) {
                                 items.push({ entity });
                             }
                         }
@@ -753,19 +758,18 @@ async function setPredefinedAcl(target, value, bucket, pathname = '', metadata =
                 case 'projectPrivate':
                     for (const user of acl) {
                         const entity = user.entity;
-                        if (entity.startsWith('project-')) {
-                            items.push({ ...user, role: entity.startsWith('project-editors-') || entity.startsWith('project-owners-') ? 'OWNER' : 'READER' });
-                        }
-                        else {
-                            items.push({ ...user, role: undefined });
+                        if (entity) {
+                            const role = entity.startsWith('project-') ? entity.startsWith('project-editors-') || entity.startsWith('project-owners-') ? 'OWNER' : 'READER' : '';
+                            items.push({ entity, role });
                         }
                     }
                     break;
                 case 'bucketOwnerRead':
                     if (bucket && owner) {
                         for (const user of acl) {
-                            if (user.entity !== owner) {
-                                items.push({ entity: user.entity, role: 'READER' });
+                            const entity = user.entity;
+                            if (entity && entity !== owner) {
+                                items.push({ entity, role: 'READER' });
                             }
                         }
                     }
@@ -794,10 +798,12 @@ function createBucketRequest(credential) {
     for (const attr in credential) {
         switch (attr) {
             case 'archive':
+            case 'autoclass':
             case 'coldline':
             case 'cors':
             case 'customPlacementConfig':
             case 'dra':
+            case 'enableObjectRetention':
             case 'location':
             case 'multiRegional':
             case 'nearline':
@@ -831,3 +837,4 @@ function getProjectId(credential) {
     return '';
 }
 exports.getProjectId = getProjectId;
+exports.CLOUD_UPLOAD_FROMDISK = true;

package/download/index.js

@@ -2,10 +2,10 @@
 const path = require("path");
 const fs = require("fs");
 const https = require("https");
-const client_1 = require("@pi-r/gcp");
 const types_1 = require("@e-mc/types");
 const Module = require("@e-mc/module");
 const Cloud = require("@e-mc/cloud");
+const client_1 = require("@pi-r/gcp");
 function download(credential, service = 'gcp') {
     const storage = client_1.createStorageClient.call(this, credential);
     return (data, callback) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-r/gcp",
-  "version": "0.3.6",
+  "version": "0.3.7",
   "description": "Google (GCP) cloud functions for E-mc.",
   "main": "client/index.js",
   "publishConfig": {

package/upload/index.js

@@ -1,11 +1,11 @@
 "use strict";
 const path = require("path");
 const fs = require("fs");
-const client_1 = require("@pi-r/gcp");
 const util_1 = require("@e-mc/cloud/util");
 const types_1 = require("@e-mc/types");
 const Module = require("@e-mc/module");
 const Cloud = require("@e-mc/cloud");
+const client_1 = require("@pi-r/gcp");
 const BUCKET_SESSION = new Set();
 const BUCKET_RESPONSE = {};
 const getBucketKey = (credential, bucket, acl = '') => Module.asString(credential, true) + bucket + '_' + acl;