@pi-r/gcp 0.9.3 → 0.10.0

package/client/index.js

@@ -54,26 +54,43 @@ function getApp(credential, isDB) {
         const { getStorage } = require('@firebase/storage');
         return getStorage(app);
     }
-    {
-        const { getApps, initializeApp, applicationDefault } = require('firebase-admin/app');
-        let app;
-        for (const item of getApps()) {
+    const adminApp = require('firebase-admin/app');
+    const clientApp = (0, types_1.isString)(credential.admin) || (0, types_1.isPlainObject)(credential.admin);
+    let app;
+    if (!clientApp) {
+        for (const item of adminApp.getApps()) {
             if (!isDB && storageBucket && item.name === storageBucket || isDB && databaseURL && item.name === databaseURL) {
                 app = item;
                 break;
             }
         }
-        if (!app) {
-            app = isDB ? ADMIN_DB[databaseURL || "[DEFAULT]"] ||= initializeApp({ credential: applicationDefault(), databaseURL }, databaseURL) : ADMIN_STORAGE[storageBucket || "[DEFAULT]"] ||= initializeApp({ credential: applicationDefault(), storageBucket }, storageBucket);
-        }
         credential.admin = true;
-        if (isDB) {
-            const { getDatabase } = require('firebase-admin/database');
-            return getDatabase(app);
+    }
+    if (!app) {
+        let key = (isDB ? databaseURL : storageBucket) || "[DEFAULT]";
+        if (clientApp) {
+            key += JSON.stringify(credential.admin);
+        }
+        app = isDB ? ADMIN_DB[key] : ADMIN_STORAGE[key];
+        if (!app) {
+            let lib, credentialApp;
+            if (clientApp) {
+                lib = require('firebase-admin');
+                credentialApp = adminApp.cert(credential.admin);
+            }
+            else {
+                lib = adminApp;
+                credentialApp = adminApp.applicationDefault();
+            }
+            app = isDB ? ADMIN_DB[key] = lib.initializeApp({ credential: credentialApp, databaseURL }, key) : ADMIN_STORAGE[key] = lib.initializeApp({ credential: credentialApp, storageBucket }, key);
         }
-        const { getStorage } = require('firebase-admin/storage');
-        return getStorage(app);
     }
+    if (isDB) {
+        const { getDatabase } = require('firebase-admin/database');
+        return getDatabase(app);
+    }
+    const { getStorage } = require('firebase-admin/storage');
+    return getStorage(app);
 }
 async function hasBucket(client) {
     const [exists] = await client.exists().catch((err) => isErrorConflict(err) ? [true] : [false]);
@@ -121,9 +138,10 @@ async function updatePredefinedAcl(acl, values, metadata = {}) {
 async function setUniformBucketLevel(bucket, enabled) {
     return bucket.setMetadata({ iamConfiguration: { uniformBucketLevelAccess: { enabled } } });
 }
+const keyLimit = (limit) => limit > 0 ? `_${limit}` : '_0';
 const asFieldPath = (params) => Array.isArray(params) && params.every(item => typeof item === 'string') ? new firestore_1.FieldPath(...params) : params;
-const getFirebasePackageName = (credential) => credential.apiKey && credential.authDomain ? 'firebase' : 'firebase-admin';
-const isErrorConflict = (err) => (err instanceof Error) && err.code === 409;
+const getFirebasePackageName = (credential) => credential.apiKey && credential.authDomain && credential ? 'firebase' : 'firebase-admin';
+const isErrorConflict = (err) => err instanceof Error && err.code === 409;
 const hasGoogleAuth = (credential) => (0, types_1.isString)(credential.keyFilename || credential.keyFile || process.env.GOOGLE_APPLICATION_CREDENTIALS) || (0, types_1.isObject)(credential.credentials) || !!credential.authClient;
 function isFirebaseApp(credential, store, admin) {
     return !!(credential.apiKey && credential.authDomain && (store || credential.databaseURL) || credential.product === "firebase" && process.env.GOOGLE_APPLICATION_CREDENTIALS) && (!admin || !credential.admin);
@@ -136,12 +154,11 @@ function validateDatabase(credential, data) {
 }
 function createStorageClient(credential, bucket) {
     if (isFirebaseApp(credential, true)) {
-        let packageName = '';
+        const packageName = getFirebasePackageName(credential);
         try {
             if (bucket) {
                 credential.storageBucket = bucket;
             }
-            packageName = getFirebasePackageName(credential);
             return getApp(credential, false);
         }
         catch (err) {
@@ -153,7 +170,7 @@ function createStorageClient(credential, bucket) {
     return new storage_1.Storage(credential);
 }
 function createDatabaseClient(credential, data) {
-    let packageName = '';
+    let packageName;
     try {
         if (data?.product === "firebase" || isFirebaseApp(credential)) {
             packageName = getFirebasePackageName(credential);
@@ -413,14 +430,12 @@ async function executeBatchQuery(credential, batch, sessionKey) {
     const clients = {};
     const caching = length > 0 && this.hasCache(batch[0].service, sessionKey);
     const cacheValue = { value: this.valueOfKey(credential, 'cache'), sessionKey };
-    const authItems = [];
     const createClient = (data, type) => {
         const client = clients[type];
         if (client) {
             return client;
         }
         const item = length === 1 ? credential : { ...credential };
-        authItems.push(item);
         return (clients[type] = createDatabaseClient.call(this, item, data));
     };
     const closeClient = () => {
@@ -454,7 +469,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
     };
     for (let i = 0; i < length; ++i) {
         const item = batch[i];
-        const { service, product, name, database, table, id, query, limit = 0, update, ignoreCache } = item;
+        const { service, product, name, database, table, id, query, limit = 0, flags = 0, update, ignoreCache } = item;
         const useCache = caching && ignoreCache !== true;
         cacheValue.exclusiveOf = Array.isArray(ignoreCache) ? ignoreCache : undefined;
         let rows, queryString = '';
@@ -526,7 +541,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
             }
             const { updateType, options = {} } = item;
             if (useCache) {
-                queryString = name + '_' + database + (table ? '_' + table + (columns ? Cloud.asString(columns, true) : '') : '') + '_' + Cloud.asString(query, true) + (limit > 0 ? '_' + limit : '');
+                queryString = name + '_' + database + (table ? '_' + table + (columns ? Cloud.asString(columns, true) : '') : '') + '_' + Cloud.asString(query, true) + keyLimit(limit);
                 if (!update && (rows = this.getCacheResult(service, credential, queryString, cacheValue, ignoreCache))) {
                     result[i] = rows;
                     continue;
@@ -672,7 +687,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                 throw (0, util_1.formatError)(item, "Missing database query");
             }
             if (useCache) {
-                queryString = (keys ? Cloud.asString(keys, true) : (kind ? (name ? '_' + name : '') + '_' + Cloud.asString(kind, true) : '') + '_' + Cloud.asString(query, true)) + (limit > 0 ? '_' + limit : 0) + (options ? Cloud.asString(options, true) : '');
+                queryString = (keys ? Cloud.asString(keys, true) : (kind ? (name ? '_' + name : '') + '_' + Cloud.asString(kind, true) : '') + '_' + Cloud.asString(query, true) + keyLimit(limit)) + (options ? Cloud.asString(options, true) : '');
                 if (!update && (rows = this.getCacheResult(service, credential, queryString, cacheValue, ignoreCache))) {
                     result[i] = rows;
                     continue;
@@ -742,7 +757,16 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                 closeClient();
                 throw (0, util_1.formatError)(item, "Missing database table");
             }
-            if (id) {
+            const db = createClient(item, 3);
+            const group = !table.includes('/') && (flags & 1) === 1;
+            if (item.aggregateSpec) {
+                const col = group ? db.collectionGroup(table) : db.collection(table);
+                const data = (await col.aggregate(item.aggregateSpec).get()).data();
+                if (data) {
+                    rows = [data];
+                }
+            }
+            else if ((0, types_1.isString)(id) || Array.isArray(id)) {
                 const options = Array.isArray(id) ? item.options : null;
                 if (useCache) {
                     queryString = table + '_' + id + (options ? Cloud.asString(options, true) : '');
@@ -751,7 +775,6 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                         continue;
                     }
                 }
-                const db = createClient(item, 3);
                 if (Array.isArray(id)) {
                     const items = id.map(ref => db.doc(table + '/' + ref));
                     if (options) {
@@ -762,7 +785,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                 else {
                     const doc = db.collection(table).doc(id);
                     if ((0, types_1.isPlainObject)(update)) {
-                        const pattern = /^__(delete|serverTimestamp|increment|vector|(?:array)(?:Union|Remove))(?:<(.+)>)?__$/s;
+                        const pattern = /^__(delete|serverTimestamp|increment|vector|array(?:Union|Remove))(?:<(.+)>)?__$/s;
                         for (const attr in update) {
                             const match = pattern.exec(attr);
                             if (match) {
@@ -832,81 +855,115 @@ async function executeBatchQuery(credential, batch, sessionKey) {
                     }
                 }
             }
-            else if (Array.isArray(query)) {
-                const db = createClient(item, 3).collection(table);
-                if (item.params && query.every(value => typeof value === 'number')) {
-                    const options = item.options;
-                    if (useCache && (rows = this.getCacheResult(service, credential, queryString = table + '_' + Cloud.asString(item.params, true) + '_' + Cloud.asString(query, true) + (options ? Cloud.asString(options, true) : ''), cacheValue, ignoreCache))) {
-                        result[i] = rows;
-                        continue;
-                    }
-                    rows = (await db.findNearest(asFieldPath(item.params), firestore_1.FieldValue.vector(query), options).get()).docs.map(doc => doc.data());
-                }
-                else {
-                    const orderBy = item.orderBy;
-                    if (useCache && (rows = this.getCacheResult(service, credential, queryString = table + '_' + Cloud.asString(query, true) + '_' + Cloud.asString(orderBy, true) + (limit > 0 ? '_' + limit : ''), cacheValue, ignoreCache))) {
-                        result[i] = rows;
-                        continue;
+            else {
+                const options = item.options || {};
+                if (options.vectorField || item.params || Array.isArray(query)) {
+                    if (options.vectorField) {
+                        if (Array.isArray(query)) {
+                            options.queryVector ||= firestore_1.FieldValue.vector(query);
+                        }
+                        if (limit > 0) {
+                            options.limit = limit;
+                        }
+                        rows = (await db.collection(table).findNearest(options).get()).docs.map(doc => doc.data());
                     }
-                    let target = db;
-                    for (const args of query) {
-                        if ((0, types_1.isArray)(args)) {
-                            const method = args[0];
-                            const params = args.slice(1);
-                            switch (method) {
-                                case 'whereAnd':
-                                case 'whereOr':
-                                    if (!params.every(p => (0, types_1.isArray)(p))) {
-                                        throw (0, types_1.errorMessage)(method, 'Conditionals are not in array format', Cloud.asString(params));
-                                    }
-                                    target = target.where(firestore_1.Filter[method === 'whereAnd' ? 'and' : 'or'](...params.map((p) => firestore_1.Filter.where(p[0], p[1], p[2]))));
-                                    break;
-                                case 'findNearest':
-                                    params[0] = asFieldPath(params[0]);
-                                    target = target.findNearest.apply(target, params).query;
-                                    break;
-                                case 'where':
-                                case 'orderBy':
-                                    params[0] = asFieldPath(params[0]);
-                                    target = target[method].apply(target, params);
-                                case 'select':
-                                    params.forEach((field, index) => {
-                                        params[index] = asFieldPath(field);
-                                    });
-                                    target = target.select.apply(target, params);
-                                    break;
-                                case 'endAt':
-                                case 'endBefore':
-                                case 'limit':
-                                case 'limitToLast':
-                                case 'offset':
-                                case 'startAfter':
-                                case 'startAt':
-                                case 'withConverter':
-                                    target = target[method].apply(target, params);
-                                    break;
+                    else if (item.params && (options.queryVector || Array.isArray(query) && query.every(value => typeof value === 'number'))) {
+                        if (!options.queryVector) {
+                            if (useCache && (rows = this.getCacheResult(service, credential, queryString = table + '_' + Cloud.asString(item.params, true) + '_' + Cloud.asString(query, true) + (options ? Cloud.asString(options, true) : '') + keyLimit(limit), cacheValue, ignoreCache))) {
+                                result[i] = rows;
+                                continue;
                             }
+                            options.queryVector = firestore_1.FieldValue.vector(query);
                         }
+                        options.vectorField = asFieldPath(item.params);
+                        if (limit > 0) {
+                            options.limit = limit;
+                        }
+                        rows = (await db.collection(table).findNearest(options).get()).docs.map(doc => doc.data());
                     }
-                    if (Array.isArray(orderBy)) {
-                        for (const order of orderBy) {
-                            if ((0, types_1.isArray)(order)) {
-                                target = target.orderBy(asFieldPath(order[0]), order[1]);
+                    else {
+                        const orderBy = item.orderBy;
+                        if (useCache && (rows = this.getCacheResult(service, credential, queryString = table + '_' + Cloud.asString(query, true) + '_' + Cloud.asString(orderBy, true) + `_${group ? '1' : '0'}_${typeof id === 'number' ? '1' : '0'}` + keyLimit(limit), cacheValue, ignoreCache))) {
+                            result[i] = rows;
+                            continue;
+                        }
+                        const targets = [];
+                        if (group) {
+                            const col = db.collectionGroup(table);
+                            if (typeof id === 'number') {
+                                for await (const partition of col.getPartitions(id)) {
+                                    targets.push(partition.toQuery());
+                                }
                             }
-                            else if ((0, types_1.isString)(order)) {
-                                target = target.orderBy(order);
+                            else {
+                                targets.push(col);
                             }
                         }
+                        else {
+                            targets.push(db.collection(table));
+                        }
+                        rows = [];
+                        for (let target of targets) {
+                            for (const args of query) {
+                                if ((0, types_1.isArray)(args)) {
+                                    const method = args[0];
+                                    const params = args.slice(1);
+                                    switch (method) {
+                                        case 'whereAnd':
+                                        case 'whereOr':
+                                            if (!params.every(p => (0, types_1.isArray)(p))) {
+                                                throw (0, types_1.errorMessage)(method, 'Conditionals are not in array format', Cloud.asString(params));
+                                            }
+                                            target = target.where(firestore_1.Filter[method === 'whereAnd' ? 'and' : 'or'](...params.map((p) => firestore_1.Filter.where(p[0], p[1], p[2]))));
+                                            break;
+                                        case 'findNearest':
+                                            params[0] = asFieldPath(params[0]);
+                                            target = target.findNearest.apply(target, params).query;
+                                            break;
+                                        case 'where':
+                                        case 'orderBy':
+                                            params[0] = asFieldPath(params[0]);
+                                            target = target[method].apply(target, params);
+                                        case 'select':
+                                            params.forEach((field, index) => {
+                                                params[index] = asFieldPath(field);
+                                            });
+                                            target = target.select.apply(target, params);
+                                            break;
+                                        case 'endAt':
+                                        case 'endBefore':
+                                        case 'limit':
+                                        case 'limitToLast':
+                                        case 'offset':
+                                        case 'startAfter':
+                                        case 'startAt':
+                                        case 'withConverter':
+                                            target = target[method].apply(target, params);
+                                            break;
+                                    }
+                                }
+                            }
+                            if (Array.isArray(orderBy)) {
+                                for (const order of orderBy) {
+                                    if ((0, types_1.isArray)(order)) {
+                                        target = target.orderBy(asFieldPath(order[0]), order[1]);
+                                    }
+                                    else if ((0, types_1.isString)(order)) {
+                                        target = target.orderBy(order);
+                                    }
+                                }
+                            }
+                            if (limit > 0) {
+                                target = target.limit(limit);
+                            }
+                            rows = rows.concat((await target.get()).docs.map(doc => doc.data()));
+                        }
                     }
-                    if (limit > 0) {
-                        target = target.limit(limit);
-                    }
-                    rows = (await target.get()).docs.map(doc => doc.data());
                 }
-            }
-            else {
-                closeClient();
-                throw (0, util_1.formatError)(item, "Missing database query");
+                else {
+                    closeClient();
+                    throw (0, util_1.formatError)(item, "Missing database query");
+                }
             }
         }
         else {
@@ -1041,6 +1098,7 @@ function createBucketRequest(credential) {
             case 'location':
             case 'locationType':
             case 'logging':
+            case 'generation':
             case 'metageneration':
             case 'name':
             case 'objectRetention':
@@ -1048,6 +1106,8 @@ function createBucketRequest(credential) {
             case 'projectNumber':
             case 'retentionPolicy':
             case 'rpo':
+            case 'softDeleteTime':
+            case 'hardDeleteTime':
             case 'softDeletePolicy':
             case 'storageClass':
             case 'timeCreated':

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-r/gcp",
-  "version": "0.9.3",
+  "version": "0.10.0",
   "description": "Google (GCP) cloud functions for E-mc.",
   "main": "client/index.js",
   "publishConfig": {
@@ -19,10 +19,10 @@
   "license": "MIT",
   "homepage": "https://github.com/anpham6/pi-r#readme",
   "dependencies": {
-    "@e-mc/cloud": "^0.11.4",
-    "@e-mc/module": "^0.11.4",
-    "@e-mc/types": "^0.11.4",
+    "@e-mc/cloud": "^0.12.0",
+    "@e-mc/module": "^0.12.0",
+    "@e-mc/types": "^0.12.0",
     "@google-cloud/firestore": "^7.11.0",
-    "@google-cloud/storage": "^7.15.2"
+    "@google-cloud/storage": "^7.16.0"
   }
 }

package/types/index.d.ts

@@ -2,8 +2,10 @@ import type { CloudDatabase, CloudStorage } from '@e-mc/types/lib/cloud';
 
 import type { StorageOptions } from '@google-cloud/storage';
 import type { PathType } from '@google-cloud/datastore';
+import type { AggregateSpec } from '@google-cloud/firestore';
 import type { entity } from '@google-cloud/datastore/build/src/entity';
 import type { FirebaseOptions as IFirebaseOptions } from '@firebase/app';
+import type { ServiceAccount } from 'firebase-admin/app';
 
 // @ts-ignore
 import type { GoogleAuthOptions } from 'google-auth-library';
@@ -17,19 +19,21 @@ export interface GCPDatabaseQuery<T = PlainObject> extends Omit<CloudDatabase<un
     service: "gcp" | "gcloud";
     product?: "firestore" | "firebase" | "bigquery" | "bigtable" | "datastore" | "spanner";
     database?: string;
-    id?: string | string[];
+    id?: string | string[] | number;
     updateType?: 0 | 1 | 2 | 3;
     columns?: string[];
     keys?: DatastoreKey | DatastoreKey[];
     kind?: string | string[];
     orderBy?: unknown[][];
+    aggregateSpec?: AggregateSpec;
+    flags?: number;
 }
 
 export type GCPDatabaseCredential = GoogleAuthOptions;
 
 export interface FirebaseOptions extends IFirebaseOptions {
     product?: "firebase";
-    admin?: boolean;
+    admin?: boolean | string | ServiceAccount;
 }
 
 export type DatastoreKey = string | PathType[] | entity.KeyOptions;

package/upload/index.js

@@ -2,6 +2,7 @@
 const path = require("node:path");
 const fs = require("node:fs");
 const stream = require("node:stream");
+const storage_1 = require("@google-cloud/storage");
 const Cloud = require("@e-mc/cloud");
 const types_1 = require("@e-mc/types");
 const util_1 = require("@e-mc/cloud/util");
@@ -61,9 +62,9 @@ function upload(credential, service = 'gcp') {
                     this.formatMessage(64, service, ["Bucket configured" + ` (${feature})`, bucket], message, { ...Cloud.LOG_CLOUD_COMMAND });
                 };
                 if (typeof retentionPolicy === 'number') {
-                    bucketClient.setRetentionPeriod(retentionPolicy)
+                    (retentionPolicy === 0 ? bucketClient.removeRetentionPeriod() : bucketClient.setRetentionPeriod(retentionPolicy))
                         .then(([response]) => {
-                        commandMessage('Retention Policy', response.retentionPolicy?.effectiveTime);
+                        commandMessage('Retention Policy', response.retentionPolicy?.[retentionPolicy === 0 ? 'retentionPeriod' : 'effectiveTime']);
                     })
                         .catch(addLog);
                 }
@@ -298,11 +299,10 @@ function upload(credential, service = 'gcp') {
                         params.predefinedAcl = acl;
                     }
                 }
-                const file = bucketClient.file(destination);
+                const file = bucketClient.file(destination, params);
                 let uploadClient;
                 if (Path[i]) {
-                    const { TransferManager } = require('@google-cloud/storage');
-                    uploadClient = new TransferManager(bucketClient).uploadFileInChunks(Path[i], { uploadName: destination, chunkSizeBytes, headers, concurrencyLimit: data.upload.chunkLimit || concurrencyLimit, maxQueueSize });
+                    uploadClient = new storage_1.TransferManager(bucketClient).uploadFileInChunks(Path[i], { uploadName: destination, chunkSizeBytes, headers, concurrencyLimit: data.upload.chunkLimit || concurrencyLimit, maxQueueSize });
                 }
                 else {
                     params.resumable ??= false;
@@ -316,8 +316,11 @@ function upload(credential, service = 'gcp') {
                                 .on('error', reject);
                         });
                     }
+                    else {
+                        uploadClient = file.save(Body[i], params);
+                    }
                 }
-                (uploadClient || file.save(Body[i], params))
+                uploadClient
                     .then(async () => {
                     const url = getURL(file.name);
                     this.formatMessage(64, service, "Upload success", url, { ...Cloud.LOG_CLOUD_UPLOAD });