@pittica/google-business-intelligence-helpers 1.15.0 → 1.15.2

package/dist/contents/backup.js

@@ -13,11 +13,13 @@
 // limitations under the License.
 
 const {
-  getBucket
+  getBucket,
+  groupJson
 } = require("@pittica/google-cloud-storage-helpers");
 const {
   getDataset,
-  jobDone
+  jobDone,
+  getTable
 } = require("@pittica/google-bigquery-helpers");
 const log = require("@pittica/logger-helpers");
 const {
@@ -39,7 +41,7 @@ exports.backup = async (id, bucket, projectId, dateFormat = "YYYY-MM-DD") => {
   });
   const [tables] = await dataset.getTables();
   const date = format(new Date(), dateFormat);
-  return await exports.extract(bucket, date, tables, 0);
+  return await exports.extract(bucket, date, tables, 0, dataset.metadata.location);
 };
 
 /**
@@ -49,20 +51,21 @@ exports.backup = async (id, bucket, projectId, dateFormat = "YYYY-MM-DD") => {
  * @param {string} date Date in format YYYY-MM-DD.
  * @param {Array} tables An array of BigQuery tables.
  * @param {number} index Array index.
+ * @param {string} location Location of the BigQuery dataset.
  * @returns {boolean} A value indicating whehter the job has been done.
  */
-exports.extract = async (bucket, date, tables, index) => {
+exports.extract = async (bucket, date, tables, index = 0, location = "us-central1") => {
   if (index < tables.length) {
     const table = tables[index];
     return await table.extract(getBucket(bucket).file(`${date}/${table.id}-*.json`), {
-      location: "us-central1",
+      location,
       format: "json",
       gzip: false
     }).then(async response => {
       if (jobDone(response)) {
         const [job] = response;
         log.success(`Job backup "${job.id}" for table "${table.id}" done.`);
-        return await exports.extract(date, tables, index + 1);
+        return await exports.extract(bucket, date, tables, index + 1, location);
       } else {
         return false;
       }
@@ -88,7 +91,7 @@ exports.rollback = async (date, id, projectId, sourceBucket, destinationDataset)
   const dataset = await getDataset(id, {
     projectId
   });
-  const bucket = getBucket(sourceBucket);
+  const bucket = getBucket(sourceBucket, projectId);
   const [response] = await bucket.getFiles({
     prefix: `${date}/`
   });
@@ -96,16 +99,19 @@ exports.rollback = async (date, id, projectId, sourceBucket, destinationDataset)
   log.info(`Begin rollback "${date}".`);
   Object.keys(files).forEach(async key => {
     const [table] = await dataset.table(key).get();
-    files[key].forEach(filename => bigquery.dataset(destinationDataset, {
-      location: dataset.metadata.location,
-      projectId
-    }).table(key).load(bucket.file(`${date}/${filename}`), {
-      sourceFormat: "NEWLINE_DELIMITED_JSON",
-      schema: table.metadata.schema,
-      timePartitioning: table.metadata.timePartitioning,
-      location: table.metadata.location,
-      tableConstraints: table.metadata.tableConstraints,
-      partitionDefinition: table.metadata.partitionDefinition
-    }));
+    files[key].forEach(async filename => {
+      const destination = await getTable(destinationDataset, key, {
+        projectId,
+        location: dataset.metadata.location
+      });
+      await destination.load(bucket.file(`${date}/${filename}`), {
+        sourceFormat: "NEWLINE_DELIMITED_JSON",
+        schema: table.metadata.schema,
+        timePartitioning: table.metadata.timePartitioning,
+        location: table.metadata.location,
+        tableConstraints: table.metadata.tableConstraints,
+        partitionDefinition: table.metadata.partitionDefinition
+      });
+    });
   });
 };

package/dist/contents/storage.js

@@ -169,9 +169,9 @@ exports.erase = async (bucket, days = process.env.BACKUP_KEEP_DAYS || 3) => {
   });
   const folders = Object.keys(result).sort();
   const blacklist = folders.slice(0, folders.length - parseInt(days));
-  blacklist.forEach(async prefix => await b.deleteFiles({
+  await Promise.all(blacklist.map(async prefix => await b.deleteFiles({
     prefix
-  }));
+  })));
   log.success("Backup bucket erased.");
   return true;
 };

package/dist/scripts/backup.js

@@ -34,10 +34,14 @@ exports.scriptBackup = backup;
 /**
  * Creates a backup and erases the bucket.
  *
+ * @param {string} id BigQuery dataset ID.
+ * @param {string} bucket Bucket name.
+ * @param {string} projectId Google Cloud project ID.
  * @param {number} days Number of days to keep.
+ * @param {string} dateFormat Date format string.
  * @returns {boolean} A value indicating whehter the job has been done.
  */
-exports.scriptBackupErase = async days => await backup().then(async () => await erase(days));
+exports.scriptBackupErase = async (id, bucket, projectId, days, dateFormat = "YYYY-MM-DD") => await backup(id, bucket, projectId, dateFormat).then(async () => await erase(bucket, days));
 
 /**
  * Rollback data using backup.

package/dist/scripts/storage.js

@@ -26,6 +26,7 @@ const {
  * @param {string} dateFormat Date format string.
  */
 exports.scriptImplode = implode;
+
 /**
  * Erases the bucket.
  *

package/dist/scripts/unlistened.js

@@ -36,12 +36,12 @@ exports.scriptUnlistened = async (bucket, folder) => {
     const storage = getStorage();
     const [files] = await getFiles(storage, bucket);
     if (files) {
-      files.forEach(file => {
+      files.forEach(async file => {
         const {
           name
         } = splitName(file.id);
         if (!hasSchema(name, folder)) {
-          deleteFile(file);
+          await deleteFile(file);
         }
       });
     }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@pittica/google-business-intelligence-helpers",
   "private": false,
-  "version": "1.15.0",
+  "version": "1.15.2",
   "description": "Helpers for business intelligence for Google Cloud.",
   "main": "dist/index.js",
   "scripts": {
@@ -36,8 +36,8 @@
     "node": ">=24.11.0"
   },
   "dependencies": {
-    "@pittica/google-bigquery-helpers": "^1.6.1",
-    "@pittica/google-cloud-storage-helpers": "^1.5.0",
+    "@pittica/google-bigquery-helpers": "^1.7.0",
+    "@pittica/google-cloud-storage-helpers": "^1.6.0",
     "@pittica/logger-helpers": "^1.2.2",
     "date-and-time": "^4.7.0"
   },