@pittica/google-business-intelligence-helpers 1.15.1 → 1.15.3

package/dist/contents/backup.js

@@ -13,11 +13,13 @@
 // limitations under the License.
 
 const {
-  getBucket
+  getBucket,
+  groupJson
 } = require("@pittica/google-cloud-storage-helpers");
 const {
   getDataset,
-  jobDone
+  jobDone,
+  getTable
 } = require("@pittica/google-bigquery-helpers");
 const log = require("@pittica/logger-helpers");
 const {
@@ -33,13 +35,13 @@ const {
  * @param {string} dateFormat Date format string.
  * @returns {boolean} A value indicating whehter the job has been done.
  */
-exports.backup = async (id, bucket, projectId, dateFormat = "YYYY-MM-DD") => {
+exports.backup = async (id, bucket, projectId = null, dateFormat = "YYYY-MM-DD") => {
   const dataset = await getDataset(id, {
     projectId
   });
   const [tables] = await dataset.getTables();
   const date = format(new Date(), dateFormat);
-  return await exports.extract(bucket, date, tables, 0);
+  return await exports.extract(bucket, date, tables, 0, dataset.metadata.location, projectId);
 };
 
 /**
@@ -49,20 +51,22 @@ exports.backup = async (id, bucket, projectId, dateFormat = "YYYY-MM-DD") => {
  * @param {string} date Date in format YYYY-MM-DD.
  * @param {Array} tables An array of BigQuery tables.
  * @param {number} index Array index.
+ * @param {string} location Location of the BigQuery dataset.
+ * @param {string} projectId Google Cloud project ID.
  * @returns {boolean} A value indicating whehter the job has been done.
  */
-exports.extract = async (bucket, date, tables, index) => {
+exports.extract = async (bucket, date, tables, index = 0, location = "us-central1", projectId = null) => {
   if (index < tables.length) {
     const table = tables[index];
-    return await table.extract(getBucket(bucket).file(`${date}/${table.id}-*.json`), {
-      location: "us-central1",
+    return await table.extract(getBucket(bucket, projectId).file(`${date}/${table.id}-*.json`), {
+      location,
       format: "json",
       gzip: false
     }).then(async response => {
       if (jobDone(response)) {
         const [job] = response;
         log.success(`Job backup "${job.id}" for table "${table.id}" done.`);
-        return await exports.extract(date, tables, index + 1);
+        return await exports.extract(bucket, date, tables, index + 1, location, projectId);
       } else {
         return false;
       }
@@ -88,7 +92,7 @@ exports.rollback = async (date, id, projectId, sourceBucket, destinationDataset)
   const dataset = await getDataset(id, {
     projectId
   });
-  const bucket = getBucket(sourceBucket);
+  const bucket = getBucket(sourceBucket, projectId);
   const [response] = await bucket.getFiles({
     prefix: `${date}/`
   });
@@ -96,16 +100,19 @@ exports.rollback = async (date, id, projectId, sourceBucket, destinationDataset)
   log.info(`Begin rollback "${date}".`);
   Object.keys(files).forEach(async key => {
     const [table] = await dataset.table(key).get();
-    files[key].forEach(filename => bigquery.dataset(destinationDataset, {
-      location: dataset.metadata.location,
-      projectId
-    }).table(key).load(bucket.file(`${date}/${filename}`), {
-      sourceFormat: "NEWLINE_DELIMITED_JSON",
-      schema: table.metadata.schema,
-      timePartitioning: table.metadata.timePartitioning,
-      location: table.metadata.location,
-      tableConstraints: table.metadata.tableConstraints,
-      partitionDefinition: table.metadata.partitionDefinition
-    }));
+    files[key].forEach(async filename => {
+      const destination = await getTable(destinationDataset, key, {
+        projectId,
+        location: dataset.metadata.location
+      });
+      await destination.load(bucket.file(`${date}/${filename}`), {
+        sourceFormat: "NEWLINE_DELIMITED_JSON",
+        schema: table.metadata.schema,
+        timePartitioning: table.metadata.timePartitioning,
+        location: table.metadata.location,
+        tableConstraints: table.metadata.tableConstraints,
+        partitionDefinition: table.metadata.partitionDefinition
+      });
+    });
   });
 };

package/dist/contents/storage.js

@@ -120,12 +120,13 @@ exports.getSafeFilenameFromBucket = ({
  * @param {string} output Path to save files.
  * @param {string} bucket Bucket name.
  * @param {string} dateFormat Date format string.
+ * @param {string} projectId Optional project ID.
  */
-exports.implode = async (date, output, bucket, dateFormat = "YYYY-MM-DD") => {
+exports.implode = async (date, output, bucket, dateFormat = "YYYY-MM-DD", projectId = null) => {
   const folder = format(parse(date, dateFormat), dateFormat);
   const destination = path.join(output, ".tmp");
   const temporary = path.join(destination, folder);
-  await downloadFolder(folder, bucket, destination).then(() => log.info(`Downloaded "gs://${bucket}/${folder}" to "${destination}".`)).catch(log.error);
+  await downloadFolder(folder, bucket, destination, projectId).then(() => log.info(`Downloaded "gs://${bucket}/${folder}" to "${destination}".`)).catch(log.error);
   const files = groupJson(fs.readdirSync(temporary));
   Object.keys(files).forEach(table => {
     const json = [];
@@ -155,10 +156,11 @@ exports.implode = async (date, output, bucket, dateFormat = "YYYY-MM-DD") => {
  *
  * @param {string} bucket Bucket name.
  * @param {number} days Number of days to keep.
+ * @param {string} projectId Optional project ID.
  * @returns {boolean} A value indicating whehter the job has been done.
  */
-exports.erase = async (bucket, days = process.env.BACKUP_KEEP_DAYS || 3) => {
-  const b = getBucket(bucket);
+exports.erase = async (bucket, days = process.env.BACKUP_KEEP_DAYS || 3, projectId = null) => {
+  const b = getBucket(bucket, projectId);
   const [files] = await b.getFiles();
   const result = {};
   files.forEach(({
@@ -169,9 +171,9 @@ exports.erase = async (bucket, days = process.env.BACKUP_KEEP_DAYS || 3) => {
   });
   const folders = Object.keys(result).sort();
   const blacklist = folders.slice(0, folders.length - parseInt(days));
-  blacklist.forEach(async prefix => await b.deleteFiles({
+  await Promise.all(blacklist.map(async prefix => await b.deleteFiles({
     prefix
-  }));
+  })));
   log.success("Backup bucket erased.");
   return true;
 };

package/dist/scripts/backup.js

@@ -41,7 +41,7 @@ exports.scriptBackup = backup;
  * @param {string} dateFormat Date format string.
  * @returns {boolean} A value indicating whehter the job has been done.
  */
-exports.scriptBackupErase = async (id, bucket, projectId, days, dateFormat = "YYYY-MM-DD") => await backup(id, bucket, projectId, dateFormat).then(async () => await erase(bucket, days));
+exports.scriptBackupErase = async (id, bucket, projectId, days, dateFormat = "YYYY-MM-DD") => await backup(id, bucket, projectId, dateFormat).then(async () => await erase(bucket, days, projectId));
 
 /**
  * Rollback data using backup.

package/dist/scripts/comparison.js

@@ -28,14 +28,15 @@ const {
  *
  * @param {string} bucket Bucket name.
  * @param {string} date Date to compare against.
+ * @param {string} projectId Google Cloud project ID.
  * @returns {Array} Array of files that are in the file but not in the bucket.
  */
-exports.scriptComparison = async (bucket, date = null) => {
+exports.scriptComparison = async (bucket, date = null, projectId = null) => {
   if (bucket) {
     const day = date ? stringToDateUTC(date) : new Date();
     const filename = mergeFilename(day, "filelist", "txt");
-    const content = await txtContentToArray(filename, bucket);
-    const files = await listFiles(bucket);
+    const content = await txtContentToArray(filename, bucket, projectId);
+    const files = await listFiles(bucket, null, projectId);
     return content.filter(file => !files.includes(file));
   }
 };

package/dist/scripts/storage.js

@@ -24,13 +24,16 @@ const {
  * @param {string} output Path to save files.
  * @param {string} bucket Bucket name.
  * @param {string} dateFormat Date format string.
+ * @param {string} projectId Optional project ID.
  */
 exports.scriptImplode = implode;
+
 /**
  * Erases the bucket.
  *
  * @param {string} bucket Bucket name.
  * @param {number} days Number of days to keep.
+ * @param {string} projectId Optional project ID.
  * @returns {boolean} A value indicating whehter the job has been done.
  */
 exports.scriptErase = erase;

package/dist/scripts/unlistened.js

@@ -29,19 +29,19 @@ const {
  *
  * @param {string} bucket Bucket name.
  * @param {string} folder Path of the local folder to scan for schemas.
- * @returns {void}
+ * @param {string} projectId Optional project ID.
  */
-exports.scriptUnlistened = async (bucket, folder) => {
+exports.scriptUnlistened = async (bucket, folder, projectId = null) => {
   if (bucket) {
-    const storage = getStorage();
+    const storage = getStorage(projectId);
     const [files] = await getFiles(storage, bucket);
     if (files) {
-      files.forEach(file => {
+      files.forEach(async file => {
         const {
           name
         } = splitName(file.id);
         if (!hasSchema(name, folder)) {
-          deleteFile(file);
+          await deleteFile(file);
         }
       });
     }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@pittica/google-business-intelligence-helpers",
   "private": false,
-  "version": "1.15.1",
+  "version": "1.15.3",
   "description": "Helpers for business intelligence for Google Cloud.",
   "main": "dist/index.js",
   "scripts": {
@@ -36,8 +36,8 @@
     "node": ">=24.11.0"
   },
   "dependencies": {
-    "@pittica/google-bigquery-helpers": "^1.6.1",
-    "@pittica/google-cloud-storage-helpers": "^1.5.0",
+    "@pittica/google-bigquery-helpers": "^1.7.0",
+    "@pittica/google-cloud-storage-helpers": "^1.6.0",
     "@pittica/logger-helpers": "^1.2.2",
     "date-and-time": "^4.7.0"
   },