@pittica/google-business-intelligence-helpers 1.14.0 → 1.15.0

package/dist/contents/backup.js

@@ -0,0 +1,111 @@
+// Copyright 2024-2026 Pittica S.r.l.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const {
+  getBucket
+} = require("@pittica/google-cloud-storage-helpers");
+const {
+  getDataset,
+  jobDone
+} = require("@pittica/google-bigquery-helpers");
+const log = require("@pittica/logger-helpers");
+const {
+  format
+} = require("date-and-time");
+
+/**
+ * Creates a backup for a BigQuery dataset to Google Cloud Storage bucket.
+ *
+ * @param {string} id BigQuery dataset ID.
+ * @param {string} bucket Bucket name.
+ * @param {string} projectId Google Cloud project ID.
+ * @param {string} dateFormat Date format string.
+ * @returns {boolean} A value indicating whehter the job has been done.
+ */
+exports.backup = async (id, bucket, projectId, dateFormat = "YYYY-MM-DD") => {
+  const dataset = await getDataset(id, {
+    projectId
+  });
+  const [tables] = await dataset.getTables();
+  const date = format(new Date(), dateFormat);
+  return await exports.extract(bucket, date, tables, 0);
+};
+
+/**
+ * Extracts a table from BigQuery to Google Cloud Storage.
+ *
+ * @param {string} bucket Bucket name.
+ * @param {string} date Date in format YYYY-MM-DD.
+ * @param {Array} tables An array of BigQuery tables.
+ * @param {number} index Array index.
+ * @returns {boolean} A value indicating whehter the job has been done.
+ */
+exports.extract = async (bucket, date, tables, index) => {
+  if (index < tables.length) {
+    const table = tables[index];
+    return await table.extract(getBucket(bucket).file(`${date}/${table.id}-*.json`), {
+      location: "us-central1",
+      format: "json",
+      gzip: false
+    }).then(async response => {
+      if (jobDone(response)) {
+        const [job] = response;
+        log.success(`Job backup "${job.id}" for table "${table.id}" done.`);
+        return await exports.extract(date, tables, index + 1);
+      } else {
+        return false;
+      }
+    }).catch(errors => {
+      log.error(errors);
+      return false;
+    });
+  } else {
+    return true;
+  }
+};
+
+/**
+ * Rollback data using backup.
+ *
+ * @param {string} date Date to download in format "YYYY-MM-DD".
+ * @param {string} id BigQuery dataset ID.
+ * @param {string} projectId Google Cloud project ID.
+ * @param {string} sourceBucket Bucket name.
+ * @param {string} destinationDataset BigQuery dataset ID.
+ */
+exports.rollback = async (date, id, projectId, sourceBucket, destinationDataset) => {
+  const dataset = await getDataset(id, {
+    projectId
+  });
+  const bucket = getBucket(sourceBucket);
+  const [response] = await bucket.getFiles({
+    prefix: `${date}/`
+  });
+  const files = groupJson(response.map(file => file.name.substring(11)));
+  log.info(`Begin rollback "${date}".`);
+  Object.keys(files).forEach(async key => {
+    const [table] = await dataset.table(key).get();
+    files[key].forEach(filename => bigquery.dataset(destinationDataset, {
+      location: dataset.metadata.location,
+      projectId
+    }).table(key).load(bucket.file(`${date}/${filename}`), {
+      sourceFormat: "NEWLINE_DELIMITED_JSON",
+      schema: table.metadata.schema,
+      timePartitioning: table.metadata.timePartitioning,
+      location: table.metadata.location,
+      tableConstraints: table.metadata.tableConstraints,
+      partitionDefinition: table.metadata.partitionDefinition
+    }));
+  });
+};

package/dist/contents/schema.js

@@ -14,6 +14,9 @@
 
 const fs = require("fs");
 const path = require("path");
+const {
+  getTables
+} = require("@pittica/google-bigquery-helpers");
 
 /**
  * Maps and imports the contents of the given folder.
@@ -38,7 +41,7 @@ exports.getSchemas = folder => {
  * @returns {object} The content related to the given name.
  */
 exports.getSchema = (name, folder) => {
-  const contents = this.getSchemas(folder);
+  const contents = exports.getSchemas(folder);
   return Object.keys(contents).includes(name) ? contents[name] : null;
 };
 
@@ -57,4 +60,39 @@ exports.getSchemaKeys = folder => fs.readdirSync(folder).map(file => file.split(
  * @param {string} folder Path of the folder to scan.
  * @returns {boolean} A value indicating whether the given name has an associated content.
  */
-exports.hasSchema = (name, folder) => this.getSchemaKeys(folder).includes(name);
+exports.hasSchema = (name, folder) => exports.getSchemaKeys(folder).includes(name);
+
+/**
+ * Downloads the schemas of a dataset.
+ *
+ * @param {string} output Path to save files.
+ * @param {string} id The dataset name from which to download schemas.
+ * @param {string} projectId The project ID to which the dataset belongs.
+ */
+exports.downloadSchemas = async (output, id, projectId = null) => {
+  const tables = await getTables(id, {
+    projectId
+  });
+  tables.forEach(async ({
+    dataset,
+    metadata: {
+      tableReference: {
+        tableId
+      }
+    }
+  }) => {
+    const [table] = await dataset.table(tableId).get();
+    const {
+      schema,
+      timePartitioning,
+      tableConstraints,
+      partitionDefinition
+    } = table.metadata;
+    fs.writeFileSync(path.join(output, `${tableId}.json`), JSON.stringify({
+      schema,
+      timePartitioning,
+      tableConstraints,
+      partitionDefinition
+    }), "utf8");
+  });
+};

package/dist/contents/storage.js

@@ -13,11 +13,18 @@
 // limitations under the License.
 
 const {
-  getFiles
+  getFiles,
+  downloadFolder,
+  groupJson,
+  getBucket
 } = require("@pittica/google-cloud-storage-helpers");
+const log = require("@pittica/logger-helpers");
 const {
-  format
+  format,
+  parse
 } = require("date-and-time");
+const path = require("path");
+const fs = require("fs");
 const {
   getSchemaKeys
 } = require("./schema");
@@ -63,7 +70,7 @@ exports.mapStorageResponse = (response, schemas = "") => {
  */
 exports.extractStorageResponse = (response, schemas = "", dateFormat = "YYYY-MM-DD") => {
   const days = {};
-  const files = this.mapStorageResponse(response, schemas);
+  const files = exports.mapStorageResponse(response, schemas);
   files.forEach(file => {
     const date = format(file.date, dateFormat);
     if (typeof days[date] === "undefined") {
@@ -104,4 +111,67 @@ exports.getSafeFilename = (storage, bucket, filename, folder = "") => getFiles(s
 exports.getSafeFilenameFromBucket = ({
   storage,
   name
-}, filename, folder = "") => this.getSafeFilename(storage, name, filename, folder);
+}, filename, folder = "") => exports.getSafeFilename(storage, name, filename, folder);
+
+/**
+ * Merges the files from a date and saves them in a local folder.
+ *
+ * @param {string} date Date to download in given format.
+ * @param {string} output Path to save files.
+ * @param {string} bucket Bucket name.
+ * @param {string} dateFormat Date format string.
+ */
+exports.implode = async (date, output, bucket, dateFormat = "YYYY-MM-DD") => {
+  const folder = format(parse(date, dateFormat), dateFormat);
+  const destination = path.join(output, ".tmp");
+  const temporary = path.join(destination, folder);
+  await downloadFolder(folder, bucket, destination).then(() => log.info(`Downloaded "gs://${bucket}/${folder}" to "${destination}".`)).catch(log.error);
+  const files = groupJson(fs.readdirSync(temporary));
+  Object.keys(files).forEach(table => {
+    const json = [];
+    files[table].forEach(current => {
+      const content = [];
+      fs.readFileSync(path.join(temporary, current), {
+        encoding: "utf8"
+      }).split("\n").forEach(line => {
+        if (line) {
+          const result = JSON.parse(`[${line}]`);
+          return content.push(result[0]);
+        }
+      });
+      json.push(content);
+    });
+    fs.writeFileSync(path.join(output, `${table}.json`), JSON.stringify(json.flat(1)), "utf8");
+  });
+  fs.rmSync(destination, {
+    recursive: true,
+    force: true
+  });
+  log.success(`JSON files created in "${output}".`);
+};
+
+/**
+ * Erases the bucket.
+ *
+ * @param {string} bucket Bucket name.
+ * @param {number} days Number of days to keep.
+ * @returns {boolean} A value indicating whehter the job has been done.
+ */
+exports.erase = async (bucket, days = process.env.BACKUP_KEEP_DAYS || 3) => {
+  const b = getBucket(bucket);
+  const [files] = await b.getFiles();
+  const result = {};
+  files.forEach(({
+    id
+  }) => {
+    const split = id.split(/^(\d{4})\-(\d{2})\-(\d{2})/gis);
+    result[`${split[1]}-${split[2]}-${split[3]}`] = null;
+  });
+  const folders = Object.keys(result).sort();
+  const blacklist = folders.slice(0, folders.length - parseInt(days));
+  blacklist.forEach(async prefix => await b.deleteFiles({
+    prefix
+  }));
+  log.success("Backup bucket erased.");
+  return true;
+};

package/dist/index.js

@@ -12,11 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+const {
+  backup,
+  rollback
+} = require("./contents/backup");
 const {
   getSchemas,
   getSchema,
   getSchemaKeys,
-  hasSchema
+  hasSchema,
+  downloadSchemas
 } = require("./contents/schema");
 const {
   getSqlFilePath
@@ -25,7 +30,9 @@ const {
   mapStorageResponse,
   extractStorageResponse,
   getSafeFilename,
-  getSafeFilenameFromBucket
+  getSafeFilenameFromBucket,
+  implode,
+  erase
 } = require("./contents/storage");
 const {
   getTemporaryTableName,
@@ -63,15 +70,32 @@ const {
 const {
   scriptUnlistened
 } = require("./scripts/unlistened");
+const {
+  scriptDownloadSchemas
+} = require("./scripts/schema");
+const {
+  scriptImplode,
+  scriptErase
+} = require("./scripts/storage");
+const {
+  scriptBackup,
+  scriptBackupErase,
+  scriptRollback
+} = require("./scripts/backup");
+exports.backup = backup;
+exports.rollback = rollback;
 exports.getSchemas = getSchemas;
 exports.getSchema = getSchema;
 exports.getSchemaKeys = getSchemaKeys;
 exports.hasSchema = hasSchema;
+exports.downloadSchemas = downloadSchemas;
 exports.getSqlFilePath = getSqlFilePath;
 exports.mapStorageResponse = mapStorageResponse;
 exports.extractStorageResponse = extractStorageResponse;
 exports.getSafeFilename = getSafeFilename;
 exports.getSafeFilenameFromBucket = getSafeFilenameFromBucket;
+exports.implode = implode;
+exports.erase = erase;
 exports.getTemporaryTableName = getTemporaryTableName;
 exports.getTemporaryTableSuffix = getTemporaryTableSuffix;
 exports.partsToDate = partsToDate;
@@ -94,5 +118,11 @@ exports.splitName = splitName;
 exports.splitIdByKeys = splitIdByKeys;
 exports.scripts = {
   comparison: scriptComparison,
-  unlistened: scriptUnlistened
+  unlistened: scriptUnlistened,
+  downloadSchemas: scriptDownloadSchemas,
+  implode: scriptImplode,
+  erase: scriptErase,
+  backup: scriptBackup,
+  backupErase: scriptBackupErase,
+  rollback: scriptRollback
 };

package/dist/naming/dataset.js

@@ -35,4 +35,4 @@ exports.getTemporaryTableName = ({
   date,
   name,
   version
-}, prefix) => prefix + name + "-" + version + "-" + this.getTemporaryTableSuffix(date);
+}, prefix) => prefix + name + "-" + version + "-" + exports.getTemporaryTableSuffix(date);

package/dist/naming/date.js

@@ -41,7 +41,7 @@ exports.partsToDateUTC = parts => new Date(Date.UTC(parseInt(parts[1]), parseInt
 exports.stringToDate = date => {
   const split = date.split(/^(\d{4})\-(\d{2})\-(\d{2})$/gis);
   if (split.length === 5) {
-    return this.partsToDate(split);
+    return exports.partsToDate(split);
   }
   return null;
 };
@@ -55,7 +55,7 @@ exports.stringToDate = date => {
 exports.stringToDateUTC = date => {
   const split = date.split(/^(\d{4})\-(\d{2})\-(\d{2})$/gis);
   if (split.length === 5) {
-    return this.partsToDateUTC(split);
+    return exports.partsToDateUTC(split);
   }
   return null;
 };
@@ -66,7 +66,7 @@ exports.stringToDateUTC = date => {
  * @param {string} dateFormat Format string.
  * @returns {string} The current date in the given format.
  */
-exports.getNow = (dateFormat = "YYYY-MM-DD") => this.formatDate(new Date(), dateFormat);
+exports.getNow = (dateFormat = "YYYY-MM-DD") => exports.formatDate(new Date(), dateFormat);
 
 /**
  * Gets the given date in the given format.
@@ -98,7 +98,7 @@ exports.getDatesUntil = (until = new Date(), dateFormat = "YYYY-MM-DD") => {
   const year = until.getUTCFullYear();
   let current = new Date(Date.UTC(year, 0, 1));
   while (current <= until && current.getUTCFullYear() === year) {
-    dates.push(this.formatDateUTC(current, dateFormat));
+    dates.push(exports.formatDateUTC(current, dateFormat));
     current.setUTCDate(current.getUTCDate() + 1);
   }
   return dates;
@@ -113,10 +113,10 @@ exports.getDatesUntil = (until = new Date(), dateFormat = "YYYY-MM-DD") => {
  */
 exports.getFormattedDate = (date = null, dateFormat = "YYYY-MM-DD") => {
   if (date != null) {
-    return this.formatDate(typeof date === "string" ? this.stringToDate(date) : date, dateFormat);
+    return exports.formatDate(typeof date === "string" ? exports.stringToDateUTC(date) : date, dateFormat);
   } else {
-    const now = this.stringToDate(this.getNow());
-    now.setDate(now.getDate() - 1);
-    return this.formatDate(now, dateFormat);
+    const now = exports.stringToDateUTC(exports.getNow());
+    now.setDate(now.getDate());
+    return exports.formatDate(now, dateFormat);
   }
 };

package/dist/naming/file.js

@@ -85,4 +85,4 @@ exports.mergeFiledata = ({
   name,
   extension,
   version
-}, increment = false) => this.mergeFilename(date, name, extension, increment ? typeof version !== "undefined" ? version + 1 : 1 : version);
+}, increment = false) => exports.mergeFilename(date, name, extension, increment ? typeof version !== "undefined" ? version + 1 : 1 : version);

package/dist/scripts/backup.js

@@ -0,0 +1,51 @@
+// Copyright 2024-2026 Pittica S.r.l.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const {
+  backup,
+  rollback
+} = require("../contents/backup");
+const {
+  erase
+} = require("../contents/storage");
+
+/**
+ * Creates a backup for a BigQuery dataset to Google Cloud Storage bucket.
+ *
+ * @param {string} id BigQuery dataset ID.
+ * @param {string} bucket Bucket name.
+ * @param {string} projectId Google Cloud project ID.
+ * @param {string} dateFormat Date format string.
+ * @returns {boolean} A value indicating whehter the job has been done.
+ */
+exports.scriptBackup = backup;
+
+/**
+ * Creates a backup and erases the bucket.
+ *
+ * @param {number} days Number of days to keep.
+ * @returns {boolean} A value indicating whehter the job has been done.
+ */
+exports.scriptBackupErase = async days => await backup().then(async () => await erase(days));
+
+/**
+ * Rollback data using backup.
+ *
+ * @param {string} date Date to download in format "YYYY-MM-DD".
+ * @param {string} id BigQuery dataset ID.
+ * @param {string} projectId Google Cloud project ID.
+ * @param {string} sourceBucket Bucket name.
+ * @param {string} destinationDataset BigQuery dataset ID.
+ */
+exports.scriptRollback = rollback;

package/dist/scripts/schema.js

@@ -0,0 +1,26 @@
+// Copyright 2024-2026 Pittica S.r.l.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const {
+  downloadSchemas
+} = require("../contents/schema");
+
+/**
+ * Downloads the schemas of a dataset.
+ *
+ * @param {string} output Path to save files.
+ * @param {string} id The dataset name from which to download schemas.
+ * @param {string} projectId The project ID to which the dataset belongs.
+ */
+exports.scriptDownloadSchemas = downloadSchemas;

package/dist/scripts/storage.js

@@ -0,0 +1,36 @@
+// Copyright 2024-2026 Pittica S.r.l.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const {
+  implode,
+  erase
+} = require("../contents/storage");
+
+/**
+ * Merges the files from a date and saves them in a local folder.
+ *
+ * @param {string} date Date to download in given format.
+ * @param {string} output Path to save files.
+ * @param {string} bucket Bucket name.
+ * @param {string} dateFormat Date format string.
+ */
+exports.scriptImplode = implode;
+/**
+ * Erases the bucket.
+ *
+ * @param {string} bucket Bucket name.
+ * @param {number} days Number of days to keep.
+ * @returns {boolean} A value indicating whehter the job has been done.
+ */
+exports.scriptErase = erase;

package/package.json

@@ -1,14 +1,20 @@
 {
   "name": "@pittica/google-business-intelligence-helpers",
   "private": false,
-  "version": "1.14.0",
+  "version": "1.15.0",
   "description": "Helpers for business intelligence for Google Cloud.",
   "main": "dist/index.js",
   "scripts": {
     "build": "rimraf ./dist && mkdirp ./dist && babel src --out-dir dist --copy-files",
     "test": "echo \"Error: no test specified\" && exit 1",
     "unlistened": "npm run build && run-func ./dist/scripts/unlistened.js scriptUnlistened",
-    "comparison": "npm run build && run-func ./dist/scripts/comparison.js scriptComparison"
+    "comparison": "npm run build && run-func ./dist/scripts/comparison.js scriptComparison",
+    "download-schema": "npm run build && run-func ./dist/scripts/schema.js scriptDownloadSchemas",
+    "implode": "npm run build && run-func ./dist/scripts/storage.js scriptImplode",
+    "erase": "npm run build && run-func ./dist/scripts/storage.js scriptErase",
+    "backup": "npm run build && run-func ./dist/scripts/backup.js scriptBackup",
+    "backup-erase": "npm run build && run-func ./dist/scripts/backup.js scriptBackupErase",
+    "rollback": "npm run build && run-func ./dist/scripts/backup.js scriptRollback"
   },
   "repository": {
     "type": "git",
@@ -30,8 +36,10 @@
     "node": ">=24.11.0"
   },
   "dependencies": {
+    "@pittica/google-bigquery-helpers": "^1.6.1",
     "@pittica/google-cloud-storage-helpers": "^1.5.0",
-    "date-and-time": "^4.6.0"
+    "@pittica/logger-helpers": "^1.2.2",
+    "date-and-time": "^4.7.0"
   },
   "devDependencies": {
     "@babel/cli": "^8.0.6",