backend-manager 2.5.13 → 2.5.15

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "backend-manager",
-  "version": "2.5.13",
+  "version": "2.5.15",
   "description": "Quick tools for developing Firebase functions",
   "main": "src/manager/index.js",
   "bin": {
@@ -65,75 +65,5 @@
     "bin/",
     "src/",
     "templates/"
-  ],
-  "allDependencies": {
-    "@firebase/rules-unit-testing": "^2.0.4",
-    "@google-cloud/storage": "^5.20.5",
-    "@sendgrid/mail": "^7.7.0",
-    "@sentry/node": "^6.19.7",
-    "backend-assistant": "^0.0.74",
-    "busboy": "^1.6.0",
-    "chalk": "^4.1.2",
-    "cors": "^2.8.5",
-    "dotenv": "^16.0.2",
-    "firebase-admin": "^9.12.0",
-    "firebase-functions": "^3.24.0",
-    "fs-jetpack": "^4.3.1",
-    "hcaptcha": "^0.1.1",
-    "inquirer": "^8.2.4",
-    "json5": "^2.2.1",
-    "jwt-decode": "^3.1.2",
-    "lodash": "^4.17.21",
-    "lowdb": "^1.0.0",
-    "mailchimp-api-v3": "^1.15.0",
-    "moment": "^2.29.4",
-    "node-fetch": "^2.6.7",
-    "node-powertools": "^0.0.21",
-    "npm-api": "^1.0.1",
-    "paypal-server-api": "^0.0.8",
-    "pushid": "^1.0.0",
-    "semver": "^7.3.7",
-    "shortid": "^2.2.16",
-    "uid-generator": "^2.0.0",
-    "ultimate-jekyll-poster": "^0.0.15",
-    "universal-analytics": "^0.5.3",
-    "uuid": "^8.3.2",
-    "wonderful-fetch": "^0.0.14",
-    "yargs": "^17.5.1"
-  },
-  "allDependenciesKeys": [
-    "@firebase/rules-unit-testing",
-    "@google-cloud/storage",
-    "@sendgrid/mail",
-    "@sentry/node",
-    "backend-assistant",
-    "busboy",
-    "chalk",
-    "cors",
-    "dotenv",
-    "firebase-admin",
-    "firebase-functions",
-    "fs-jetpack",
-    "hcaptcha",
-    "inquirer",
-    "json5",
-    "jwt-decode",
-    "lodash",
-    "lowdb",
-    "mailchimp-api-v3",
-    "moment",
-    "node-fetch",
-    "node-powertools",
-    "npm-api",
-    "paypal-server-api",
-    "pushid",
-    "semver",
-    "shortid",
-    "uid-generator",
-    "ultimate-jekyll-poster",
-    "universal-analytics",
-    "uuid",
-    "wonderful-fetch",
-    "yargs"
   ]
-}
+}

package/src/cli/cli.js

@@ -1322,16 +1322,26 @@ async function cmd_iamImportExport(self) {
 async function cmd_setStorageLifecycle(self) {
   return new Promise(function(resolve, reject) {
     const command = `gsutil lifecycle set {config} gs://{bucket}`
-      .replace(/{config}/ig, path.resolve(`${__dirname}/../../templates/storage-lifecycle-config.json`))
+      .replace(/{config}/ig, path.resolve(`${__dirname}/../../templates/storage-lifecycle-config-1-day.json`))
       .replace(/{bucket}/ig, `us.artifacts.${self.projectName}.appspot.com`)
+    const command2 = `gsutil lifecycle set {config} gs://{bucket}`
+      .replace(/{config}/ig, path.resolve(`${__dirname}/../../templates/storage-lifecycle-config-30-days.json`))
+      .replace(/{bucket}/ig, `bm-backup-firestore-${self.projectName}`)      
 
-    let cmd = exec(command, function (error, stdout, stderr) {
+    exec(command, function (error, stdout, stderr) {
       if (error) {
         console.log(chalk.red(`Failed to run command`, error));
         reject(error);
       } else {
-        // console.log(chalk.green(`Added lifecycle`));
-        resolve(stdout);
+        exec(command2, function (error, stdout, stderr) {
+          if (error) {
+            console.log(chalk.red(`Failed to run command`, error));
+            reject(error);
+          } else {
+            // console.log(chalk.green(`Added lifecycle`));
+            resolve(stdout);
+          }
+        })
       }
     });
   });

package/src/manager/functions/core/actions/api/admin/backup.js

@@ -36,7 +36,7 @@ Module.prototype.main = function () {
       const bucketAddress = `gs://${bucketName}`;
 
       await self.createBucket(bucketName, resourceZone);
-      await self.deleteOldFiles(bucketName, resourceZone);
+      // await self.deleteOldFiles(bucketName, resourceZone);
 
       client.exportDocuments({
         name: databaseName,
@@ -127,7 +127,86 @@ Module.prototype.createBucket = function (bucketName, resourceZone) {
 };
 
 // https://github.com/zsolt-dev/auto-delete-gcp-storage-backups/blob/master/index.js
-Module.prototype.deleteOldFiles = function (bucketName, resourceZone) {
+Module.prototype.__RETRY_deleteOldFiles = function (bucketName, resourceZone) {
+  const self = this;
+  const Manager = self.Manager;
+  const Api = self.Api;
+  const assistant = self.assistant;
+  const payload = self.payload;
+
+  return new Promise(async function(resolve, reject) {
+    const now = moment();
+    const deletionRegex = payload.data.payload.deletionRegex;
+
+    // Helpers
+    const getFileObjectWithMetaData = async (bucketName, fileName) => {
+      const [metaData] = await storage.bucket(bucketName).file(fileName).getMetadata();
+      return ({ fileName, created: metaData.timeCreated });
+    };
+
+    const deleteFileFromBucket = async (bucketName, fileName) => {
+      assistant.log(`Deleting item: ${fileName}...`, );
+      return await storage.bucket(bucketName).file(fileName).delete();
+    };
+
+    // Main
+    // get the file names as an array
+    let [allFiles] = await storage.bucket(bucketName).getFiles();
+    let deletePromises = [];
+    let foldersToDelete = [];
+    allFiles = allFiles.map(file => file.name);
+    // console.log(`all files: ${allFiles.join(', ')}`);
+
+    allFiles.forEach((filePath, i) => {
+      const fileName = filePath.split('/')[0];
+      const date = moment(fileName.split('T')[0]);
+      const day = date.date();
+      const month = date.month();
+      const age = now.diff(date, 'days', false);      
+      
+      if (age >= 30) {
+        if (day === 1) { return }
+        deletePromises.push(deleteFileFromBucket(bucketName, backup.fileName))
+        assistant.log(`Preparing to delete ${filePath}: date=${date.format('MMM Do, YYYY')}, day=${day}, month=${month}, age=${age}`, );
+        // if (!foldersToDelete.includes(fileName)) {
+        //   assistant.log(`Preparing to delete ${fileName}: date=${date.format('MMM Do, YYYY')}, day=${day}, month=${month}, age=${age}`, );
+        //   foldersToDelete = foldersToDelete.concat(fileName);
+        //   deletePromises.push(deleteFileFromBucket(bucketName, fileName))
+        // }
+      }
+    })
+
+    console.log('---deletePromises.length', deletePromises.length);
+
+    return
+
+    // transform to array of objects with creation timestamp { fileName: abc, created: xyz }
+    allFiles = allFiles.map(fileName => getFileObjectWithMetaData(bucketName, fileName));
+    allFiles = await Promise.all(allFiles);
+
+    allFiles.forEach((backup, i) => {
+      const date = moment(backup.created);
+      const day = date.date();
+      const month = date.month();
+      const age = now.diff(date, 'days', false);
+
+      assistant.log(`Sorting item ${i}: date=${date.format('MMM Do, YYYY')}, day=${day}, month=${month}, age=${age}`, );
+
+      if (age >= 31) {
+        if (day === 1) { return }
+        deletePromises.push(deleteFileFromBucket(bucketName, backup.fileName))
+      } else if ((deletionRegex && backup.fileName.match(deletionRegex))) {
+        deletePromises.push(deleteFileFromBucket(bucketName, backup.fileName))
+      }
+    })
+
+    await Promise.all(deletePromises);
+
+    return resolve();
+  });
+};
+
+Module.prototype.__deleteOldFiles = function (bucketName, resourceZone) {
   const self = this;
   const Manager = self.Manager;
   const Api = self.Api;

package/src/manager/functions/core/cron/daily.js

@@ -24,6 +24,7 @@ Module.prototype.main = function() {
   return new Promise(async function(resolve, reject) {
     fetch(`${Manager.project.functionsUrl}/bm_api`, {
       method: 'post',
+      response: 'json',
       body: {
         backendManagerKey: Manager.config.backend_manager.key,
         command: 'admin:backup',

package/src/manager/index.js

@@ -48,6 +48,7 @@ Manager.prototype.init = function (exporter, options) {
   options.serviceAccountPath = typeof options.serviceAccountPath === 'undefined' ? 'service-account.json' : options.serviceAccountPath;
   options.backendManagerConfigPath = typeof options.backendManagerConfigPath === 'undefined' ? 'backend-manager-config.json' : options.backendManagerConfigPath;
   options.fetchStats = typeof options.fetchStats === 'undefined' ? true : options.fetchStats;
+  options.checkNodeVersion = typeof options.checkNodeVersion === 'undefined' ? true : options.checkNodeVersion;
   options.uniqueAppName = options.uniqueAppName || undefined;
   options.assistant = options.assistant || {};
   // options.assistant.optionsLogString = options.assistant.optionsLogString || undefined;
@@ -75,7 +76,7 @@ Manager.prototype.init = function (exporter, options) {
   self.project.resourceZone = options.resourceZone;
   self.project.serviceAccountPath = path.resolve(self.cwd, options.serviceAccountPath)
   self.project.backendManagerConfigPath = path.resolve(self.cwd, options.backendManagerConfigPath)
-  
+
   self.package = resolveProjectPackage();
   self.config = merge(
     requireJSON5(self.project.backendManagerConfigPath),
@@ -117,7 +118,9 @@ Manager.prototype.init = function (exporter, options) {
     // Reject if package.json does not exist
     if (semverUsing !== semverRequired) {
       self.assistant.error(new Error(`Node.js version mismatch: using ${semverUsing} but asked for ${semverRequired}`), {environment: 'production'});
-      return process.exit(1);
+      if (options.checkNodeVersion) {
+        return process.exit(1);
+      }
     }
   }  
 
@@ -386,7 +389,7 @@ Manager.prototype.init = function (exporter, options) {
     // Cron
     exporter.bm_cronDaily =
     self.libraries.functions
-    .runWith({ memory: '256MB', timeoutSeconds: 60 })
+    .runWith({ memory: '256MB', timeoutSeconds: 120 })
     .pubsub.schedule('every 24 hours')
     .onRun(async (context) => {
       return self._process((new (require(`${core}/cron/daily.js`))()).init(self, { context: context, }))

package/templates/storage-lifecycle-config-30-days.json

@@ -0,0 +1,9 @@
+{
+  "rule":
+  [
+    {
+      "action": {"type": "Delete"},
+      "condition": {"age": 30}
+    }
+  ]
+}