backend-manager 2.5.13 → 2.5.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "backend-manager",
-  "version": "2.5.13",
+  "version": "2.5.14",
   "description": "Quick tools for developing Firebase functions",
   "main": "src/manager/index.js",
   "bin": {
@@ -136,4 +136,4 @@
     "wonderful-fetch",
     "yargs"
   ]
-}
+}

package/src/cli/cli.js

@@ -1322,16 +1322,26 @@ async function cmd_iamImportExport(self) {
 async function cmd_setStorageLifecycle(self) {
   return new Promise(function(resolve, reject) {
     const command = `gsutil lifecycle set {config} gs://{bucket}`
-      .replace(/{config}/ig, path.resolve(`${__dirname}/../../templates/storage-lifecycle-config.json`))
+      .replace(/{config}/ig, path.resolve(`${__dirname}/../../templates/storage-lifecycle-config-1-day.json`))
       .replace(/{bucket}/ig, `us.artifacts.${self.projectName}.appspot.com`)
+    const command2 = `gsutil lifecycle set {config} gs://{bucket}`
+      .replace(/{config}/ig, path.resolve(`${__dirname}/../../templates/storage-lifecycle-config-30-days.json`))
+      .replace(/{bucket}/ig, `bm-backup-firestore-${self.projectName}`)      
 
-    let cmd = exec(command, function (error, stdout, stderr) {
+    exec(command, function (error, stdout, stderr) {
       if (error) {
         console.log(chalk.red(`Failed to run command`, error));
         reject(error);
       } else {
-        // console.log(chalk.green(`Added lifecycle`));
-        resolve(stdout);
+        exec(command2, function (error, stdout, stderr) {
+          if (error) {
+            console.log(chalk.red(`Failed to run command`, error));
+            reject(error);
+          } else {
+            // console.log(chalk.green(`Added lifecycle`));
+            resolve(stdout);
+          }
+        })
       }
     });
   });

package/src/manager/functions/core/actions/api/admin/backup.js

@@ -36,7 +36,7 @@ Module.prototype.main = function () {
       const bucketAddress = `gs://${bucketName}`;
 
       await self.createBucket(bucketName, resourceZone);
-      await self.deleteOldFiles(bucketName, resourceZone);
+      // await self.deleteOldFiles(bucketName, resourceZone);
 
       client.exportDocuments({
         name: databaseName,
@@ -127,7 +127,86 @@ Module.prototype.createBucket = function (bucketName, resourceZone) {
 };
 
 // https://github.com/zsolt-dev/auto-delete-gcp-storage-backups/blob/master/index.js
-Module.prototype.deleteOldFiles = function (bucketName, resourceZone) {
+Module.prototype.__RETRY_deleteOldFiles = function (bucketName, resourceZone) {
+  const self = this;
+  const Manager = self.Manager;
+  const Api = self.Api;
+  const assistant = self.assistant;
+  const payload = self.payload;
+
+  return new Promise(async function(resolve, reject) {
+    const now = moment();
+    const deletionRegex = payload.data.payload.deletionRegex;
+
+    // Helpers
+    const getFileObjectWithMetaData = async (bucketName, fileName) => {
+      const [metaData] = await storage.bucket(bucketName).file(fileName).getMetadata();
+      return ({ fileName, created: metaData.timeCreated });
+    };
+
+    const deleteFileFromBucket = async (bucketName, fileName) => {
+      assistant.log(`Deleting item: ${fileName}...`, );
+      return await storage.bucket(bucketName).file(fileName).delete();
+    };
+
+    // Main
+    // get the file names as an array
+    let [allFiles] = await storage.bucket(bucketName).getFiles();
+    let deletePromises = [];
+    let foldersToDelete = [];
+    allFiles = allFiles.map(file => file.name);
+    // console.log(`all files: ${allFiles.join(', ')}`);
+
+    allFiles.forEach((filePath, i) => {
+      const fileName = filePath.split('/')[0];
+      const date = moment(fileName.split('T')[0]);
+      const day = date.date();
+      const month = date.month();
+      const age = now.diff(date, 'days', false);      
+      
+      if (age >= 30) {
+        if (day === 1) { return }
+        deletePromises.push(deleteFileFromBucket(bucketName, backup.fileName))
+        assistant.log(`Preparing to delete ${filePath}: date=${date.format('MMM Do, YYYY')}, day=${day}, month=${month}, age=${age}`, );
+        // if (!foldersToDelete.includes(fileName)) {
+        //   assistant.log(`Preparing to delete ${fileName}: date=${date.format('MMM Do, YYYY')}, day=${day}, month=${month}, age=${age}`, );
+        //   foldersToDelete = foldersToDelete.concat(fileName);
+        //   deletePromises.push(deleteFileFromBucket(bucketName, fileName))
+        // }
+      }
+    })
+
+    console.log('---deletePromises.length', deletePromises.length);
+
+    return
+
+    // transform to array of objects with creation timestamp { fileName: abc, created: xyz }
+    allFiles = allFiles.map(fileName => getFileObjectWithMetaData(bucketName, fileName));
+    allFiles = await Promise.all(allFiles);
+
+    allFiles.forEach((backup, i) => {
+      const date = moment(backup.created);
+      const day = date.date();
+      const month = date.month();
+      const age = now.diff(date, 'days', false);
+
+      assistant.log(`Sorting item ${i}: date=${date.format('MMM Do, YYYY')}, day=${day}, month=${month}, age=${age}`, );
+
+      if (age >= 31) {
+        if (day === 1) { return }
+        deletePromises.push(deleteFileFromBucket(bucketName, backup.fileName))
+      } else if ((deletionRegex && backup.fileName.match(deletionRegex))) {
+        deletePromises.push(deleteFileFromBucket(bucketName, backup.fileName))
+      }
+    })
+
+    await Promise.all(deletePromises);
+
+    return resolve();
+  });
+};
+
+Module.prototype.__deleteOldFiles = function (bucketName, resourceZone) {
   const self = this;
   const Manager = self.Manager;
   const Api = self.Api;

package/src/manager/functions/core/cron/daily.js

@@ -24,6 +24,7 @@ Module.prototype.main = function() {
   return new Promise(async function(resolve, reject) {
     fetch(`${Manager.project.functionsUrl}/bm_api`, {
       method: 'post',
+      response: 'json',
       body: {
         backendManagerKey: Manager.config.backend_manager.key,
         command: 'admin:backup',

package/src/manager/index.js

@@ -386,7 +386,7 @@ Manager.prototype.init = function (exporter, options) {
     // Cron
     exporter.bm_cronDaily =
     self.libraries.functions
-    .runWith({ memory: '256MB', timeoutSeconds: 60 })
+    .runWith({ memory: '256MB', timeoutSeconds: 120 })
     .pubsub.schedule('every 24 hours')
     .onRun(async (context) => {
       return self._process((new (require(`${core}/cron/daily.js`))()).init(self, { context: context, }))

package/templates/storage-lifecycle-config-30-days.json

@@ -0,0 +1,9 @@
+{
+  "rule":
+  [
+    {
+      "action": {"type": "Delete"},
+      "condition": {"age": 30}
+    }
+  ]
+}