backend-manager 3.0.35 → 3.0.37

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "backend-manager",
-  "version": "3.0.35",
+  "version": "3.0.37",
   "description": "Quick tools for developing Firebase functions",
   "main": "src/manager/index.js",
   "bin": {
@@ -10,6 +10,7 @@
   "scripts": {
     "_test": "npm run prepare && ./node_modules/mocha/bin/mocha test/ --recursive --timeout=10000",
     "test": "./node_modules/mocha/bin/mocha test/ --recursive --timeout=10000",
+    "test:usage": "./node_modules/mocha/bin/mocha test/usage.js --timeout=10000",
     "start": "node src/index.js"
   },
   "engines": {
@@ -57,7 +58,7 @@
     "npm-api": "^1.0.1",
     "paypal-server-api": "^1.0.3",
     "pushid": "^1.0.0",
-    "resolve-account": "^1.0.7",
+    "resolve-account": "^1.0.10",
     "semver": "^7.5.4",
     "shortid": "^2.2.16",
     "sizeitup": "^1.0.7",

package/src/manager/functions/core/actions/api/admin/cron.js

@@ -0,0 +1,37 @@
+function Module() {
+
+}
+
+Module.prototype.main = function () {
+  const self = this;
+  const Manager = self.Manager;
+  const Api = self.Api;
+  const assistant = self.assistant;
+  const payload = self.payload;
+
+  return new Promise(async function(resolve, reject) {
+
+    // Check if the user is an admin
+    if (!payload.user.roles.admin && assistant.meta.environment === 'production') {
+      return reject(assistant.errorManager(`Admin required.`, {code: 401, sentry: false, send: false, log: false}).error)
+    }
+
+    // Check if the ID is set
+    if (!payload.data.payload.id) {
+      return reject(assistant.errorManager(`Missing parameter {id}`, {code: 400, sentry: false, send: false, log: false}).error)
+    }
+
+    // Run the cron job
+    Manager._process((new (require(`../../../cron/${payload.data.payload.id}.js`))()).init(Manager, { context: {}, }))
+    .then((res) => {
+      return resolve({data: res});
+    })
+    .catch(e => {
+      return reject(assistant.errorManager(e, {code: 400, sentry: false, send: false, log: false}).error)
+    })
+  });
+
+};
+
+
+module.exports = Module;

package/src/manager/functions/core/cron/daily/reset-usage.js

@@ -0,0 +1,171 @@
+const _ = require('lodash');
+const fetch = require('wonderful-fetch');
+
+function Module() {
+
+}
+
+Module.prototype.init = function (Manager, data) {
+  const self = this;
+  self.Manager = Manager;
+
+  self.libraries = Manager.libraries;
+  self.assistant = Manager.Assistant();
+
+  self.storage = Manager.storage({name: 'usage', temporary: true, clear: false, log: false});
+
+  self.context = data.context;
+  return self;
+}
+
+Module.prototype.main = function() {
+  const self = this;
+  const Manager = self.Manager;
+  const libraries = self.libraries;
+  const assistant = self.assistant;
+  const context = self.context;
+
+  return new Promise(async function(resolve, reject) {
+    assistant.log(`cron/daily/reset-usage(): Starting main...`);
+
+    // Clear local
+    await self.clearLocal();
+
+    // Clear firestore
+    await self.clearFirestore();
+
+    return resolve();
+  });
+}
+
+Module.prototype.clearLocal = function() {
+  const self = this;
+  const Manager = self.Manager;
+  const libraries = self.libraries;
+  const assistant = self.assistant;
+  const context = self.context;
+
+  return new Promise(async function(resolve, reject) {
+    // Log status
+    assistant.log(`cron/daily/reset-usage() [local]: Starting...`);
+
+    // Set variables
+    const now = new Date();
+
+    // Log storage
+    assistant.log(`cron/daily/reset-usage() [local]: storage(apps)`, self.storage.get('apps', {}).value());
+    assistant.log(`cron/daily/reset-usage() [local]: storage(users)`, self.storage.get('users', {}).value());
+
+    // Clear storage
+    self.storage.setState({}).write();
+
+    // Log status
+    assistant.log(`cron/daily/reset-usage() [local]: Completed!`);
+
+    return resolve();
+  });
+}
+
+Module.prototype.clearFirestore = function() {
+  const self = this;
+  const Manager = self.Manager;
+  const libraries = self.libraries;
+  const assistant = self.assistant;
+  const context = self.context;
+
+  return new Promise(async function(resolve, reject) {
+    // Log status
+    assistant.log(`cron/daily/reset-usage() [firestore]: Starting...`);
+
+    // Clear storage
+    const metrics = await fetch(`https://us-central1-itw-creative-works.cloudfunctions.net/getApp`, {
+      method: 'post',
+      response: 'json',
+      body: {
+        id: Manager.config.app.id,
+      }
+    })
+    .then(response => {
+      response.products = response.products || {};
+
+      for (let product of Object.values(response.products)) {
+        product = product || {};
+        product.planId = product.planId || '';
+
+        if (product.planId.includes('basic')) {
+          return product.limits;
+        }
+      }
+
+      return new Error('No basic product found');
+    })
+    .catch(e => e);
+
+    // Log status
+    assistant.log(`cron/daily/reset-usage() [firestore]: Resetting metrics`, metrics);
+
+    if (metrics instanceof Error) {
+      return reject(assistant.errorManager(`Failed to check providers: ${metrics}`, {code: 500, sentry: false, send: false, log: false}).error)
+    }
+
+    // Reset all metrics with for loop of metrics
+    // TODO: OPTIMIZATION: Put all of the changes into a single batch
+    for (const metric of Object.keys(metrics)) {
+      assistant.log(`cron/daily/reset-usage() [firestore]: Resetting ${metric} for all users`);
+
+      await Manager.Utilities().iterateCollection((batch, index) => {
+        return new Promise(async (resolve, reject) => {
+          for (const doc of batch.docs) {
+            const data = doc.data();
+
+            // Normalize the metric
+            data.usage = data.usage || {};
+            data.usage[metric] = data.usage[metric] || {};
+            data.usage[metric].period = data.usage[metric].period || 0;
+            data.usage[metric].total = data.usage[metric].total || 0;
+            data.usage[metric].last = data.usage[metric].last || {};
+
+            // Bail if its 0
+            if (data.usage[metric].period <= 0) {
+              continue;
+            }
+
+            // Reset the metric
+            const original = data.usage[metric].period;
+            data.usage[metric].period = 0;
+
+            // Update the doc
+            await doc.ref.update({usage: data.usage})
+            .then(r => {
+              assistant.log(`cron/daily/reset-usage() [firestore]: Reset ${metric} for ${doc.id} (${original} -> 0)`);
+            })
+            .catch(e => {
+              assistant.errorManager(`Error resetting ${metric} for ${doc.id}: ${e}`, {sentry: false, send: false, log: true})
+            })
+          }
+
+          // Complete
+          return resolve();
+        });
+      }, {
+        collection: 'users',
+        where: [
+          {field: `usage.${metric}.period`, operator: '>', value: 0},
+        ],
+        batchSize: 5000,
+        log: true,
+      })
+      .then((r) => {
+        assistant.log(`cron/daily/reset-usage() [firestore]: Reset ${metric} for all users complete!`);
+      })
+      .catch(e => {
+        assistant.errorManager(`Error resetting ${metric} for all users: ${e}`, {sentry: false, send: false, log: true})
+      })
+    }
+
+    return resolve();
+  });
+}
+
+
+module.exports = Module;

package/src/manager/functions/core/cron/daily.js

@@ -1,4 +1,5 @@
 const fetch = require('wonderful-fetch');
+const jetpack = require('fs-jetpack');
 
 function Module() {
 
@@ -22,47 +23,42 @@ Module.prototype.main = function() {
   const context = self.context;
 
   return new Promise(async function(resolve, reject) {
-    // Wait for all the promises to resolve
-    Promise.all([
-      // Backup the database
-      // TODO: Disabled this because of Firebase's new PITR Disaster Recovery feature
-      // fetch(`${Manager.project.functionsUrl}/bm_api`, {
-      //   method: 'post',
-      //   response: 'json',
-      //   body: {
-      //     backendManagerKey: Manager.config.backend_manager.key,
-      //     command: 'admin:backup',
-      //   }
-      // })
-      // .then(response => {
-      //   assistant.log(`Successfully executed backup:`, response)
-      // }),
+    assistant.log(`cron/daily(): Starting...`);
 
-      // Sync Firestore users to the database
-      // TODO: This is not working becaues the pageToken is not relative any more when its saved...
-      // fetch(`${Manager.project.functionsUrl}/bm_api`, {
-      //   method: 'post',
-      //   response: 'json',
-      //   body: {
-      //     backendManagerKey: Manager.config.backend_manager.key,
-      //     command: 'admin:sync-users',
-      //   }
-      // })
-      // .then(response => {
-      //   assistant.log(`Successfully executed sync-users:`, response)
-      // }),
+    const jobsPath = `${__dirname}/daily`;
+    const jobs = jetpack.list(jobsPath);
+    let caught;
 
-      // More daily processes
-      // ...
-    ])
-    .then(() => {
-      assistant.log(`Successfully executed all daily processes:`)
-      return resolve();
-    })
-    .catch(e => {
-      assistant.errorManager(`Error executing all processes: ${e}`, {sentry: true, send: false, log: true})
-      return reject(e);
-    })
+    // For of loop for jobs, load the job, execute it, and log the result
+    for (let i = 0; i < jobs.length; i++) {
+      const job = jobs[i];
+      const jobName = job.replace('.js', '');
+
+      // Log
+      assistant.log(`cron/daily(): Job ${jobName} starting...`);
+
+      // Load the job
+      const Job = require(`${jobsPath}/${job}`);
+      const jobInstance = new Job();
+      jobInstance.init(Manager, { context: context, });
+
+      // Execute the job
+      await jobInstance.main()
+      .then(res => {
+        assistant.log(`cron/daily(): Job ${jobName} completed...`);
+      })
+      .catch(e => {
+        assistant.errorManager(`Error executing ${jobName}: ${e}`, {sentry: true, send: false, log: true});
+        caught = e;
+      })
+    }
+
+    if (caught) {
+      return reject(caught);
+    }
+
+    // Return
+    return resolve();
   });
 }
 

package/src/manager/helpers/settings.js

@@ -0,0 +1,32 @@
+/**
+ * Settings
+ *
+ */
+function Settings(m) {
+  const self = this;
+
+  self.Manager = m;
+
+  self.user = null;
+
+  self.initialized = false;
+}
+
+Settings.prototype.init = function (assistant, options) {
+  const self = this;
+
+  return new Promise(async function(resolve, reject) {
+    const Manager = self.Manager;
+
+    // Set options
+    options = options || {};
+
+    // Set initialized to true
+    self.initialized = true;
+
+    // Resolve
+    return resolve(self);
+  });
+};
+
+module.exports = Settings;

package/src/manager/helpers/usage.js

@@ -0,0 +1,242 @@
+/**
+ * Usage
+ * Meant to check and update usage for a user
+ * Uses the ITWCW apps/{app}/products/{product}/limits/{metric} to check limits
+ * Stores usage in the user's firestore document OR in local/temp storage if no user
+ */
+
+const fetch = require('wonderful-fetch');
+const moment = require('moment');
+const _ = require('lodash');
+const hcaptcha = require('hcaptcha');
+
+function Usage(m) {
+  const self = this;
+
+  self.Manager = m;
+
+  self.user = null;
+  self.app = null;
+  self.options = null;
+  self.assistant = null;
+  self.storage = null;
+
+  self.paths = {
+    user: '',
+    app: '',
+  }
+
+  self.initialized = false;
+}
+
+Usage.prototype.init = function (assistant, options) {
+  const self = this;
+
+  return new Promise(async function(resolve, reject) {
+    const Manager = self.Manager;
+
+    // Set options
+    options = options || {};
+    options.app = options.app || Manager.config.app.id;
+    options.refetch = typeof options.refetch === 'undefined' ? false : options.refetch;
+    options.clear = typeof options.clear === 'undefined' ? false : options.clear;
+    options.today = options.today || undefined;
+    options.log = typeof options.log === 'undefined' ? false : options.log;
+
+    // Check for required options
+    if (!assistant) {
+      return reject(new Error('Missing required {assistant} parameter'));
+    }
+
+    // Set options
+    self.options = options;
+
+    // Set assistant
+    self.assistant = assistant;
+
+    // Setup storage
+    self.storage = Manager.storage({name: 'usage', temporary: true, clear: options.clear, log: options.log});
+
+    self.paths.user = `users.${self.assistant.request.geolocation.ip.replace(/[\.:]/g, '_')}`;
+    self.paths.app = `apps.${options.app}`;
+
+    // Get storage data
+    const appLastFetched = moment(self.storage.get(`${self.paths.app}.lastFetched`, 0).value());
+    const diff = moment().diff(appLastFetched, 'hours');
+
+    // Authenticate user (user will be resolved as well)
+    self.user = await assistant.authenticate();
+
+    // Replce with local if no user
+    if (!self.user.auth.uid) {
+      const existingLocal = self.storage.get(self.paths.user, {}).value();
+      self.user.usage = existingLocal && existingLocal.usage ? existingLocal.usage : self.user.usage;
+    }
+
+    // Log
+    self.log(`Usage.init(): Checking if usage data needs to be fetched (${diff} hours)...`);
+
+    // Get app data to get plan limits using cached data if possible
+    if (diff > 1 || options.refetch) {
+      await fetch('https://us-central1-itw-creative-works.cloudfunctions.net/getApp', {
+        method: 'post',
+        response: 'json',
+        body: {
+          id: options.app,
+        },
+      })
+      .then((json) => {
+        // Write data and last fetched to storage
+        self.storage.set(`${self.paths.app}.data`, json).write();
+        self.storage.set(`${self.paths.app}.lastFetched`, new Date().toISOString()).write();
+      })
+      .catch(e => {
+        assistant.errorManager(`Usage.init(): Error fetching app data: ${e}`, {sentry: true, send: false, log: true});
+      })
+    }
+
+    // Get app data
+    self.app = self.storage.get(`${self.paths.app}.data`, {}).value();
+
+    if (!self.app) {
+      return reject(new Error('Usage.init(): No app data found'));
+    }
+
+    self.log(`Usage.init(): Got app data`, self.app);
+    self.log(`Usage.init(): Got user`, self.user);
+
+    // Resolve with the user to get their current plan limits
+    // If there is no user, go forward with their IP and make them a basic plan user
+
+    // Set initialized to true
+    self.initialized = true;
+
+    // Resolve
+    return resolve(self);
+  });
+};
+
+Usage.prototype.validate = function (path, options) {
+  const self = this;
+
+  return new Promise(async function(resolve, reject) {
+    const Manager = self.Manager;
+    const assistant = self.assistant;
+
+    options = options || {};
+    options.useCaptchaResponse = typeof options.useCaptchaResponse === 'undefined' ? true : options.useCaptchaResponse;
+
+    // Check for required options
+    const period = _.get(self.user, `usage.${path}.period`, 0);
+    const allowed = _.get(self.app, `products.${self.options.app}-${self.user.plan.id}.limits.${path}`, 0);
+
+    // Log
+    self.log(`Usage.validate(): Checking ${period}/${allowed} for ${path}...`);
+
+    // If they are under the limit, resolve
+    if (period < allowed) {
+      self.log(`Usage.validate(): Valid for ${path}`);
+
+      return resolve(true);
+    }
+
+    // If they are using captcha, attempt to resolve
+    const captchaResponse = assistant.request.data['h-captcha-response'];
+    if (captchaResponse && options.useCaptchaResponse) {
+      self.log(`Usage.validate(): Checking captcha response`, captchaResponse);
+
+      const captchaResult = await hcaptcha.verify(process.env.HCAPTCHA_SECRET, captchaResponse)
+        .then((data) => data)
+        .catch((e) => e);
+
+      // If the captcha is valid, resolve
+      if (!captchaResult || captchaResult instanceof Error || !captchaResult.success) {
+        return reject(
+          assistant.errorManager(`Captcha verification failed.`, {code: 400, sentry: false, send: false, log: false}).error
+        );
+      }
+    }
+
+    // Otherwise, they are over the limit, reject
+    return reject(
+      assistant.errorManager(`You have exceeded your ${path} usage limit of ${period}/${allowed}.`, {code: 429, sentry: false, send: false, log: false}).error
+    );
+  });
+};
+
+Usage.prototype.increment = function (path, value, options) {
+  const self = this;
+  const Manager = self.Manager;
+  const assistant = self.assistant;
+
+  options = options || {};
+  options.id = options.id || null;
+
+  // Update total and period
+  ['total', 'period', 'last'].forEach((key) => {
+    const resolved = `usage.${path}.${key}`;
+
+    if (key === 'last') {
+      const now = moment(
+        typeof self.options.today === 'undefined' ? new Date() : self.options.today
+      );
+
+      _.set(self.user, resolved, {
+        id: options.id,
+        timestamp: now.toISOString(),
+        timestampUNIX: now.unix(),
+      });
+    } else {
+      _.set(self.user, resolved,
+        _.get(self.user, resolved, 0) + value
+      );
+    }
+  });
+
+  // Log the updated user
+  self.log(`Usage.init(): Incremented user`, self.user);
+
+  return self;
+};
+
+Usage.prototype.update = function () {
+  const self = this;
+
+  return new Promise(async function(resolve, reject) {
+    const Manager = self.Manager;
+    const assistant = self.assistant;
+
+    // Write self.user to firestore or local if no user
+    if (self.user.auth.uid) {
+      Manager.libraries.admin.firestore(`users/${self.user.auth.uid}`)
+        .set({
+          usage: self.user.usage,
+        }, {merge: true})
+        .then(() => {
+          self.log(`Usage.update(): Updated user in firestore`);
+
+          return resolve();
+        })
+        .catch(e => {
+          return reject(assistant.errorManager(e, {sentry: true, send: false, log: false}));
+        });
+    } else {
+      self.storage.set(`${self.paths.user}.usage`, self.user.usage).write();
+
+      self.log(`Usage.update(): Updated user in local storage`);
+
+      return resolve();
+    }
+  });
+};
+
+Usage.prototype.log = function () {
+  const self = this;
+
+  // Log
+  if (self.options.log) {
+    self.assistant.log(...arguments);
+  }
+};
+
+module.exports = Usage;

package/src/manager/helpers/user.js

@@ -107,6 +107,17 @@ function User(Manager, settings, options) {
       clientId: _.get(settings, 'api.clientId', defaults ? `${uuid4()}` : null),
       privateKey: _.get(settings, 'api.privateKey', defaults ? `${uidgen.generateSync()}` : null),
     },
+    usage: {
+      requests: {
+        period: _.get(settings, 'usage.requests.period', defaults ? 0 : null),
+        total: _.get(settings, 'usage.requests.total', defaults ? 0 : null),
+        last: {
+          id: _.get(settings, 'usage.requests.last.id', defaults ? '' : null),
+          timestamp: _.get(settings, 'usage.requests.last.timestamp', defaults ? oldDate : null),
+          timestampUNIX: _.get(settings, 'usage.requests.last.timestampUNIX', defaults ? oldDateUNIX : null),
+        },
+      },
+    },
     personal: {
       birthday: {
         timestamp: _.get(settings, 'personal.birthday.timestamp', defaults ? oldDate : null),

package/src/manager/index.js

@@ -74,8 +74,8 @@ Manager.prototype.init = function (exporter, options) {
   self.options = options;
   self.project = options.firebaseConfig || JSON.parse(process.env.FIREBASE_CONFIG || '{}');
   self.project.resourceZone = options.resourceZone;
-  self.project.serviceAccountPath = path.resolve(self.cwd, options.serviceAccountPath)
-  self.project.backendManagerConfigPath = path.resolve(self.cwd, options.backendManagerConfigPath)
+  self.project.serviceAccountPath = path.resolve(self.cwd, options.serviceAccountPath);
+  self.project.backendManagerConfigPath = path.resolve(self.cwd, options.backendManagerConfigPath);
 
   self.package = resolveProjectPackage();
   self.config = merge(
@@ -600,6 +600,18 @@ Manager.prototype.SubscriptionResolver = function () {
   return new self.libraries.SubscriptionResolver(self, ...arguments);
 };
 
+Manager.prototype.Usage = function () {
+  const self = this;
+  self.libraries.Usage = self.libraries.Usage || require('./helpers/usage.js');
+  return new self.libraries.Usage(self, ...arguments);
+};
+
+Manager.prototype.Settings = function () {
+  const self = this;
+  self.libraries.Settings = self.libraries.Settings || require('./helpers/settings.js');
+  return new self.libraries.Settings(self, ...arguments);
+};
+
 Manager.prototype.Metadata = function () {
   const self = this;
   self.libraries.Metadata = self.libraries.Metadata || require('./helpers/metadata.js');
@@ -657,21 +669,26 @@ Manager.prototype.storage = function (options) {
   if (!self._internal.storage[options.name]) {
     options.temporary = typeof options.temporary === 'undefined' ? false : options.temporary;
     options.clear = typeof options.clear === 'undefined' ? true : options.clear;
+    options.log = typeof options.log === 'undefined' ? false : options.log;
 
     const low = require('lowdb');
     const FileSync = require('lowdb/adapters/FileSync');
-    const dbPath = options.temporary
+    const location = options.temporary
       ? `${require('os').tmpdir()}/storage/${options.name}.json`
       : `./.data/storage/${options.name}.json`;
-    const adapter = new FileSync(dbPath);
+    const adapter = new FileSync(location);
+
+    if (options.log) {
+      self.assistant.log('storage(): Location', location);
+    }
 
     if (
       options.temporary
       && self.assistant.meta.environment === 'development'
       && options.clear
     ) {
-      self.assistant.log('Removed temporary file @', dbPath);
-      jetpack.remove(dbPath);
+      self.assistant.log('Removed temporary file @', location);
+      jetpack.remove(location);
     }
 
     options.clearInvalid = typeof options.clearInvalid === 'undefined'
@@ -679,27 +696,27 @@ Manager.prototype.storage = function (options) {
       : options.clearInvalid;
 
     function _setup() {
-      if (!jetpack.exists(dbPath)) {
-        jetpack.write(dbPath, {});
+      if (!jetpack.exists(location)) {
+        jetpack.write(location, {});
       }
       self._internal.storage[options.name] = low(adapter);
 
-      self._internal.storage[options.name].set('_location', dbPath)
+      self._internal.storage[options.name].set('_location', location)
     }
 
     try {
       _setup()
     } catch (e) {
-      self.assistant.error(`Could not setup storage: ${dbPath}`, e);
+      self.assistant.error(`Could not setup storage: ${location}`, e);
 
       try {
         if (options.clearInvalid) {
-          self.assistant.log(`Clearing invalid storage: ${dbPath}`);
-          jetpack.write(dbPath, {});
+          self.assistant.log(`Clearing invalid storage: ${location}`);
+          jetpack.write(location, {});
         }
         _setup()
       } catch (e) {
-        self.assistant.error(`Failed to clear invalid storage: ${dbPath}`, e);
+        self.assistant.error(`Failed to clear invalid storage: ${location}`, e);
       }
     }
   }
@@ -707,29 +724,6 @@ Manager.prototype.storage = function (options) {
   return self._internal.storage[options.name]
 };
 
-// Manager.prototype.LocalDatabase = function () {
-//   const self = this;
-//   if (!self.libraries.LocalDatabase) {
-//     const low = require('lowdb');
-//     const FileSync = require('lowdb/adapters/FileSync');
-//     // const dbPath = path.resolve(process.cwd(), './.data/db.json');
-//     const dbPath = './.data/db.json';
-//     const adapter = new FileSync(dbPath);
-//     const jetpack = require('fs-jetpack');
-//
-//     try {
-//       if (!jetpack.exists(dbPath)) {
-//         jetpack.write(dbPath, {});
-//       }
-//       self.localDatabase = low(adapter);
-//     } catch (e) {
-//       console.error('Could not load .data', e);
-//     }
-//   }
-//   self.libraries.LocalDatabase = self.libraries.LocalDatabase || require('./helpers/api-manager.js');
-//   return new self.libraries.LocalDatabase(self, ...arguments);
-// };
-
 Manager.prototype.require = function (p) {
   return require(p);
 };