@abtnode/core 1.16.8-beta-186fd5aa → 1.16.8-next-d1e52353

package/lib/states/webhook.js

@@ -3,11 +3,10 @@ const logger = require('@abtnode/logger')('@abtnode/core:states:webhook');
 const BaseState = require('./base');
 const { validateWebhook } = require('../validators/webhook');
 
+/**
+ * @extends BaseState<import('@abtnode/models').WebHookState>
+ */
 class WebhookState extends BaseState {
-  constructor(baseDir, config = {}) {
-    super(baseDir, { filename: 'webhook.db', ...config });
-  }
-
   async create(info, { mock } = {}) {
     const { type, params } = info;
     const filterParams = params.map((item) => {
@@ -46,7 +45,7 @@ class WebhookState extends BaseState {
       throw new Error('webhookId should not be empty');
     }
     const webhook = await this.findOne(id);
-    const num = await this.remove({ _id: id });
+    const num = await this.remove({ id });
     if (num <= 0) {
       throw new Error(`${id} does not exist`);
     }
@@ -56,8 +55,7 @@ class WebhookState extends BaseState {
 
   async findOne(id) {
     try {
-      const webhook = await super.findOne({ _id: id });
-
+      const webhook = await super.findOne({ id });
       return webhook;
     } catch (error) {
       throw new Error(`webhook with id ${id} not exist`);

package/lib/team/manager.js

@@ -1,3 +1,4 @@
+/* eslint-disable global-require */
 /**
  * Access Control Manager is the business logic layer of access control related
  */
@@ -7,22 +8,36 @@ const upperFirst = require('lodash/upperFirst');
 const get = require('lodash/get');
 const pick = require('lodash/pick');
 
-const { createRBAC, MemoryStorage, NedbStorage } = require('@abtnode/rbac');
+const { createRBAC, MemoryStorage, SequelizeStorage } = require('@abtnode/rbac');
 const logger = require('@abtnode/logger')('@abtnode/core:team:manager');
 const { ROLES, RBAC_CONFIG } = require('@abtnode/constant');
 const { BlockletEvents } = require('@blocklet/constant');
 const Lock = require('@abtnode/util/lib/lock');
 const sleep = require('@abtnode/util/lib/sleep');
-const UserState = require('../states/user');
-const SessionState = require('../states/session');
+const { BaseState, doSchemaMigration, createSequelize, getBlockletModels, setupModels } = require('@abtnode/models');
 
-const { isCLI } = require('../util');
+const { isCLI, getDbFilePath } = require('../util');
 
 const rbacCreationLock = new Lock('rbac-creation-lock');
 
+const States = {
+  User: require('../states/user'),
+  Passport: require('../states/passport'),
+  ConnectedAccount: require('../states/connect-account'),
+  Session: require('../states/session'),
+};
+
+const getDefaultTeamState = () => ({
+  rbac: null,
+  user: null,
+  passport: null,
+  connectedAccount: null,
+  session: null,
+});
+
 const closeDatabase = async (db) =>
   new Promise((resolve, reject) => {
-    if (!db) {
+    if (!db || process.env.NODE_ENV === 'test') {
       resolve(true);
       return;
     }
@@ -53,6 +68,8 @@ class TeamManager extends EventEmitter {
     // cache: { [did]: { rbac, user, session } }
     this.cache = {};
 
+    this.models = new Map();
+
     this.init();
   }
 
@@ -60,11 +77,7 @@ class TeamManager extends EventEmitter {
     // listen blocklet state
     ['add', 'upgrade'].forEach((event) => {
       this.states.blocklet.on(event, ({ meta: { did } }) => {
-        this.cache[did] = {
-          rbac: null,
-          user: null,
-          session: null,
-        };
+        this.cache[did] = getDefaultTeamState();
       });
     });
 
@@ -73,119 +86,69 @@ class TeamManager extends EventEmitter {
       .getBlocklets()
       .then((blocklets) => {
         blocklets.forEach(({ meta: { did } }) => {
-          this.cache[did] = {
-            rbac: null,
-            user: null,
-            session: null,
-          };
+          this.cache[did] = getDefaultTeamState();
         });
       })
       .catch((error) => {
         logger.error('get blocklets failed', error);
       });
 
-    // init abtnode
-    this.cache[this.nodeDid] = {
-      rbac: null,
-      user: null,
-      session: null,
-    };
-
-    logger.info('init node rbac', { nodeDid: this.nodeDid });
-
-    const rbac = await createRBAC({ storage: new MemoryStorage(), data: RBAC_CONFIG });
-
-    logger.info('init node user', { nodeDid: this.nodeDid });
-    const user = await this.createUserState(this.nodeDid);
-
-    logger.info('init node session', { nodeDid: this.nodeDid });
-    const session = await this.createSessionState(this.nodeDid);
+    // init server
+    this.cache[this.nodeDid] = getDefaultTeamState();
+    logger.info('init node team manager', { nodeDid: this.nodeDid });
 
     this.cache[this.nodeDid] = {
-      rbac,
-      user,
-      session,
+      rbac: await createRBAC({ storage: new MemoryStorage(), data: RBAC_CONFIG }),
+      user: await this.createState(this.nodeDid, 'User'),
+      passport: await this.createState(this.nodeDid, 'Passport'),
+      connectedAccount: await this.createState(this.nodeDid, 'ConnectedAccount'),
+      session: await this.createState(this.nodeDid, 'Session'),
     };
   }
 
-  // user state
-
-  async getUserState(did) {
-    const pid = await this.getPid(did);
-
-    // validate exist
-    if (!pid) {
-      logger.error('Did does not exist', { action: 'getUserState', did });
-      throw new Error(`Did does not exist: ${did}`);
-    }
-
-    if (!this.cache[pid]) {
-      this.cache[pid] = {
-        rbac: null,
-        user: null,
-        session: null,
-      };
-    }
-
-    // first get userState after blocklet added
-    if (!this.cache[pid].user) {
-      const user = await this.createUserState(did);
-      logger.info('Team state created', { teamDid: did });
-      this.cache[pid].user = user;
-      return user;
-    }
-
-    // use cache
-    return this.cache[pid].user;
+  async getUserState(teamDid) {
+    return this.getState(teamDid, 'user');
   }
 
-  async createUserState(did) {
-    const dir = await this.getDataDirByDid(did);
-    const userState = new UserState(dir);
-    return new Promise((resolve) => {
-      userState.onReady(() => {
-        resolve(userState);
-      });
-    });
+  async getPassportState(teamDid) {
+    return this.getState(teamDid, 'passport');
   }
 
-  // session state
+  async getConnectedAccount(teamDid) {
+    return this.getState(teamDid, 'connectedAccount');
+  }
 
-  async getSessionState(did) {
-    const pid = await this.getPid(did);
+  async getSessionState(teamDid) {
+    return this.getState(teamDid, 'session');
+  }
 
-    // validate exist
+  async getState(teamDid, key) {
+    const pid = await this.getPid(teamDid);
     if (!pid) {
-      logger.error('Did does not exist', { action: 'getSessionState', did });
-      throw new Error(`Did does not exist: ${did}`);
+      logger.error('teamDid does not exist', { action: 'getState', key, teamDid });
+      throw new Error(`teamDid does not exist: ${teamDid}`);
     }
 
     if (!this.cache[pid]) {
-      this.cache[pid] = {
-        rbac: null,
-        user: null,
-        session: null,
-      };
+      this.cache[pid] = getDefaultTeamState();
     }
 
-    // first get userState after blocklet added
-    if (!this.cache[pid].session) {
-      const session = await this.createSessionState(pid);
-      logger.info('session state created', { did, pid });
-      this.cache[pid].session = session;
-      return session;
+    if (this.cache[pid][key]) {
+      return this.cache[pid][key];
     }
 
-    // use cache
-    return this.cache[pid].session;
+    const state = await this.createState(teamDid, upperFirst(key));
+    logger.info(`${key} state created`, { teamDid });
+    this.cache[pid][key] = state;
+    return state;
   }
 
-  async createSessionState(did) {
-    const dir = await this.getDataDirByDid(did);
-    const sessionState = new SessionState(dir);
+  async createState(did, key) {
+    const models = await this.getModels(did);
+    const state = new States[key](models[key], {}, models);
     return new Promise((resolve) => {
-      sessionState.onReady(() => {
-        resolve(sessionState);
+      state.onReady(() => {
+        resolve(state);
       });
     });
   }
@@ -202,11 +165,7 @@ class TeamManager extends EventEmitter {
     }
 
     if (!this.cache[pid]) {
-      this.cache[pid] = {
-        rbac: null,
-        user: null,
-        session: null,
-      };
+      this.cache[pid] = getDefaultTeamState();
     }
 
     // first getRBAC after blocklet added
@@ -219,8 +178,9 @@ class TeamManager extends EventEmitter {
       await rbacCreationLock.acquire();
       try {
         logger.info('create rbac', { did });
-        const file = await this.getRbacFileByDid(pid);
-        const rbac = await createRBAC({ storage: new NedbStorage({ file }) });
+        const models = await this.getModels(did);
+        const db = new BaseState(models.Rbac);
+        const rbac = await createRBAC({ storage: new SequelizeStorage(db) });
         this.cache[pid].rbac = rbac;
 
         const roles = await rbac.getRoles();
@@ -351,9 +311,7 @@ class TeamManager extends EventEmitter {
 
   async getRoles(did) {
     const rbac = await this.getRBAC(did);
-
     const roles = await rbac.getRoles();
-
     return roles.map((d) => pick(d, ['name', 'grants', 'title', 'description']));
   }
 
@@ -365,18 +323,25 @@ class TeamManager extends EventEmitter {
 
     logger.info('initTeam', { did });
 
+    await this.getModels(did);
+
     const rbac = await this.getRBAC(did);
-    const user = await this.getUserState(did);
-    const session = await this.getSessionState(did);
+    const user = await this.createState(did, 'User');
+    const passport = await this.createState(did, 'Passport');
+    const connectedAccount = await this.createState(did, 'ConnectedAccount');
+    const session = await this.createState(did, 'Session');
 
     this.cache[did] = {
       rbac,
       user,
       session,
+      passport,
+      connectedAccount,
     };
   }
 
-  async deleteTeam(did, { closeDatabase: closeDB = true } = {}) {
+  // FIXME: @wangshijun should we clear models cache here
+  async deleteTeam(did, { closeDatabase: closeDB = false } = {}) {
     if (!did) {
       logger.error('deleteTeam: did does not exist');
       return;
@@ -385,17 +350,21 @@ class TeamManager extends EventEmitter {
     logger.info('deleteTeam', { did, closeDB });
 
     const pid = await this.getPid(did);
-
     if (closeDB && this.cache[pid]) {
+      logger.info('deleteTeam.closeDatabase', { did });
       try {
         if (this.cache[pid].rbac) {
           await closeDatabase(this.cache[pid].rbac.storage.db);
         }
-
         if (this.cache[pid].user) {
           await closeDatabase(this.cache[pid].user.db);
         }
-
+        if (this.cache[pid].passport) {
+          await closeDatabase(this.cache[pid].passport.db);
+        }
+        if (this.cache[pid].connectedAccount) {
+          await closeDatabase(this.cache[pid].connectedAccount.db);
+        }
         if (this.cache[pid].session) {
           await closeDatabase(this.cache[pid].session.db);
         }
@@ -415,10 +384,9 @@ class TeamManager extends EventEmitter {
   // =======
   // Private
   // =======
-
-  async getRbacFileByDid(did) {
+  async getDataFileByDid(did) {
     const dir = await this.getDataDirByDid(did);
-    return path.join(dir, 'rbac.db');
+    return getDbFilePath(path.join(dir, 'blocklet.db'));
   }
 
   async getDataDirByDid(did) {
@@ -430,12 +398,36 @@ class TeamManager extends EventEmitter {
     return path.join(this.dataDirs.data, blocklet.meta.name);
   }
 
-  async getPid(did) {
+  async getModels(did) {
+    if (this.isNodeTeam(did)) {
+      return this.states.models;
+    }
+
+    if (this.models.has(did) === false) {
+      const file = await this.getDataFileByDid(did);
+      const sequelize = createSequelize(file);
+      const models = getBlockletModels();
+      logger.info('initModels', { did, models: Object.keys(models) });
+      setupModels(models, sequelize);
+      await this.initDatabase(did);
+      this.models.set(did, models);
+    }
+
+    return this.models.get(did);
+  }
+
+  async initDatabase(did) {
     if (this.isNodeTeam(did)) {
-      return did;
+      return;
     }
 
-    return this.states.blocklet.getBlockletMetaDid(did);
+    const dbPath = await this.getDataFileByDid(did);
+    logger.info('initDatabase', { did, dbPath });
+    await doSchemaMigration(dbPath, 'blocklet');
+  }
+
+  async getPid(did) {
+    return this.isNodeTeam(did) ? did : this.states.blocklet.getBlockletMetaDid(did);
   }
 }
 

package/lib/util/blocklet.js

@@ -125,7 +125,6 @@ const statusMap = {
 };
 
 const PRIVATE_NODE_ENVS = [
-  'NEDB_MULTI_PORT',
   'ABT_NODE_UPDATER_PORT',
   'ABT_NODE_SESSION_TTL',
   'ABT_NODE_ROUTER_PROVIDER',

package/lib/util/index.js

@@ -467,6 +467,8 @@ const isBlockletSite = (domain) => domain.endsWith(BLOCKLET_SITE_GROUP_SUFFIX);
 const isServerSite = (domain) =>
   [DOMAIN_FOR_DEFAULT_SITE, DOMAIN_FOR_IP_SITE, DOMAIN_FOR_IP_SITE_REGEXP].includes(domain);
 
+const getDbFilePath = (filePath) => (process.env.NODE_ENV === 'test' ? `${filePath}:memory:` : filePath);
+
 const lib = {
   validateOwner,
   getProviderFromNodeInfo,
@@ -502,6 +504,7 @@ const lib = {
   isGatewayCacheEnabled,
   isBlockletSite,
   isServerSite,
+  getDbFilePath,
 };
 
 module.exports = lib;

package/lib/util/queue.js

@@ -1,8 +1,18 @@
-const path = require('path');
 const createQueue = require('@abtnode/queue');
+const QueueStore = require('@abtnode/queue/lib/store/sequelize');
 
 /**
- *
+ * @param {object} params
+ * @param {string} params.name name of the queue
+ * @param {object} params.model model
+ * @param {function} params.onJob called on receives job
+ * @param {boolean} params.daemon [param=false] is the running in a daemon environment, default false
+ * @param {object} params.options options
+ * @param {any} [params.options.id] id of the job
+ * @param {number} [params.options.concurrency] [param=1] number of concurrent jobs
+ * @param {number} [params.options.maxRetries] [param=1] number of max retries, default 1
+ * @param {number} [params.options.maxTimeout] [param=86400000] max timeout, in ms, default 86400000ms(1d)
+ * @param {number} [params.options.retryDelay] [param=0] retry delay, in ms, default 0ms
  * @typedef {
  *  EventEmitter & {
  *    store: JobStore;
@@ -18,26 +28,10 @@ const createQueue = require('@abtnode/queue');
  *  }
  * } Queue
  */
-
-/**
- *
- * @param {Object} Options
- * @param {string} Options.name name of the queue
- * @param {string} Options.dataDir directory path of the queue data
- * @param {function} Options.onJob called on receives job
- * @param {boolean} Options.daemon [param=false] is the running in a daemon environment, default false
- * @param {Object} Options.options options
- * @param {string} Options.options.id id of the job
- * @param {number} Options.options.concurrency [param=1] number of concurrent jobs
- * @param {number} Options.options.maxRetries [param=1] number of max retries, default 1
- * @param {number} Options.options.maxTimeout [param=86400000] max timeout, in ms, default 86400000ms(1d)
- * @param {number} Options.options.retryDelay [param=0] retry delay, in ms, default 0ms
- * @returns {Queue}
- */
-module.exports = ({ name, dataDir, onJob, daemon = false, options = {} }) => {
+module.exports = ({ name, model, onJob, daemon = false, options = {} }) => {
   if (daemon) {
     const queue = createQueue({
-      file: path.join(dataDir, `${name}.db`),
+      store: new QueueStore(model, name),
       onJob,
       options,
     });

package/lib/util/ready.js

@@ -27,7 +27,7 @@ const createStateReadyHandler =
           const count = await states.node.count();
           if (count > 1) {
             // eslint-disable-next-line no-underscore-dangle
-            await states.node.remove({ _id: state._id });
+            await states.node.remove({ did: state.did });
             console.error('\n\x1b[31m======================================================');
             console.error(`Data dir: ${options.dataDir} is used by another Blocklet Server instance, abort!`);
             console.error('Sharing data dir between Blocklet Server instances may break things!');

package/lib/webhook/index.js

@@ -31,11 +31,11 @@ const getSlackUrlInfo = async (actionPath = '/notifications', urls) => {
     });
   }
 
-  const prioritys = await evaluateURLs(
+  const priorities = await evaluateURLs(
     urls.map((item) => item.url),
-    { checkAccessble: checkURLAccessible }
+    { checkAccessible: checkURLAccessible }
   );
-  const priorityUrl = prioritys[0].url;
+  const priorityUrl = priorities[0].url;
 
   const { protocol } = new URL(priorityUrl);
   const normalized = `${priorityUrl}${actionPath}`.replace(`${protocol}//`, '').replace(/\/+/g, '/');
@@ -60,6 +60,7 @@ const getSlackUrlInfo = async (actionPath = '/notifications', urls) => {
   return info;
 };
 
+// eslint-disable-next-line no-unused-vars
 module.exports = ({ events, dataDirs, instance }) => {
   const nodeState = states.node;
   const webhookState = states.webhook;
@@ -105,8 +106,9 @@ module.exports = ({ events, dataDirs, instance }) => {
 
   const queue = createQueue({
     name: 'webhook-messages',
-    dataDir: dataDirs.core,
+    model: states.job,
     daemon: true,
+    options: {},
     onJob: async (job) => {
       if (job.entityType === 'webhook') {
         return;

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.16.8-beta-186fd5aa",
+  "version": "1.16.8-next-d1e52353",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -19,18 +19,18 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "MIT",
   "dependencies": {
-    "@abtnode/auth": "1.16.8-beta-186fd5aa",
-    "@abtnode/certificate-manager": "1.16.8-beta-186fd5aa",
-    "@abtnode/constant": "1.16.8-beta-186fd5aa",
-    "@abtnode/cron": "1.16.8-beta-186fd5aa",
-    "@abtnode/db": "1.16.8-beta-186fd5aa",
-    "@abtnode/logger": "1.16.8-beta-186fd5aa",
-    "@abtnode/queue": "1.16.8-beta-186fd5aa",
-    "@abtnode/rbac": "1.16.8-beta-186fd5aa",
-    "@abtnode/router-provider": "1.16.8-beta-186fd5aa",
-    "@abtnode/static-server": "1.16.8-beta-186fd5aa",
-    "@abtnode/timemachine": "1.16.8-beta-186fd5aa",
-    "@abtnode/util": "1.16.8-beta-186fd5aa",
+    "@abtnode/auth": "1.16.8-next-d1e52353",
+    "@abtnode/certificate-manager": "1.16.8-next-d1e52353",
+    "@abtnode/constant": "1.16.8-next-d1e52353",
+    "@abtnode/cron": "1.16.8-next-d1e52353",
+    "@abtnode/logger": "1.16.8-next-d1e52353",
+    "@abtnode/models": "1.16.8-next-d1e52353",
+    "@abtnode/queue": "1.16.8-next-d1e52353",
+    "@abtnode/rbac": "1.16.8-next-d1e52353",
+    "@abtnode/router-provider": "1.16.8-next-d1e52353",
+    "@abtnode/static-server": "1.16.8-next-d1e52353",
+    "@abtnode/timemachine": "1.16.8-next-d1e52353",
+    "@abtnode/util": "1.16.8-next-d1e52353",
     "@arcblock/did": "1.18.78",
     "@arcblock/did-auth": "1.18.78",
     "@arcblock/did-ext": "^1.18.78",
@@ -41,9 +41,9 @@
     "@arcblock/pm2-events": "^0.0.5",
     "@arcblock/validator": "^1.18.78",
     "@arcblock/vc": "1.18.78",
-    "@blocklet/constant": "1.16.8-beta-186fd5aa",
-    "@blocklet/meta": "1.16.8-beta-186fd5aa",
-    "@blocklet/sdk": "1.16.8-beta-186fd5aa",
+    "@blocklet/constant": "1.16.8-next-d1e52353",
+    "@blocklet/meta": "1.16.8-next-d1e52353",
+    "@blocklet/sdk": "1.16.8-next-d1e52353",
     "@did-space/client": "^0.2.91",
     "@fidm/x509": "^1.2.1",
     "@ocap/mcrypto": "1.18.78",
@@ -77,6 +77,7 @@
     "p-limit": "^3.1.0",
     "p-retry": "4.6.1",
     "semver": "^7.3.8",
+    "sequelize": "^6.31.0",
     "shelljs": "^0.8.5",
     "ssri": "^8.0.1",
     "stream-throttle": "^0.1.3",
@@ -86,7 +87,7 @@
     "transliteration": "^2.3.5",
     "ua-parser-js": "^1.0.2",
     "url-join": "^4.0.1",
-    "uuid": "7.0.3",
+    "uuid": "^8.3.2",
     "valid-url": "^1.0.9"
   },
   "devDependencies": {
@@ -95,5 +96,5 @@
     "express": "^4.18.2",
     "jest": "^27.5.1"
   },
-  "gitHead": "3150c3a5c3e041fe7f5a3902418d9d62adee3173"
+  "gitHead": "d357376aa3df9ef789befc7f548629deecb04d96"
 }

package/lib/states/challenge.js

@@ -1,58 +0,0 @@
-const Mcrypto = require('@ocap/mcrypto');
-const { fromPublicKey } = require('@ocap/wallet');
-const { stripHexPrefix } = require('@ocap/util');
-const logger = require('@abtnode/logger')('@abtnode/core:states:challenge');
-
-const BaseState = require('./base');
-
-class ChallengeState extends BaseState {
-  /**
-   * Creates an instance of ChallengeState
-   * @param {string} baseDir
-   * @param {object} config
-   * @memberof ChallengeState
-   */
-  constructor(baseDir, config = {}) {
-    super(baseDir, { filename: 'challenge.db', ...config });
-  }
-
-  generate() {
-    return new Promise((resolve, reject) => {
-      const challenge = stripHexPrefix(Mcrypto.getRandomBytes(16)).toUpperCase();
-      this.insert({ challenge }, (err, data) => {
-        if (err) {
-          logger.error('generating error', { error: err });
-          return reject(err);
-        }
-
-        logger.info('generated', { data });
-        return resolve(challenge);
-      });
-    });
-  }
-
-  verify(challenge, publicKey, signature) {
-    return new Promise((resolve, reject) => {
-      if (!signature) {
-        reject(new Error('Cannot verify challenge without signature.'));
-        return;
-      }
-
-      this.findOne({ challenge }, (err, data) => {
-        if (err) {
-          logger.error('error find challenge', { error: err });
-        }
-
-        if (data) {
-          // TODO: we may need wallet type info here from wallet did
-          const wallet = fromPublicKey(publicKey);
-          return resolve(wallet.verify(challenge, signature));
-        }
-
-        return resolve(false);
-      });
-    });
-  }
-}
-
-module.exports = ChallengeState;