@abtnode/blocklet-services 1.6.26 → 1.6.29

package/README.md

@@ -5,7 +5,7 @@ Aggregator to mount all blocklet services and make them work together.
 ## Usage
 
 ```shell
-yarn add @abtnode/service-server
+yarn add @abtnode/blocklet-service
 ```
 
 Then:
@@ -17,10 +17,6 @@ const ABTNode = require('@abtnode/core');
 const node = ABTNode({ ...nodeOptions });
 const server = createServer(node, { ...globalOptions });
 
-server.mountService('@abtnode/auth-service', { ...loginServiceOptions });
-server.mountService('@abtnode/acl-service', { ...aclServiceOptions });
-server.mountService('@abtnode/payment-service', { ...paymentServiceOptions });
-
 node.onReady(() => {
   server.listen(5000, () => {
     console.log('ABT Node Service server ready on port 5000');

package/api/index.js

@@ -0,0 +1,296 @@
+const http = require('http');
+const get = require('lodash/get');
+const morgan = require('morgan');
+const express = require('express');
+require('express-async-errors');
+const cors = require('cors');
+const compression = require('compression');
+const cookieParser = require('cookie-parser');
+const bodyParser = require('body-parser');
+const RotatingFileStream = require('rotating-file-stream');
+const httpProxy = require('http-proxy');
+const moment = require('dayjs');
+const minimatch = require('minimatch');
+
+const { WELLKNOWN_SERVICE_PATH_PREFIX, NODE_SERVICES_PREFIX, EVENTS } = require('@abtnode/constant');
+const { BlockletEvents } = require('@blocklet/meta/lib/constants');
+const normalizePathPrefix = require('@abtnode/util/lib/normalize-path-prefix');
+const eventHub =
+  process.env.NODE_ENV === 'test' ? require('@arcblock/event-hub/single') : require('@arcblock/event-hub');
+const logger = require('@abtnode/logger')(require('../package.json').name);
+
+const cache = require('./cache');
+const { ensureProxyUrl } = require('./util');
+const { isProduction, isE2E } = require('./libs/env');
+const { init: initStates } = require('./state/index');
+
+const { init: initNotification } = require('./services/notification');
+const { init: initAuth } = require('./services/auth');
+const StaticService = require('./services/static');
+const createEnvRoutes = require('./routes/env');
+const createBlockletRoutes = require('./routes/blocklet');
+const checkRunning = require('./middlewares/check-running');
+const attachSharedUtils = require('./util/attach-shared-utils');
+
+const logFileGenerator = (time, index) => {
+  if (!time) {
+    return 'service.log';
+  }
+
+  let filename = `service-${moment(time).subtract(1, 'day').format('YYYY-MM-DD')}`; // prev date
+
+  if (index > 1) {
+    filename = `${filename}-${index}`;
+  }
+
+  return `${filename}.log.gz`;
+};
+
+module.exports = function createServer(node, serverOptions = {}) {
+  const options = {
+    dataDir: node.dataDirs.services,
+    sessionSecret: process.env.ABT_NODE_SESSION_SECRET,
+    sessionTtl: process.env.ABT_NODE_SESSION_TTL,
+    webWalletUrl: 'https://web.abtwallet.io',
+    loginTokenKey: 'login_token',
+    ...serverOptions,
+    isProduction,
+    isE2E,
+  };
+
+  if (!options.dataDir) {
+    throw new Error('Blocklet services requires dataDir to start');
+  }
+  if (!options.sessionSecret) {
+    throw new Error('Blocklet services requires sessionSecret to start');
+  }
+
+  logger.info('init blocklet service', { isProduction });
+
+  initStates(options.dataDir);
+
+  const { middlewares: authMiddlewares, routes: authRoutes } = initAuth({ node, options });
+  const notificationService = initNotification({ node, options });
+
+  // Proxy engine
+  const proxy = httpProxy.createProxyServer({});
+  // eslint-disable-next-line no-unused-vars
+  proxy.on('proxyReq', (proxyReq, req, res) => {
+    if (req.rawBody) {
+      // Since we already consumed request stream, we need to write req.rawBody to proxy request here
+      proxyReq.write(req.rawBody);
+    }
+  });
+
+  // Cross process events
+  [BlockletEvents.updated, BlockletEvents.started, BlockletEvents.removed, BlockletEvents.statusChange].forEach(
+    (name) => {
+      eventHub.on(name, (data) => {
+        const did = get(data, 'meta.did');
+        if (did) {
+          logger.info('delete blocklet cache on update', { did, pid: process.pid });
+          cache.del(cache.keyFns.blocklet(did));
+        }
+      });
+    }
+  );
+  eventHub.on(EVENTS.NODE_UPDATED, () => {
+    logger.info('node update', { pid: process.pid });
+    cache.del(cache.keyFns.node());
+  });
+
+  // Http server
+  const server = express();
+
+  server.set('trust proxy', 'loopback');
+  server.disable('x-powered-by');
+
+  server.use(compression());
+  server.use(cookieParser());
+
+  server.use(
+    bodyParser.json({
+      // We have to set a larger hard limit since
+      limit: '2mb',
+
+      // https://flaviocopes.com/express-get-raw-body/
+      // Side effects: this will double the memory consumption
+      verify: (req, res, buf) => {
+        req.rawBody = buf;
+      },
+    })
+  );
+
+  // NOTE: will be overwrite by Blocklet Server in production
+  server.use(cors());
+
+  // Shared util functions on current request
+  server.use((req, res, next) => {
+    attachSharedUtils({ node, req, options });
+    StaticService.attachUtils({ req, res, proxy });
+    next();
+  });
+
+  /* istanbul ignore if */
+  if (isProduction) {
+    const accessLogStream = RotatingFileStream.createStream(logFileGenerator, {
+      interval: '1d',
+      path: process.env.ABT_NODE_LOG_DIR,
+      compress: 'gzip',
+    });
+    server.use(morgan('combined', { stream: accessLogStream }));
+    /* istanbul ignore else */
+  } else {
+    server.use(
+      morgan((tokens, req, res) => {
+        const log = [
+          tokens.method(req, res),
+          tokens.url(req, res),
+          tokens.status(req, res),
+          tokens.res(req, res, 'content-length'),
+          '-',
+          tokens['response-time'](req, res),
+          'ms',
+        ].join(' ');
+
+        return log;
+      })
+    );
+  }
+
+  // Static assets
+  StaticService.attachStaticResources({ app: server, proxy });
+
+  // API: notification
+  notificationService.sendToUser.attach(server);
+
+  // Middleware: auth info
+  server.use(authMiddlewares.bearerToken);
+  server.use(authMiddlewares.userInfo);
+
+  // API: auth
+  createEnvRoutes.init(server, node, options);
+  createBlockletRoutes.init(server, node, options);
+  authRoutes.attachDidAuthHandlers(server);
+  authRoutes.createPassportRoutes.init(server, node, options);
+  authRoutes.createSessionRoutes.init(server, node, options);
+
+  // Web Page
+  server.get(`${WELLKNOWN_SERVICE_PATH_PREFIX}/**`, (req, res) => {
+    res.sendWebPage();
+  });
+
+  // Middleware: check running
+  // Request would not arrive here before blocklet is installed, because there is no config in router provider(nginx)
+  server.use(checkRunning);
+
+  // Middleware: check auth
+  server.use(authMiddlewares.checkAuth);
+
+  // Block invalid path in reserved prefix
+  server.use((req, res, next) => {
+    if ([NODE_SERVICES_PREFIX.AUTH_SERVICE, WELLKNOWN_SERVICE_PATH_PREFIX].some((x) => req.path.startsWith(x))) {
+      res.status(400).send('Bad request');
+      return;
+    }
+
+    next();
+  });
+
+  // After all service middleware, we can now safely pass all traffic to blocklets
+  server.use((req, res, next) => {
+    const { target } = ensureProxyUrl(req);
+
+    if (target) {
+      proxy.web(
+        req,
+        res,
+        {
+          target,
+        },
+        (error) => {
+          if (error) {
+            console.error(error);
+            logger.error('http proxy error', { error });
+            res.status(502).send(`Can not proxy to upstream blocklet: ${target}`);
+          }
+        }
+      );
+    } else {
+      next();
+    }
+  });
+
+  // Following handlers exist just in case
+
+  // 404 handler
+  server.use((req, res) => {
+    res.status(404).send('Blocklet Service: You should not be here!');
+  });
+
+  // error handler
+  // eslint-disable-next-line no-unused-vars
+  server.use((err, req, res, next) => {
+    console.error('service error', { error: err });
+    res.status(500).send('Blocklet Service: Something broke!');
+  });
+
+  // Web socket server
+
+  // Simple websocket router like http router
+  const httpServer = http.createServer(server);
+  server.listen = httpServer.listen.bind(httpServer);
+  const wsRoutingRules = [];
+  httpServer.on('upgrade', async (req, socket, head) => {
+    attachSharedUtils({ node, req, options });
+
+    // find matched handler registered by each service
+    const { pathname } = new URL(req.url, `http://${req.headers.host || 'unknown'}`);
+    const routes = wsRoutingRules.filter((x) => minimatch(normalizePathPrefix(pathname), normalizePathPrefix(x.path)));
+
+    let routeIndex = 0;
+    const next = () => {
+      const route = routes[routeIndex];
+      if (route) {
+        routeIndex++;
+        route.handle(req, socket, head, next);
+      }
+    };
+
+    next();
+  });
+  const wsRouter = {
+    use(mountPoint, handler) {
+      wsRoutingRules.push({ path: mountPoint, handle: handler });
+    },
+  };
+
+  // Notification
+  notificationService.attach(wsRouter);
+
+  // Only for Development
+  StaticService.attachDevProxy({ wsRouter, proxy });
+
+  // Auth
+  wsRouter.use('**', authMiddlewares.ensureWsAuth);
+
+  // Final: directly proxy all websocket request to target blocklet
+  wsRouter.use('**', async (req, socket, head) => {
+    const { target } = ensureProxyUrl(req);
+    if (target) {
+      proxy.ws(req, socket, head, { target }, (error) => {
+        if (error) {
+          logger.error('socket proxy error', { from: req.url, to: target, error });
+        }
+      });
+    } else {
+      logger.error('socket proxy error: cannot find target service');
+      socket.write('HTTP/1.1 404 Not Found\r\n\r\n');
+      socket.destroy();
+    }
+  });
+
+  return server;
+};
+
+module.exports.logFileGenerator = logFileGenerator;

package/{services/auth → api}/libs/auth.js

@@ -5,6 +5,7 @@ const DiskStorage = require('@arcblock/did-auth-storage-nedb');
 const { WalletAuthenticator } = require('@arcblock/did-auth');
 const WalletHandlers = require('@blocklet/sdk/lib/wallet-handler');
 const sendNotification = require('@blocklet/sdk/lib/util/send-notification');
+const { WELLKNOWN_SERVICE_PATH_PREFIX, NODE_SERVICES_PREFIX } = require('@abtnode/constant');
 
 const { getBlockletLogo } = require('../util');
 
@@ -25,14 +26,18 @@ module.exports = (node, opts) => {
       ]);
 
       // logo
-      const logo = getBlockletLogo({ baseUrl: joinUrl(baseUrl, opts.prefix), blocklet, nodeInfo: info });
+      const logo = getBlockletLogo({
+        baseUrl: joinUrl(baseUrl, WELLKNOWN_SERVICE_PATH_PREFIX),
+        blocklet,
+        nodeInfo: info,
+      });
 
       return {
         name: meta.name,
         description: meta.description || `Login to ${meta.name}`,
         icon: logo,
         updateSubEndpoint: true,
-        subscriptionEndpoint: joinUrl(groupPathPrefix, opts.prefix, 'websocket'),
+        subscriptionEndpoint: joinUrl(groupPathPrefix, WELLKNOWN_SERVICE_PATH_PREFIX, 'websocket'),
         nodeDid: info.did,
       };
     },
@@ -48,12 +53,8 @@ module.exports = (node, opts) => {
     },
   });
 
-  const handlers = new WalletHandlers({
+  const handlerOpts = {
     authenticator,
-    options: {
-      prefix: `${opts.prefix}/api/did`,
-    },
-    tokenGenerator: () => Date.now().toString(),
     tokenStorage: new DiskStorage({
       dbPath: path.join(opts.dataDir, 'auth.db'),
       dbPort: process.env.NODE_ENV === 'test' ? null : Number(process.env.NEDB_MULTI_PORT),
@@ -67,10 +68,25 @@ module.exports = (node, opts) => {
       };
       return sendNotification(connectedDid, message, sender, process.env.ABT_NODE_SERVICE_PORT);
     },
+  };
+
+  // backward compatible
+  const oldHandler = new WalletHandlers({
+    ...handlerOpts,
+    options: {
+      prefix: `${NODE_SERVICES_PREFIX.AUTH_SERVICE}/api/did`,
+    },
+  });
+
+  const handler = new WalletHandlers({
+    ...handlerOpts,
+    options: {
+      prefix: `${WELLKNOWN_SERVICE_PATH_PREFIX}/api/did`,
+    },
   });
 
   return {
     authenticator,
-    handlers,
+    handlers: [handler, oldHandler],
   };
 };

package/api/libs/env.js

@@ -0,0 +1,7 @@
+const isProduction = process.env.NODE_ENV === 'production';
+const isE2E = process.env.NODE_ENV === 'e2e';
+
+module.exports = {
+  isProduction,
+  isE2E,
+};

package/{services/auth → api}/libs/jwt.js

@@ -3,6 +3,7 @@ const jwt = require('jsonwebtoken');
 const { createAuthToken } = require('@abtnode/auth/lib/auth');
 const { isUserPassportRevoked } = require('@abtnode/auth/lib/passport');
 
+// FIXME: we need to test performance for this code
 const getUser = async (node, teamDid, userDid) => {
   const user = await node.getUser({ teamDid, user: { did: userDid } });
   if (user && user.approved) {

package/api/middlewares/check-admin-permission.js

@@ -0,0 +1,15 @@
+const { ROLES } = require('@abtnode/constant');
+
+module.exports = async (req, res, next) => {
+  if (!req.user) {
+    res.status(401).json({ code: 'forbidden', error: 'not authorized' });
+    return;
+  }
+
+  if (![ROLES.OWNER, ROLES.ADMIN].includes(req.user.role)) {
+    res.status(403).json({ code: 'forbidden', error: 'no permission' });
+    return;
+  }
+
+  next();
+};

package/api/middlewares/check-running.js

@@ -0,0 +1,46 @@
+const joinUrl = require('url-join');
+
+const { BlockletStatus } = require('@blocklet/meta/lib/constants');
+const { WELLKNOWN_SERVICE_PATH_PREFIX } = require('@abtnode/constant');
+const getBlockletNotRunningTemplate = require('@abtnode/router-templates/lib/blocklet-not-running');
+const getBlockletMaintenanceTemplate = require('@abtnode/router-templates/lib/blocklet-maintenance');
+const { getBaseUrl } = require('@abtnode/router-adapter');
+
+const { shouldGotoStartPage } = require('../util');
+
+const checkRunning = async (req, res, next) => {
+  const blocklet = await req.getBlocklet();
+  if (
+    ![
+      BlockletStatus.running,
+      // Waiting, Downloading should be allowed because blocklet is currently being upgrading
+      BlockletStatus.waiting,
+      BlockletStatus.downloading,
+    ].includes(blocklet.status)
+  ) {
+    if (shouldGotoStartPage(req)) {
+      const baseUrl = getBaseUrl(req);
+      const redirect = encodeURIComponent(
+        `${baseUrl.replace(/\/+$/, '')}/${req.url.replace(/^\/+/, '').replace('__start__=1', '')}`
+      );
+      res.redirect(joinUrl(baseUrl, WELLKNOWN_SERVICE_PATH_PREFIX, '/start', `/?redirect=${redirect}`));
+      return;
+    }
+
+    const nodeInfo = await req.getNodeInfo();
+    res.status(500);
+    // return json if json has high priority, e.g. "*/*,application/json"
+    if (req.accepts(['html', 'json']) === 'json') {
+      res.json({ code: 'error', error: 'blocklet is not running' });
+    } else if (blocklet.status === BlockletStatus.starting) {
+      res.send(getBlockletMaintenanceTemplate(blocklet, nodeInfo));
+    } else {
+      res.send(getBlockletNotRunningTemplate(blocklet, nodeInfo));
+    }
+    return;
+  }
+
+  next();
+};
+
+module.exports = checkRunning;

package/api/routes/blocklet.js

@@ -0,0 +1,98 @@
+/* eslint-disable no-console */
+const fs = require('fs');
+const path = require('path');
+const get = require('lodash/get');
+const cloneDeep = require('lodash/cloneDeep');
+
+const { fixBlockletStatus, wipeSensitiveData } = require('@blocklet/meta/lib/util');
+const { WELLKNOWN_SERVICE_PATH_PREFIX } = require('@abtnode/constant');
+const logger = require('@abtnode/logger')(require('../../package.json').name);
+
+const checkAdminPermission = require('../middlewares/check-admin-permission');
+
+const polishBlocklet = (doc) => {
+  const res = cloneDeep(doc);
+  fixBlockletStatus(res);
+  return wipeSensitiveData(res);
+};
+
+const prefix = WELLKNOWN_SERVICE_PATH_PREFIX;
+
+module.exports = {
+  init(server, node) {
+    server.get(`${prefix}/blocklet/logo`, async (req, res) => {
+      const sendOptions = { maxAge: '1d' };
+
+      try {
+        const blocklet = await req.getBlocklet();
+
+        if (blocklet && get(blocklet, 'env.appDir') && blocklet.meta.logo) {
+          const logoFile = path.join(get(blocklet, 'env.appDir'), blocklet.meta.logo);
+
+          if (fs.existsSync(logoFile)) {
+            res.sendFile(logoFile, sendOptions);
+            return;
+          }
+        }
+
+        res.sendStaticFile('/images/blocklet.png', sendOptions);
+      } catch (err) {
+        logger.error('failed to send blocklet logo', { did: req.params.did, error: err });
+        res.sendStaticFile('/images/blocklet.png', sendOptions);
+      }
+    });
+
+    server.get(`${prefix}/api/blocklet/detail`, checkAdminPermission, async (req, res) => {
+      const blocklet = await req.getBlocklet();
+
+      res.json(polishBlocklet(blocklet));
+    });
+
+    server.post(`${prefix}/api/blocklet/config`, checkAdminPermission, async (req, res) => {
+      const blocklet = await req.getBlocklet();
+      const { configs, childDid } = req.body;
+      const doc = await node.configBlocklet({
+        did: blocklet.meta.did,
+        childDid,
+        configs,
+      });
+
+      res.json(polishBlocklet(doc));
+    });
+
+    server.post(`${prefix}/api/blocklet/start`, checkAdminPermission, async (req, res) => {
+      const blocklet = await req.getBlocklet();
+      const doc = await node.startBlocklet({
+        did: blocklet.meta.did,
+        checkHealthImmediately: true,
+        throwOnError: true,
+      });
+
+      res.json(polishBlocklet(doc));
+    });
+
+    // backward compatible
+    server.get(`${prefix}/blocklet/logo/:did`, async (req, res) => {
+      const sendOptions = { maxAge: '1d' };
+
+      let blocklet = null;
+      try {
+        blocklet = await node.ensureBlockletIntegrity(req.params.did);
+
+        if (blocklet && get(blocklet, 'env.appDir') && blocklet.meta.logo) {
+          const logoFile = path.join(get(blocklet, 'env.appDir'), blocklet.meta.logo);
+
+          if (fs.existsSync(logoFile)) {
+            res.sendFile(logoFile, sendOptions);
+            return;
+          }
+        }
+
+        res.sendStaticFile('/images/blocklet.png', sendOptions);
+      } catch (err) {
+        logger.error('failed to send blocklet logo', { did: req.params.did, error: err });
+        res.sendStaticFile('/images/blocklet.png', sendOptions);
+      }
+    });
+  },
+};

package/api/routes/env.js

@@ -0,0 +1,27 @@
+const { NODE_SERVICES } = require('@abtnode/constant');
+const { WELLKNOWN_SERVICE_PATH_PREFIX } = require('@abtnode/constant');
+
+module.exports = {
+  init(server, node, opts) {
+    server.get(`**${WELLKNOWN_SERVICE_PATH_PREFIX}/api/env`, async (req, res) => {
+      res.type('js');
+
+      const [blocklet, config, info] = await Promise.all([
+        req.getBlockletInfo(),
+        req.getServiceConfig(NODE_SERVICES.AUTH),
+        req.getNodeInfo(),
+      ]);
+      const pathPrefix = req.headers['x-path-prefix'] || '/';
+      const groupPathPrefix = req.headers['x-group-path-prefix'];
+
+      res.send(`window.env = {
+  appId: "${blocklet.did}",
+  appName: "${blocklet.name}",
+  pathPrefix: "${pathPrefix}",
+  apiPrefix: "${pathPrefix.replace(/\/+$/, '')}${WELLKNOWN_SERVICE_PATH_PREFIX}",
+  ${groupPathPrefix ? `groupPathPrefix: "${groupPathPrefix}",` : ''}
+  webWalletUrl: "${info.webWalletUrl || config.webWalletUrl || opts.webWalletUrl}",
+}`);
+    });
+  },
+};

package/{services/auth/routes → api/services/auth/connect}/invite.js

@@ -7,9 +7,10 @@ const {
   checkWalletVersion,
   beforeInvitationRequest,
 } = require('@abtnode/auth/lib/auth');
-const logger = require('@abtnode/logger')(require('../meta.json').name);
+const { NODE_SERVICES, WELLKNOWN_SERVICE_PATH_PREFIX } = require('@abtnode/constant');
+const logger = require('@abtnode/logger')(require('../../../../package.json').name);
 
-module.exports = function createRoutes(node, authenticator, login, opts) {
+module.exports = function createRoutes(node, authenticator, login) {
   return {
     action: 'invite',
 
@@ -24,7 +25,7 @@ module.exports = function createRoutes(node, authenticator, login, opts) {
       profile: async ({ extraParams, context }) => {
         const { locale } = extraParams;
 
-        const config = await context.request.getServiceConfig();
+        const config = await context.request.getServiceConfig(NODE_SERVICES.AUTH);
         const profileFields = get(config, 'profileFields');
 
         return {
@@ -53,7 +54,7 @@ module.exports = function createRoutes(node, authenticator, login, opts) {
       const { locale, inviteId } = extraParams;
       const nodeInfo = await req.getNodeInfo();
       const teamDid = req.headers['x-blocklet-did'];
-      const statusEndpointBaseUrl = joinUrl(baseUrl, opts.prefix);
+      const statusEndpointBaseUrl = joinUrl(baseUrl, WELLKNOWN_SERVICE_PATH_PREFIX);
       const endpoint = baseUrl;
 
       const { passport, response, role } = await handleInvitationResponse({

package/{services/auth/routes → api/services/auth/connect}/issue-passport.js

@@ -6,7 +6,11 @@ const {
   beforeIssuePassportRequest,
 } = require('@abtnode/auth/lib/auth');
 
-module.exports = function createRoutes(node, authenticator, login, opts) {
+const { WELLKNOWN_SERVICE_PATH_PREFIX } = require('@abtnode/constant');
+
+// todo: move to connect dir
+
+module.exports = function createRoutes(node) {
   return {
     action: 'issue-passport',
 
@@ -38,7 +42,7 @@ module.exports = function createRoutes(node, authenticator, login, opts) {
       const { locale, id } = extraParams;
       const nodeInfo = await node.getNodeInfo();
       const teamDid = req.headers['x-blocklet-did'];
-      const statusEndpointBaseUrl = joinUrl(baseUrl, opts.prefix);
+      const statusEndpointBaseUrl = joinUrl(baseUrl, WELLKNOWN_SERVICE_PATH_PREFIX);
       const endpoint = baseUrl;
 
       return handleIssuePassportResponse({