lupine.api 1.0.41

package/src/app/process-dev-requests.ts

@@ -0,0 +1,90 @@
+import { Logger } from '../lib/logger';
+import { ServerResponse } from 'http';
+import { AddressInfo } from 'net';
+import { appLoader } from './app-loader';
+import { DebugService } from '../api/debug-service';
+import { AppCacheGlobal, AppCacheKeys, getAppCache, ServerRequest } from '../models';
+import { cleanupAndExit } from './cleanup-exit';
+const logger = new Logger('process-dev-requests');
+
+function deleteRequireCache(moduleName: string) {
+  var solvedName = require.resolve(moduleName),
+    nodeModule = require.cache[solvedName];
+  if (nodeModule) {
+    for (var i = 0; i < nodeModule.children.length; i++) {
+      var child = nodeModule.children[i];
+      deleteRequireCache(child.filename);
+    }
+    delete require.cache[solvedName];
+  }
+}
+
+// this is a worker and msg is from Primary
+// when debug is on, it's in primary, but it shouldn't receive those msgs
+export const processDebugMessage = async (msgObject: any) => {
+  logger.info(`processDebugMessage, id: ${msgObject && msgObject.id}, message: ${msgObject && msgObject.message}`);
+  if (msgObject.id === 'debug' && msgObject.message === 'refresh') {
+    if (msgObject.appName) {
+      const appConfig = getAppCache().get(msgObject.appName, AppCacheKeys.API_CONFIG);
+      appLoader.refreshApi(appConfig);
+    } else {
+      // refresh all in a worker (app scope)
+      let appList = getAppCache().get(AppCacheGlobal, AppCacheKeys.APP_LIST);
+      for (const appName of appList) {
+        const appConfig = getAppCache().get(appName, AppCacheKeys.API_CONFIG);
+        appLoader.refreshApi(appConfig);
+      }
+    }
+
+    // TemplateCache should be only used in api scope, so shouldn't clear it here
+    // apiCache.clearTemplateCache();
+
+    // this only works in debug mode (no clusters)
+    DebugService.broadcastRefresh();
+  }
+  if (msgObject.id === 'debug' && msgObject.message === 'suspend') {
+    // Only when it's debug mode, it can go here, otherwise suspend should be processed in processMessageFromWorker
+    console.log(`[server] Received suspend command.`);
+    cleanupAndExit();
+  }
+};
+
+export async function processRefreshCache(req: ServerRequest) {
+  // if this is a child process, we need to notice parent process to broadcast to all clients to refresh
+  if (process.send) {
+    const appName = req.locals.query.get('appName');
+    process.send({ id: 'debug', message: 'refresh', appName });
+  }
+  // if it's debug mode (only one process)
+  else {
+    // if (getAppCache().get(APP_GLOBAL, AppCacheKeys.API_DEBUG) === true)
+    const appName = req.locals.query.get('appName');
+    processDebugMessage({ id: 'debug', message: 'refresh', appName });
+  }
+}
+
+// this is only for local development
+export async function processDevRequests(req: ServerRequest, res: ServerResponse, rootUrl?: string) {
+  res.end();
+  const address = req.socket.address() as AddressInfo;
+  if (address.address !== '127.0.0.1') {
+    console.log(`[server] Ignore request from: `, req.url, address.address);
+    return true;
+  }
+  if (req.url === '/debug/suspend') {
+    console.log(`[server] Received suspend command.`);
+    if (process.send) {
+      // send to parent process to kill all
+      process.send({ id: 'debug', message: 'suspend' });
+    }
+    // if it's debug mode (only one process)
+    else if (getAppCache().get(AppCacheGlobal, AppCacheKeys.APP_DEBUG) === true) {
+      await processDebugMessage({ id: 'debug', message: 'suspend' });
+    }
+  } else if (req.url === '/debug/refresh') {
+    await processRefreshCache(req);
+  }
+  if (req.url === '/debug/client') {
+  }
+  return true;
+}

package/src/app/web-listener.ts

@@ -0,0 +1,230 @@
+import { IncomingMessage, ServerResponse } from 'http';
+import { Logger } from '../lib/logger';
+import crypto from 'crypto';
+import { parseCookies } from '../lib/utils/cookie-util';
+import { WebProcessor } from './web-processor';
+import { handler403, handler404, handler500, SimpleStorage } from '../api';
+import { JsonObject, AsyncStorageProps, ServerRequest, SetCookieProps } from '../models';
+import { HostToPath } from './host-to-path';
+import { serializeCookie } from '../lib/utils/cookie-util';
+const logger = new Logger('listener');
+
+let MAX_REQUEST_SIZE = 1024 * 1024 * 5;
+export const setMaxRequestSize = (size: number) => {
+  MAX_REQUEST_SIZE = size;
+};
+
+// The maximum number of requests being processed. If there are no requests for 10 minutes, this number will be reset to 0.
+let MAX_REQUEST_COUNT = 1024 * 1;
+let REQUEST_COUNT = 0;
+export const setMaxRequestCount = (count: number) => {
+  MAX_REQUEST_COUNT = count;
+};
+export const getRequestCount = () => {
+  return REQUEST_COUNT;
+};
+
+let REQUEST_TIMEOUT = 1000 * 30;
+export const setRequestTimeout = (timeout: number) => {
+  REQUEST_TIMEOUT = timeout;
+};
+
+let accessControlAllowHosts: string[] = [];
+export const setAccessControlAllowHost = (allowHosts: string[]) => {
+  accessControlAllowHosts = allowHosts;
+};
+
+let SERVER_NAME: string = 'nginx/1.19.2';
+export const setServerName = (serverName: string) => {
+  SERVER_NAME = serverName;
+};
+
+let lastRequestTime = new Date().getTime();
+
+// type ProcessRequest = (req: ServerRequest, res: ServerResponse) => void;
+export class WebListener {
+  processor: WebProcessor;
+
+  constructor(processRequest: WebProcessor) {
+    this.processor = processRequest;
+  }
+
+  listener(reqOrigin: IncomingMessage, res: ServerResponse) {
+    // If there is no request in the last 10 minutes, reset the request count.
+    if (new Date().getTime() - lastRequestTime > 1000 * 60 * 10) {
+      if (REQUEST_COUNT != 0) {
+        // in case any errors skipped (--REQUEST_COUNT)
+        logger.warn(`!!!!!!!!!! ========== REQUEST_COUNT is not counted properly: ${REQUEST_COUNT}`);
+      }
+      REQUEST_COUNT = 0;
+      lastRequestTime = new Date().getTime();
+    }
+
+    if (REQUEST_COUNT > MAX_REQUEST_COUNT) {
+      logger.warn(`Too many requests, count: ${REQUEST_COUNT} > ${MAX_REQUEST_COUNT}`);
+      handler403(res, 'Too many requests');
+      return;
+    }
+
+    // const requestStart = process.hrtime.bigint();
+    const uuid = crypto.randomUUID();
+    const url = reqOrigin.url || '';
+    const requestInfo = `uuid: ${uuid}, Access url: ${url}`;
+    const req = reqOrigin as ServerRequest;
+
+    const host = (req.headers.host || '').split(':')[0]; // req.headers.host contains port
+    const hostPath = HostToPath.findHostPath(host);
+    if (!hostPath || !hostPath.webPath || !hostPath.appName) {
+      const msg = `Web root is not defined properly for host: ${host}.`;
+      logger.error(msg);
+      handler404(res, msg);
+      return true;
+    }
+
+    REQUEST_COUNT++;
+    logger.debug(
+      `Request started. Count: ${REQUEST_COUNT}, Log uuid: ${uuid}, access: ${
+        req.headers.host
+      }, url: ${url}, time: ${new Date().toISOString()}, from: ${req.socket.remoteAddress}`
+    );
+
+    const urlSplit = url.split('?');
+    req.setTimeout(REQUEST_TIMEOUT);
+    req.on('timeout', () => {
+      REQUEST_COUNT--;
+      logger.warn('timeout');
+      req.destroy(new Error('timeout handling'));
+    });
+
+    const jsonFn = (): JsonObject | undefined => {
+      if (!req.locals._json && req.locals.body) {
+        const sBody = req.locals.body.toString();
+        if (!sBody) {
+          req.locals._json = undefined;
+        } else {
+          try {
+            req.locals._json = JSON.parse(sBody);
+          } catch (err: any) {
+            logger.warn(`JSON.parse error: ${err.message}`);
+          }
+        }
+      }
+      return req.locals._json;
+    };
+    const cookiesFn = (): SimpleStorage => {
+      if (!req.locals._cookies) {
+        req.locals._cookies = new SimpleStorage(req.headers ? parseCookies(req.headers.cookie) : {});
+      }
+      return req.locals._cookies;
+    };
+    const setCookieFn = (name: string, value: string, options: SetCookieProps): void => {
+      const cookies: string[] = [];
+      const cookiesOld = res.getHeader('Set-Cookie');
+      if (cookiesOld) {
+        if (!Array.isArray(cookiesOld)) {
+          cookies.push(cookiesOld as any);
+        } else {
+          cookies.push(...cookiesOld);
+        }
+      }
+
+      const cookiePair = serializeCookie(name, value, options);
+      cookies.push(cookiePair);
+      res.setHeader('Set-Cookie', cookies);
+
+      const localCookies = req.locals.cookies();
+      localCookies.set(name, value);
+    };
+
+    req.locals = {
+      uuid,
+      host,
+      url,
+      hostPath,
+      // urlSections: urlSplit[0].split('/').filter((i) => !!i),
+      query: new URLSearchParams(urlSplit[1] || ''),
+      urlWithoutQuery: urlSplit[0],
+      urlParameters: new SimpleStorage({}),
+      body: undefined,
+      json: jsonFn,
+      cookies: cookiesFn,
+      setCookie: setCookieFn,
+      clearCookie: (name: string) => {
+        res.setHeader('Set-Cookie', `${name}=; max-age=0`);
+      },
+    };
+
+    let bigRequest = false;
+    let totalLength = 0;
+    const bodyData: any[] = [];
+    req.on('error', (err: any) => {
+      REQUEST_COUNT--;
+      logger.error(`${requestInfo}, count: ${REQUEST_COUNT}, Request Error: `, err);
+      handler500(res, `listener error: ${err && err.message}`);
+    });
+
+    req.on('data', (chunk: any) => {
+      totalLength += chunk.length;
+      logger.debug(`${requestInfo}, Request data length: ${chunk.length}, total: ${totalLength}`);
+      // Limit Request Size
+      if (!bigRequest && totalLength < MAX_REQUEST_SIZE) {
+        bodyData.push(chunk);
+      } else {
+        if (!bigRequest) {
+          bigRequest = true;
+          logger.warn(`Warn, request data is too big: ${totalLength} > ${MAX_REQUEST_SIZE}`);
+        }
+        req.socket.destroy();
+      }
+    });
+
+    req.on('end', async () => {
+      try {
+        if (bigRequest) {
+          logger.warn(`Request data is too big to process, url: ${req.locals.url}`);
+          handler403(res, `Request data is too big to process, url: ${req.locals.url}`);
+          return;
+        }
+
+        const body = Buffer.concat(bodyData);
+        const contentType = req.headers['content-type'];
+        logger.debug(`url: ${url}, Request body length: ${body.length}, contentType: ${contentType}`);
+        req.locals.body = body;
+
+        res.setHeader('Server', SERVER_NAME);
+        if (accessControlAllowHosts.includes(host)) {
+          const allowOrigin = req.headers.origin && req.headers.origin !== 'null' ? req.headers.origin : '*';
+          res.setHeader('Access-Control-Allow-Origin', allowOrigin);
+          res.setHeader('Access-Control-Allow-Credentials', 'true');
+        }
+
+        const store: AsyncStorageProps = {
+          uuid: uuid,
+          hostPath: hostPath,
+          appName: hostPath.appName,
+          locals: req.locals,
+          lang: req.locals.cookies().get('lang', 'en') || 'en',
+        };
+        await this.processor.processRequest(store, req, res);
+      } finally {
+        REQUEST_COUNT--;
+      }
+      // await new Promise(resolve => setTimeout(resolve, 3000));
+
+      // asyncLocalStorage.run(store, async () => {
+      //   try {
+      //     await onEnd();
+      //   } catch (error: any) {
+      //     logger.error(`url: ${url}, Request end error: `, error.message);
+      //   }
+
+      //   lastRequestTime = new Date().getTime();
+      //   const requestEnd = process.hrtime.bigint();
+      //   REQUEST_COUNT--;
+      //   logger.debug(
+      //     `Request finished. Count: ${REQUEST_COUNT}, url: ${url}, time: ${new Date().toISOString()}, duration: ${Number(requestEnd - requestStart) / 1000000} ms`
+      //   );
+      // });
+    });
+  }
+}

package/src/app/web-processor.ts

@@ -0,0 +1,42 @@
+import { ServerResponse } from 'http';
+import { Logger } from '../lib/logger';
+import { handler404 } from '../api';
+import { ApiRouterCallback, AppCacheKeys, AsyncStorageProps, getAppCache, ServerRequest } from '../models';
+const logger = new Logger('web-processor');
+
+export class WebProcessor {
+  static debugPath: string | undefined;
+  static debugHandler: ApiRouterCallback | undefined;
+
+  static enableDebug(path: string, debugHandler: ApiRouterCallback) {
+    WebProcessor.debugPath = path;
+    WebProcessor.debugHandler = debugHandler;
+  }
+
+  async processRequest(store: AsyncStorageProps, req: ServerRequest, res: ServerResponse) {
+    if (WebProcessor.debugPath && req.locals.urlWithoutQuery.startsWith(WebProcessor.debugPath)) {
+      if (WebProcessor.debugHandler) {
+        await WebProcessor.debugHandler(req, res, req.locals.urlWithoutQuery);
+        return true;
+      }
+    }
+
+    // check if the request is handled by the api
+    try {
+      const _lupineApi = getAppCache().get(store.appName, AppCacheKeys.API_MODULE);
+      if (_lupineApi && _lupineApi.processApi) {
+        const result = await _lupineApi.processApi(store, store.locals.urlWithoutQuery, req, res);
+        if (result) {
+          return true;
+        }
+      } else {
+        logger.error(`url: ${store.locals.url}, appName: ${store.appName}, no api module found`);
+      }
+    } catch (e: any) {
+      logger.error(`url: ${store.locals.url}, appName: ${store.appName}, process api error: `, e.message);
+    }
+
+    handler404(res, `Request is not processed, url: ${req.locals.url}`);
+    return true;
+  }
+}

package/src/app/web-server.ts

@@ -0,0 +1,86 @@
+import { ServerOptions } from 'https';
+import { Logger } from '../lib/logger';
+import { WebListener } from './web-listener';
+import * as fs from 'fs';
+import * as http from 'http';
+import * as https from 'https';
+import { IncomingMessage } from 'http';
+import { Duplex } from 'stream';
+import { WebProcessor } from './web-processor';
+import { DebugService } from '../api/debug-service';
+const logger = new Logger('web-server');
+
+export class WebServer {
+  webListener: WebListener;
+  constructor(webListener?: WebListener) {
+    this.webListener = webListener || new WebListener(new WebProcessor());
+  }
+
+  handleUpgrade(req: IncomingMessage, socket: Duplex, head: Buffer) {
+    const clientIp = `${(socket as any).remoteAddress}:${(socket as any).remotePort}`;
+    if (req.url?.startsWith('/debug') && socket.readable && socket.writable) {
+      logger.debug(`Upgrade WebSocket access: ${req.url} from ${clientIp}.`);
+      DebugService.handleUpgrade(req, socket, head);
+    } else {
+      logger.error(`Unexpected web socket access: ${req.url} from ${clientIp}`);
+      socket.write("HTTP/1.1 404 Not Found\r\n\r\n");
+      socket.destroy();
+    }
+  }
+
+  startHttp(httpPort: number, bindIp?: string, timeout?: number) {
+    const httpServer = http.createServer(this.webListener.listener.bind(this.webListener));
+    if (typeof timeout === 'number') httpServer.setTimeout(timeout);
+
+    httpServer.on('upgrade', this.handleUpgrade.bind(this));
+
+    httpServer.listen(httpPort, bindIp, () => {
+      logger.info(`Http Server is started: http://localhost:${httpPort}`);
+    });
+    httpServer.on('error', (error: any) => {
+      logger.error('Error occurred on http server', error);
+    });
+    return httpServer;
+  }
+
+  // multi domain https hosting
+  // https://stackoverflow.com/questions/38162077/expressjs-multi-domain-https-hosting
+  startHttps(httpsPort: number, bindIp?: string, sslKeyPath?: string, sslCrtPath?: string, timeout?: number) {
+    const httpsOptions: ServerOptions = {};
+    if (sslKeyPath && fs.existsSync(sslKeyPath) && sslCrtPath && fs.existsSync(sslCrtPath)) {
+      logger.info('Load site ssl certificate.');
+      // The options to https.createServer must include key and cert as they are required.
+      // Even though that set won't be used if SNI provides a hostname.
+      httpsOptions['key'] = fs.readFileSync(sslKeyPath, 'utf8');
+      httpsOptions['cert'] = fs.readFileSync(sslCrtPath, 'utf8');
+      // httpsOptions['ca'] = 'pem';
+      // httpsOptions['SNICallback'] = function (domain, cb) {
+      //   if (typeof sites[domain] === "undefined") {
+      //     cb(new Error("domain not found"), null);
+      //     console.log("Error: domain not found: " + domain);
+
+      //   } else {
+      //     cb(null, sites[domain].context);
+      //   }
+      // };
+    } else {
+      logger.warn(
+        `Ssl certificate is not defined or doesn't exist, key file: ${sslKeyPath}, certificate file: ${sslCrtPath}`
+      );
+    }
+
+    const httpsServer = https.createServer(httpsOptions, this.webListener.listener.bind(this.webListener));
+    httpsServer.on('upgrade', this.handleUpgrade.bind(this));
+
+    if (typeof timeout === 'number') {
+      httpsServer.setTimeout(timeout);
+    }
+    httpsServer.listen(httpsPort, bindIp, () => {
+      logger.info(`Https Server is started: https://localhost:${httpsPort}`);
+    });
+    httpsServer.on('error', (error: any) => {
+      logger.error('Error occurred on https server', error);
+    });
+    return httpsServer;
+  }
+}

package/src/common-js/web-env.js

@@ -0,0 +1,104 @@
+// This file is defined as CommonJS module, because it's used in dev as well
+const fs = require('fs/promises');
+const path = require('path');
+
+exports.readFile = async (filePath) => {
+  try {
+    const text = await fs.readFile(filePath, 'utf-8');
+    return text;
+  } catch {
+    return undefined;
+  }
+};
+
+exports.parseEnv = async (envFile) => {
+  let obj = {};
+  const text = await exports.readFile(envFile);
+  if (!text) {
+    return obj;
+  }
+
+  let lines = text.replace(/\r\n?/gm, '\n').split('\n');
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].split(/=(.*)/s);
+    const key = line[0].trim();
+
+    if (key && !key.startsWith('#')) {
+      if (key.endsWith('+')) {
+        // if the line is like [key+=...] then it will be added to the same key
+        obj[key.substring(0, key.length - 1)] += line[1] || '';
+      } else if (key.endsWith('*')) {
+        // if the line is like [key*=LINE_MARKER] then it's a multiline string, ending with "LINE_MARKER"
+        const LINE_MARKER = line[1].trim();
+        line[1] = '';
+        for (i++; i < lines.length; i++) {
+          if (lines[i].trim() === LINE_MARKER) {
+            break;
+          }
+          line[1] += lines[i] + '\n';
+        }
+        obj[key.substring(0, key.length - 1)] = line[1];
+      } else {
+        obj[key] = line[1] || ''; // .replace(/\\n/g, '\n').replace(/\\r/g, '\r')
+      }
+    } else if (key.startsWith('#!import ')) {
+      // use "#!import file_name" to import another env file
+      const file = key.substring(9).trim();
+      const newObj = await exports.parseEnv(path.join(path.dirname(envFile), file));
+      obj = Object.assign(obj, newObj);
+    }
+  }
+
+  return obj;
+};
+
+exports.copyToProcessEnv = (envObject, overrideEnv) => {
+  for (const key of Object.keys(envObject)) {
+    if (overrideEnv || typeof process.env[key] === 'undefined') {
+      process.env[key] = envObject[key];
+    }
+  }
+};
+
+exports.loadEnv = async (envFile, overrideEnv = false) => {
+  console.log(`Load env from: ${envFile}`);
+  const envObject = await exports.parseEnv(envFile);
+  exports.copyToProcessEnv(envObject, overrideEnv);
+};
+
+exports.getWebEnv = (appName) => {
+  const envWeb = {};
+  for (const envName in process.env) {
+    if (envName.startsWith(`WEB.`)) {
+      envWeb[envName.substring(4)] = process.env[envName];
+    } else if (envName.startsWith(`${appName}.WEB.`)) {
+      envWeb[envName.substring(appName.length + 5)] = process.env[envName];
+    }
+  }
+  return envWeb;
+};
+
+// defined app-shared-storage-props.ts
+const AppSharedStorageWebPrefix = 'WEB.';
+exports.getWebConfig = (allConfig) => {
+  const result = {};
+  for (let key in allConfig) {
+    if (key.startsWith(AppSharedStorageWebPrefix)) {
+      result[key.substring(AppSharedStorageWebPrefix.length)] = allConfig[key];
+    }
+  }
+  return result;
+};
+
+// // Replace <!--META-ENV-START-->...<!--META-ENV-END--> for mobile app and replace it again for web app
+// exports.replaceWebEnv = (html, appName, addMetaTag) => {
+//   const envWeb = exports.getWebEnv(appName);
+//   return html.replace(
+//     /\<\!--META-ENV-START--\>(.*?)\<\!--META-ENV-END--\>/gm,
+//     addMetaTag
+//       ? '<!--META-ENV-START--><script id="web-env" type="application/json">' +
+//           JSON.stringify(envWeb) +
+//           '</script><!--META-ENV-END-->'
+//       : '<script id="web-env" type="application/json">' + JSON.stringify(envWeb) + '</script>'
+//   );
+// };

package/src/index.ts

@@ -0,0 +1,7 @@
+export * from './models';
+export * from './lib';
+export * from './lang';
+
+export * from './admin-api';
+export * from './api';
+export * from './app';

package/src/lang/api-lang-en.ts

@@ -0,0 +1,27 @@
+import { OneLangProps } from './lang-props';
+
+export const apiLangEn: OneLangProps = {
+  langName: 'en',
+  langs: {
+    'shared:operation_success': 'Operation success',
+    'shared:operation_error': 'Operation error',
+
+    'shared:wrong_data': 'Wrong data.',
+    'shared:permission_denied': 'Permission denied.',
+    'shared:name_not_existed': '{name} not existed.',
+    'shared:name_not_set': '{name} not set.',
+    'shared:name_is_wrong': '{name} is wrong.',
+
+    'shared:login_success': 'Logged in.',
+    'shared:login_failed': 'Login failed.',
+    'shared:not_logged_in': 'Not logged in.',
+
+    'shared:user_not_found': 'User not found.',
+    'shared:user_locked': 'User locked.',
+
+    'shared:not_found_file': 'File {fileName} is not found.',
+
+    'shared:wrong_hash': 'Wrong hash.',
+    'shared:crypto_key_not_set': 'Crypto key [{cryptoKey}] not set',
+  },
+};

package/src/lang/api-lang-zh-cn.ts

@@ -0,0 +1,28 @@
+import { OneLangProps } from './lang-props';
+
+export const apiLangZhCn: OneLangProps = {
+  langName: 'zh-cn',
+  langs: {
+    'shared:operation_success': '操作成功',
+    'shared:operation_error': '操作失败',
+
+    'shared:wrong_data': '数据错误',
+    'shared:permission_denied': '权限拒绝',
+
+    'shared:name_not_existed': '{name} 不存在',
+    'shared:name_not_set': '{name} 未设置',
+    'shared:name_is_wrong': '{name} 是错误的',
+
+    'shared:login_success': '登录成功',
+    'shared:login_failed': '登录失败',
+    'shared:not_logged_in': '未登录',
+
+    'shared:user_not_found': '用户未找到',
+    'shared:user_locked': '用户已封锁',
+
+    'shared:not_found_file': '文件 {fileName} 未找到',
+
+    'shared:wrong_hash': '错误的hash',
+    'shared:crypto_key_not_set': 'Crypto key [{cryptoKey}] 未设置',
+  },
+};

package/src/lang/index.ts

@@ -0,0 +1,2 @@
+export * from './lang-helper';
+export * from './lang-props';