lupine.api 1.1.47 → 1.1.49

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lupine.api",
-  "version": "1.1.47",
+  "version": "1.1.49",
   "license": "MIT",
   "author": "uuware.com",
   "homepage": "https://github.com/uuware/lupine.js",

package/src/api/handle-status.ts

@@ -10,6 +10,10 @@ export const handler304 = (res: ServerResponse, msg?: string | object, contentTy
   handlerResponse(res, 304, '304 Not Modified', msg, contentType);
 };
 
+export const handler400 = (res: ServerResponse, msg?: string | object, contentType?: string) => {
+  handlerResponse(res, 400, '400 Bad Request', msg, contentType);
+};
+
 export const handler403 = (res: ServerResponse, msg?: string | object, contentType?: string) => {
   handlerResponse(res, 403, '403 Forbidden', msg, contentType);
 };
@@ -30,8 +34,8 @@ export const handler500 = (res: ServerResponse, msg?: string | object, contentTy
   handlerResponse(res, 500, '500 Internal Server Error', msg, contentType);
 };
 
-export const handler400 = (res: ServerResponse, msg?: string | object, contentType?: string) => {
-  handlerResponse(res, 400, '400 Bad Request', msg, contentType);
+export const handler503 = (res: ServerResponse, msg?: string | object, contentType?: string) => {
+  handlerResponse(res, 503, '503 Service Unavailable', msg, contentType);
 };
 
 export const handlerResponse = (

package/src/app/web-listener copy.ts

@@ -0,0 +1,274 @@
+import { IncomingMessage, ServerResponse } from 'http';
+import { Logger } from '../lib/logger';
+import crypto from 'crypto';
+import { parseCookies } from '../lib/utils/cookie-util';
+import { WebProcessor } from './web-processor';
+import { handler403, handler404, handler500, handler503, SimpleStorage } from '../api';
+import { JsonObject, AsyncStorageProps, ServerRequest, SetCookieProps } from '../models';
+import { HostToPath } from './host-to-path';
+import { serializeCookie } from '../lib/utils/cookie-util';
+const logger = new Logger('listener');
+
+let MAX_REQUEST_SIZE = 1024 * 1024 * 5;
+export const setMaxRequestSize = (size: number) => {
+  MAX_REQUEST_SIZE = size;
+};
+
+// The maximum number of requests being processed. If there are no requests for 10 minutes, this number will be reset to 0.
+let MAX_REQUEST_COUNT = 100;
+let REQUEST_COUNT = 0;
+export const setMaxRequestCount = (count: number) => {
+  MAX_REQUEST_COUNT = count;
+};
+export const getRequestCount = () => {
+  return REQUEST_COUNT;
+};
+
+let REQUEST_TIMEOUT = 1000 * 30;
+export const setRequestTimeout = (timeout: number) => {
+  REQUEST_TIMEOUT = timeout;
+};
+
+let accessControlAllowHosts: string[] = [];
+export const setAccessControlAllowHost = (allowHosts: string[]) => {
+  accessControlAllowHosts = allowHosts;
+};
+
+let SERVER_NAME: string = 'nginx/1.19.2';
+export const setServerName = (serverName: string) => {
+  SERVER_NAME = serverName;
+};
+
+let IP_LIMIT_RATE_PER_SECOND = 3;
+export const setIpLimitRatePerSecond = (rate: number) => {
+  IP_LIMIT_RATE_PER_SECOND = rate;
+};
+let IP_LIMIT_RATE_PER_MINUTE = 60;
+export const setIpLimitRatePerMinute = (rate: number) => {
+  IP_LIMIT_RATE_PER_MINUTE = rate;
+};
+
+let IP_BLOCK_TIME = 60_000;
+export const setIpBlockTime = (time: number) => {
+  IP_BLOCK_TIME = time;
+};
+
+export const rawMiddlewareIpLimit = (req: IncomingMessage, res: ServerResponse, next: () => void) => {
+  const clientIp = req.socket.remoteAddress || 'unknown';
+};
+
+export type RawMiddleware = (req: IncomingMessage, res: ServerResponse, next: () => void) => void;
+
+let lastRequestTime = new Date().getTime();
+
+// type ProcessRequest = (req: ServerRequest, res: ServerResponse) => void;
+export class WebListener {
+  // process requests before business logic, for example IP filter, rate limit, etc.
+  rawMiddlewares: RawMiddleware[];
+  processor: WebProcessor;
+
+  constructor(processRequest: WebProcessor) {
+    this.rawMiddlewares = [];
+    this.processor = processRequest;
+  }
+
+  addRawMiddlewareChain(middleware: RawMiddleware) {
+    this.rawMiddlewares.push(middleware);
+  }
+  async handleRawMiddlewares(req: IncomingMessage, res: ServerResponse) {
+    const runChain = (list: RawMiddleware[], context: { req: IncomingMessage; res: ServerResponse }) => {
+      const dispatch = async (i: number) => {
+        const fn = list[i];
+        if (!fn) return;
+        await fn(context.req, context.res, () => dispatch(i + 1));
+      };
+      return dispatch(0);
+    };
+    await runChain(this.rawMiddlewares, { req, res });
+  }
+
+  async listener(reqOrigin: IncomingMessage, res: ServerResponse) {
+    // If there is no request in the last 10 minutes, reset the request count.
+    if (new Date().getTime() - lastRequestTime > 1000 * 60 * 10) {
+      if (REQUEST_COUNT != 0) {
+        // in case any errors skipped (--REQUEST_COUNT)
+        logger.warn(`!!!!!!!!!! ========== REQUEST_COUNT is not counted properly: ${REQUEST_COUNT}`);
+      }
+      REQUEST_COUNT = 0;
+      lastRequestTime = new Date().getTime();
+    }
+
+    // back-pressure
+    if (REQUEST_COUNT > MAX_REQUEST_COUNT) {
+      logger.warn(`Too many requests, count: ${REQUEST_COUNT} > ${MAX_REQUEST_COUNT}`);
+      handler503(res, 'Server is busy, please retry later.');
+      return;
+    }
+
+    await this.handleRawMiddlewares(reqOrigin, res);
+    if (res.writableEnded || res.headersSent) {
+      return;
+    }
+
+    // const requestStart = process.hrtime.bigint();
+    const uuid = crypto.randomUUID();
+    const url = reqOrigin.url || '';
+    const requestInfo = `uuid: ${uuid}, Access url: ${url}`;
+    const req = reqOrigin as ServerRequest;
+
+    const host = (req.headers.host || '').split(':')[0]; // req.headers.host contains port
+    const hostPath = HostToPath.findHostPath(host);
+    if (!hostPath || !hostPath.webPath || !hostPath.appName) {
+      const msg = `Web root is not defined properly for host: ${host}.`;
+      logger.error(msg);
+      handler404(res, msg);
+      return;
+    }
+
+    REQUEST_COUNT++;
+    logger.debug(
+      `Request started. Count: ${REQUEST_COUNT}, Log uuid: ${uuid}, access: ${
+        req.headers.host
+      }, url: ${url}, time: ${new Date().toISOString()}, from: ${req.socket.remoteAddress}`
+    );
+
+    const urlSplit = url.split('?');
+    req.setTimeout(REQUEST_TIMEOUT);
+    req.on('timeout', () => {
+      REQUEST_COUNT--;
+      logger.warn('timeout');
+      req.destroy(new Error('timeout handling'));
+    });
+
+    const jsonFn = (): JsonObject | undefined => {
+      if (!req.locals._json && req.locals.body) {
+        const sBody = req.locals.body.toString();
+        if (!sBody) {
+          req.locals._json = undefined;
+        } else {
+          try {
+            req.locals._json = JSON.parse(sBody);
+          } catch (err: any) {
+            logger.warn(`JSON.parse error: ${err.message}`);
+          }
+        }
+      }
+      return req.locals._json;
+    };
+    const cookiesFn = (): SimpleStorage => {
+      if (!req.locals._cookies) {
+        req.locals._cookies = new SimpleStorage(req.headers ? parseCookies(req.headers.cookie) : {});
+      }
+      return req.locals._cookies;
+    };
+    const setCookieFn = (name: string, value: string, options: SetCookieProps): void => {
+      const cookies: string[] = [];
+      const cookiesOld = res.getHeader('Set-Cookie');
+      if (cookiesOld) {
+        if (!Array.isArray(cookiesOld)) {
+          cookies.push(cookiesOld as any);
+        } else {
+          cookies.push(...cookiesOld);
+        }
+      }
+
+      const cookiePair = serializeCookie(name, value, options);
+      cookies.push(cookiePair);
+      res.setHeader('Set-Cookie', cookies);
+
+      const localCookies = req.locals.cookies();
+      localCookies.set(name, value);
+    };
+
+    req.locals = {
+      uuid,
+      host,
+      url,
+      hostPath,
+      // urlSections: urlSplit[0].split('/').filter((i) => !!i),
+      query: new URLSearchParams(urlSplit[1] || ''),
+      urlWithoutQuery: urlSplit[0],
+      urlParameters: new SimpleStorage({}),
+      body: undefined,
+      json: jsonFn,
+      cookies: cookiesFn,
+      setCookie: setCookieFn,
+      clearCookie: (name: string) => {
+        res.setHeader('Set-Cookie', `${name}=; max-age=0`);
+      },
+    };
+
+    let bigRequest = false;
+    let totalLength = 0;
+    const bodyData: any[] = [];
+    req.on('error', (err: any) => {
+      REQUEST_COUNT--;
+      logger.error(`${requestInfo}, count: ${REQUEST_COUNT}, Request Error: `, err);
+      handler500(res, `listener error: ${err && err.message}`);
+    });
+
+    req.on('data', (chunk: any) => {
+      totalLength += chunk.length;
+      logger.debug(`${requestInfo}, Request data length: ${chunk.length}, total: ${totalLength}`);
+      // Limit Request Size
+      if (!bigRequest && totalLength < MAX_REQUEST_SIZE) {
+        bodyData.push(chunk);
+      } else {
+        if (!bigRequest) {
+          bigRequest = true;
+          logger.warn(`Warn, request data is too big: ${totalLength} > ${MAX_REQUEST_SIZE}`);
+        }
+        req.socket.destroy();
+      }
+    });
+
+    req.on('end', async () => {
+      try {
+        if (bigRequest) {
+          logger.warn(`Request data is too big to process, url: ${req.locals.url}`);
+          handler403(res, `Request data is too big to process, url: ${req.locals.url}`);
+          return;
+        }
+
+        const body = Buffer.concat(bodyData);
+        const contentType = req.headers['content-type'];
+        logger.debug(`url: ${url}, Request body length: ${body.length}, contentType: ${contentType}`);
+        req.locals.body = body;
+
+        res.setHeader('Server', SERVER_NAME);
+        if (accessControlAllowHosts.includes(host)) {
+          const allowOrigin = req.headers.origin && req.headers.origin !== 'null' ? req.headers.origin : '*';
+          res.setHeader('Access-Control-Allow-Origin', allowOrigin);
+          res.setHeader('Access-Control-Allow-Credentials', 'true');
+        }
+
+        const store: AsyncStorageProps = {
+          uuid: uuid,
+          hostPath: hostPath,
+          appName: hostPath.appName,
+          locals: req.locals,
+          lang: req.locals.cookies().get('lang', 'en') || 'en',
+        };
+        await this.processor.processRequest(store, req, res);
+      } finally {
+        REQUEST_COUNT--;
+      }
+      // await new Promise(resolve => setTimeout(resolve, 3000));
+
+      // asyncLocalStorage.run(store, async () => {
+      //   try {
+      //     await onEnd();
+      //   } catch (error: any) {
+      //     logger.error(`url: ${url}, Request end error: `, error.message);
+      //   }
+
+      //   lastRequestTime = new Date().getTime();
+      //   const requestEnd = process.hrtime.bigint();
+      //   REQUEST_COUNT--;
+      //   logger.debug(
+      //     `Request finished. Count: ${REQUEST_COUNT}, url: ${url}, time: ${new Date().toISOString()}, duration: ${Number(requestEnd - requestStart) / 1000000} ms`
+      //   );
+      // });
+    });
+  }
+}

package/src/app/web-listener.ts

@@ -3,7 +3,7 @@ import { Logger } from '../lib/logger';
 import crypto from 'crypto';
 import { parseCookies } from '../lib/utils/cookie-util';
 import { WebProcessor } from './web-processor';
-import { handler403, handler404, handler500, SimpleStorage } from '../api';
+import { handler403, handler404, handler500, handler503, SimpleStorage } from '../api';
 import { JsonObject, AsyncStorageProps, ServerRequest, SetCookieProps } from '../models';
 import { HostToPath } from './host-to-path';
 import { serializeCookie } from '../lib/utils/cookie-util';
@@ -15,7 +15,7 @@ export const setMaxRequestSize = (size: number) => {
 };
 
 // The maximum number of requests being processed. If there are no requests for 10 minutes, this number will be reset to 0.
-let MAX_REQUEST_COUNT = 1024 * 1;
+let MAX_REQUEST_COUNT = 100;
 let REQUEST_COUNT = 0;
 export const setMaxRequestCount = (count: number) => {
   MAX_REQUEST_COUNT = count;
@@ -39,17 +39,37 @@ export const setServerName = (serverName: string) => {
   SERVER_NAME = serverName;
 };
 
+export type RawMiddleware = (req: IncomingMessage, res: ServerResponse, next: () => void) => void;
+
 let lastRequestTime = new Date().getTime();
 
 // type ProcessRequest = (req: ServerRequest, res: ServerResponse) => void;
 export class WebListener {
+  // process requests before business logic, for example IP filter, rate limit, etc.
+  rawMiddlewares: RawMiddleware[];
   processor: WebProcessor;
 
   constructor(processRequest: WebProcessor) {
+    this.rawMiddlewares = [];
     this.processor = processRequest;
   }
 
-  listener(reqOrigin: IncomingMessage, res: ServerResponse) {
+  addRawMiddlewareChain(middleware: RawMiddleware) {
+    this.rawMiddlewares.push(middleware);
+  }
+  async handleRawMiddlewares(req: IncomingMessage, res: ServerResponse) {
+    const runChain = (list: RawMiddleware[], context: { req: IncomingMessage; res: ServerResponse }) => {
+      const dispatch = async (i: number) => {
+        const fn = list[i];
+        if (!fn) return;
+        await fn(context.req, context.res, () => dispatch(i + 1));
+      };
+      return dispatch(0);
+    };
+    await runChain(this.rawMiddlewares, { req, res });
+  }
+
+  async listener(reqOrigin: IncomingMessage, res: ServerResponse) {
     // If there is no request in the last 10 minutes, reset the request count.
     if (new Date().getTime() - lastRequestTime > 1000 * 60 * 10) {
       if (REQUEST_COUNT != 0) {
@@ -60,9 +80,15 @@ export class WebListener {
       lastRequestTime = new Date().getTime();
     }
 
+    // back-pressure
     if (REQUEST_COUNT > MAX_REQUEST_COUNT) {
       logger.warn(`Too many requests, count: ${REQUEST_COUNT} > ${MAX_REQUEST_COUNT}`);
-      handler403(res, 'Too many requests');
+      handler503(res, 'Server is busy, please retry later.');
+      return;
+    }
+
+    await this.handleRawMiddlewares(reqOrigin, res);
+    if (res.writableEnded || res.headersSent) {
       return;
     }
 
@@ -78,7 +104,7 @@ export class WebListener {
       const msg = `Web root is not defined properly for host: ${host}.`;
       logger.error(msg);
       handler404(res, msg);
-      return true;
+      return;
     }
 
     REQUEST_COUNT++;

package/src/app/web-server.ts

@@ -4,7 +4,7 @@ import { WebListener } from './web-listener';
 import * as fs from 'fs';
 import * as http from 'http';
 import * as https from 'https';
-import { IncomingMessage } from 'http';
+import { IncomingMessage, ServerResponse } from 'http';
 import { Duplex } from 'stream';
 import { WebProcessor } from './web-processor';
 import { DebugService } from '../api/debug-service';
@@ -23,13 +23,25 @@ export class WebServer {
       DebugService.handleUpgrade(req, socket, head);
     } else {
       logger.error(`Unexpected web socket access: ${req.url} from ${clientIp}`);
-      socket.write("HTTP/1.1 404 Not Found\r\n\r\n");
+      socket.write('HTTP/1.1 404 Not Found\r\n\r\n');
       socket.destroy();
     }
   }
 
+  async listenerWrap(reqOrigin: IncomingMessage, res: ServerResponse) {
+    try {
+      await this.webListener.listener.bind(this.webListener)(reqOrigin, res);
+    } catch (err) {
+      console.error('Request error:', err);
+      if (!res.headersSent) {
+        res.statusCode = 500;
+        res.end('Internal Server Error');
+      }
+    }
+  }
+
   startHttp(httpPort: number, bindIp?: string, timeout?: number) {
-    const httpServer = http.createServer(this.webListener.listener.bind(this.webListener));
+    const httpServer = http.createServer(this.listenerWrap.bind(this));
     if (typeof timeout === 'number') httpServer.setTimeout(timeout);
 
     httpServer.on('upgrade', this.handleUpgrade.bind(this));
@@ -69,7 +81,7 @@ export class WebServer {
       );
     }
 
-    const httpsServer = https.createServer(httpsOptions, this.webListener.listener.bind(this.webListener));
+    const httpsServer = https.createServer(httpsOptions, this.listenerWrap.bind(this));
     httpsServer.on('upgrade', this.handleUpgrade.bind(this));
 
     if (typeof timeout === 'number') {

package/src/lib/utils/fs-utils.ts

@@ -124,21 +124,21 @@ export class FsUtils {
   };
   private static getDirsFullpathDepthSub = async (dirPath: string, depth = 0, maxDepth = 1): Promise<Dirent[]> => {
     try {
+      const ret = [];
       const files = await fs.readdir(dirPath, {
         recursive: false,
         withFileTypes: true,
       });
+      ret.push(...files);
       if (depth + 1 < maxDepth) {
         for (const entry of files) {
           if (entry.isDirectory()) {
-            if (depth < maxDepth) {
-              const fullPath = path.join(dirPath, entry.name);
-              (entry as any).sub = await this.getDirsFullpathDepthSub(fullPath, depth + 1, maxDepth);
-            }
+            const fullPath = path.join(dirPath, entry.name);
+            ret.push(...(await this.getDirsFullpathDepthSub(fullPath, depth + 1, maxDepth)));
           }
         }
       }
-      return files;
+      return ret;
     } catch {
       return [];
     }

package/src/lib/utils/is-type.ts

@@ -17,7 +17,7 @@ export class IsType {
 
   static isObject(input: any) {
     // null is not object
-    return input != null && Object.prototype.toString.call(input) === '[object Object]';
+    return input !== null && Object.prototype.toString.call(input) === '[object Object]';
   }
 
   static isObjectEmpty(obj: any) {