lupine.api 1.0.41

package/src/api/api-router.ts

@@ -0,0 +1,177 @@
+import { ServerResponse } from 'http';
+import { Logger } from '../lib';
+import { ServerRequest } from '../models/locals-props';
+import { handler404 } from './handle-status';
+import { SimpleStorage } from './simple-storage';
+import { ApiRouterCallback, ApiRouterData, ApiRouterMethod, IApiRouter } from '../models/api-router-props';
+const logger = new Logger('api-router');
+
+/*
+If there is a common logic to be called all endpoints, then you can set filter.
+Or, for a particular endpoint, if you want a logic to be called before any other 
+WebRouterCallback or WebRouter it can be done like this:
+
+const commLogic: WebRouterCallback = async (req: ServerRequest, res: ServerResponse) => {
+  console.log('this is called by all...');
+  return false;
+}
+router.use('/auth', commLogic, otherLogic);
+*/
+
+function isIApiRouter(handler: ApiRouterCallback | IApiRouter): handler is IApiRouter {
+  return (handler as IApiRouter).findRoute !== undefined;
+}
+
+export class ApiRouter implements IApiRouter {
+  private routerData: ApiRouterData[] = [];
+  private filter: ApiRouterCallback | undefined;
+
+  setFilter(filter: ApiRouterCallback) {
+    this.filter = filter;
+  }
+
+  // the path should start with / and end without /, and it can be
+  //    /aaa/:bbb/ccc/:ddd (ccc is a fixed section)
+  //    /aaa/:bbb/ccc/?ddd/?eee (from ddd, all sections are optional)
+  private storeRouter(path: string, handler: (ApiRouterCallback | IApiRouter)[], method: ApiRouterMethod) {
+    let fixedPath;
+    if (path === '*' || path === '/' || path === '' || path === '/*') {
+      fixedPath = '*';
+    } else {
+      fixedPath = path;
+      if (!fixedPath.startsWith('/')) {
+        fixedPath = '/' + fixedPath;
+      }
+      if (fixedPath.endsWith('/')) {
+        fixedPath = fixedPath.substring(0, fixedPath.length - 1);
+      }
+    }
+
+    let parameterLength = 0;
+    let parameterVariables: string[] = [];
+    const ind = fixedPath.indexOf('/:');
+    if (ind >= 0) {
+      parameterVariables = fixedPath.substring(ind + 1).split('/');
+      fixedPath = fixedPath.substring(0, ind);
+      // from optionInd, all will be optional
+      const optionInd = parameterVariables.findIndex((item) => item.startsWith('?'));
+      parameterLength = optionInd >= 0 ? optionInd : parameterVariables.length;
+    }
+
+    this.routerData.push({
+      path: fixedPath,
+      handler,
+      method: method,
+      parameterVariables,
+      parameterLength,
+    });
+  }
+
+  get(path: string, ...handler: (ApiRouterCallback | IApiRouter)[]) {
+    this.storeRouter(path, handler, ApiRouterMethod.GET);
+  }
+  post(path: string, ...handler: (ApiRouterCallback | IApiRouter)[]) {
+    this.storeRouter(path, handler, ApiRouterMethod.POST);
+  }
+
+  use(path: string, ...handler: (ApiRouterCallback | IApiRouter)[]) {
+    this.storeRouter(path, handler, ApiRouterMethod.ALL);
+  }
+
+  private async callHandle(
+    handle: ApiRouterCallback,
+    path: string,
+    req: ServerRequest,
+    res: ServerResponse
+  ): Promise<boolean> {
+    try {
+      if ((await handle(req, res, path)) || res.writableEnded) {
+        logger.debug(`Processed path: ${path}`);
+        return true;
+      }
+    } catch (e: any) {
+      logger.error(`Processed path: ${path}, error: ${e.message}`);
+      res.writeHead(500, { 'Content-Type': 'text/html' });
+      res.write(
+        JSON.stringify({
+          status: 'error',
+          message: `Processed path: ${path}, error: ${e.message}`,
+        })
+      );
+      res.end();
+      return true;
+    }
+    return false;
+  }
+
+  async findRoute(url: string, req: ServerRequest, res: ServerResponse, handleNotFound: boolean): Promise<boolean> {
+    for (let i = 0, routerList; (routerList = this.routerData[i]); i++) {
+      if (
+        (routerList.method === ApiRouterMethod.ALL || routerList.method === req.method) &&
+        (routerList.path === '*' || url === routerList.path || url.startsWith(routerList.path + '/'))
+      ) {
+        const parameters: { [key: string]: string } = {};
+        let meet = true;
+        if (routerList.parameterVariables.length > 0) {
+          meet = false;
+          const restPath = url.substring(routerList.path.length + 1).split('/');
+          // the path must have mandatory parameters but some parameters can be optional
+          if (
+            restPath.length >= routerList.parameterLength &&
+            restPath.length <= routerList.parameterVariables.length
+          ) {
+            meet = true;
+            for (const [index, item] of routerList.parameterVariables.entries()) {
+              if (!item.startsWith(':') && !item.startsWith('?') && item !== restPath[index]) {
+                meet = false;
+                break;
+              } else if ((item.startsWith(':') || item.startsWith('?')) && index < restPath.length) {
+                parameters[item.replace(/[:?]/g, '')] = restPath[index];
+              }
+            }
+            req.locals.urlParameters = new SimpleStorage(parameters);
+          }
+        }
+
+        if (meet) {
+          for (let j = 0, router; (router = routerList.handler[j]); j++) {
+            if (isIApiRouter(router)) {
+              // it's a sub-level router
+              const nextPath = routerList.path === '*' ? url : url.substring(routerList.path.length);
+              // the sub-level router will not have the appName
+              if ((await router.findRoute(nextPath, req, res, handleNotFound)) || res.writableEnded) {
+                return true;
+              }
+            } else {
+              // it should be a function
+              // the query's url should match the api's path
+              if (await this.callHandle(router, url, req, res)) {
+                return true;
+              }
+            }
+          }
+        }
+
+        if (handleNotFound) {
+          // no match under this path
+          logger.debug(`Processed path: ${url}, router path: ${routerList.path}`);
+          const html = JSON.stringify({
+            status: 'error',
+            message: `Can't find any matches under router ${routerList.path} for path: ${url}.`,
+          });
+          handler404(res, html);
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  async handleRequest(url: string, req: ServerRequest, res: ServerResponse): Promise<boolean> {
+    if (this.filter && (await this.callHandle(this.filter, url, req, res))) {
+      return true;
+    }
+
+    return await this.findRoute(url, req, res, true);
+  }
+}

package/src/api/api-shared-storage.ts

@@ -0,0 +1,64 @@
+/**
+ * A persistent storage for the Api
+ */
+import { IAppSharedStorage } from '../models';
+import { SimpleStorageDataProps } from '../models';
+import { apiCache } from './api-cache';
+
+// ApiSharedStorage is used in api module to store variables inside an api scope
+export class ApiSharedStorage {
+  private storage: IAppSharedStorage | undefined;
+
+  public setAppSharedStorage(storage: IAppSharedStorage): void {
+    this.storage = storage;
+  }
+  public getAppSharedStorage(): IAppSharedStorage {
+    if (!this.storage) {
+      throw new Error('AppSharedStorage not initialized');
+    }
+    return this.storage;
+  }
+
+  // called from primary before exit, or from api to save changes
+  async save() {
+    const appName = apiCache.getAppName();
+    await this.getAppSharedStorage().save(appName);
+  }
+
+  get(key: string): Promise<string> {
+    const appName = apiCache.getAppName();
+    return this.getAppSharedStorage().get(appName, key);
+  }
+  getWeb(key: string): Promise<string> {
+    const appName = apiCache.getAppName();
+    return this.getAppSharedStorage().getWeb(appName, key);
+  }
+  getApi(key: string): Promise<string> {
+    const appName = apiCache.getAppName();
+    return this.getAppSharedStorage().getApi(appName, key);
+  }
+  getWebAll(): Promise<SimpleStorageDataProps> {
+    const appName = apiCache.getAppName();
+    return this.getAppSharedStorage().getWebAll(appName);
+  }
+  getWithPrefix(prefixKey: string): Promise<SimpleStorageDataProps> {
+    const appName = apiCache.getAppName();
+    return this.getAppSharedStorage().getWithPrefix(appName, prefixKey);
+  }
+
+  set(key: string, value: any) {
+    const appName = apiCache.getAppName();
+    return this.getAppSharedStorage().set(appName, key, value);
+  }
+  setWeb(key: string, value: any) {
+    const appName = apiCache.getAppName();
+    return this.getAppSharedStorage().setWeb(appName, key, value);
+  }
+  setApi(key: string, value: any) {
+    const appName = apiCache.getAppName();
+    return this.getAppSharedStorage().setApi(appName, key, value);
+  }
+}
+
+// this can be used in app, but in api, it should use getAppStorage()
+export const apiStorage = /* @__PURE__ */ new ApiSharedStorage();

package/src/api/async-storage.ts

@@ -0,0 +1,5 @@
+// It is similar to a thread-local storage in other languages.
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { AsyncStorageProps } from '../models/async-storage-props';
+
+export const asyncLocalStorage = new AsyncLocalStorage<AsyncStorageProps>();

package/src/api/debug-service.ts

@@ -0,0 +1,56 @@
+import { IncomingMessage } from 'http';
+import { Duplex } from 'stream';
+import { MiniWebSocket } from './mini-web-socket';
+import { ShellService } from './shell-service';
+
+// This is only used in debug mode (no clusters)
+export class DebugService {
+  static clientRefreshFlag = Date.now();
+  static miniWebSocket = new MiniWebSocket(this.onMessage.bind(this));
+  static shellMap = new Map<Duplex, ShellService>();
+
+  public static onMessage(msg: string, socket: Duplex) {
+    try {
+      const json = JSON.parse(msg);
+      if (json.message === 'get-flag') {
+        this.miniWebSocket.sendMessage(
+          socket,
+          JSON.stringify({
+            message: 'flag',
+            flag: DebugService.clientRefreshFlag,
+          })
+        );
+      } else if (json.message === 'shell') {
+        console.log(json);
+        let shell: ShellService;
+        if (this.shellMap.has(socket)) {
+          shell = this.shellMap.get(socket)!;
+        } else {
+          shell = new ShellService(socket, this.miniWebSocket);
+          this.shellMap.set(socket, shell);
+          socket.on('close', () => {
+            shell.stop();
+            this.shellMap.delete(socket);
+          });
+        }
+        shell.cmd(json.cmd);
+      }
+    } catch (error) {
+      console.error(error);
+    }
+  }
+
+  public static handleUpgrade(req: IncomingMessage, socket: Duplex, head: Buffer) {
+    this.miniWebSocket.handleUpgrade(req, socket, head);
+
+    // socket.write(JSON.stringify({ message: 'flag', flag: DebugService.clientRefreshFlag }));
+  }
+
+  // broadcast to all frontend clients
+  public static broadcastRefresh() {
+    console.log(`broadcast refresh request to clients.`);
+    this.clientRefreshFlag = Date.now();
+    const msg = { message: 'Refresh', flag: this.clientRefreshFlag };
+    this.miniWebSocket.broadcast(JSON.stringify(msg));
+  }
+}

package/src/api/encode-html.ts

@@ -0,0 +1,27 @@
+export const encodeHtml = (str: string): string => {
+  return str.replace(
+    /[&<>'"]/g,
+    (tag) =>
+      ({
+        '&': '&amp;',
+        '<': '&lt;',
+        '>': '&gt;',
+        "'": '&#39;',
+        '"': '&quot;',
+      }[tag] || '')
+  );
+};
+
+export const decodeHtml = (str: string): string => {
+  return str.replace(
+    /&(\D+);/gi,
+    (tag) =>
+      ({
+        '&amp;': '&',
+        '&lt;': '<',
+        '&gt;': '>',
+        '&#39;': "'",
+        '&quot;': '"',
+      }[tag] || '')
+  );
+};

package/src/api/handle-status.ts

@@ -0,0 +1,71 @@
+import { ServerResponse } from 'http';
+import { encodeHtml } from './encode-html';
+import { IsType } from '../lib';
+
+export const handler200 = (res: ServerResponse, msg: string | object, title = '', contentType?: string) => {
+  handlerResponse(res, 200, title, msg, contentType);
+};
+
+export const handler304 = (res: ServerResponse, msg?: string | object, contentType?: string) => {
+  handlerResponse(res, 304, '304 Not Modified', msg, contentType);
+};
+
+export const handler403 = (res: ServerResponse, msg?: string | object, contentType?: string) => {
+  handlerResponse(res, 403, '403 Forbidden', msg, contentType);
+};
+
+export const handler405 = (res: ServerResponse, msg?: string | object, contentType?: string) => {
+  handlerResponse(res, 405, '405 Method Not Allowed', msg, contentType);
+};
+
+export const handler404 = (res: ServerResponse, msg?: string | object, contentType?: string) => {
+  handlerResponse(res, 404, '404 Not Found', msg, contentType);
+};
+
+export const handler416 = (res: ServerResponse, msg?: string | object, contentType?: string) => {
+  handlerResponse(res, 416, '416 Range Not Satisfiable', msg, contentType);
+};
+
+export const handler500 = (res: ServerResponse, msg?: string | object, contentType?: string) => {
+  handlerResponse(res, 500, '500 Internal Server Error', msg, contentType);
+};
+
+export const handler400 = (res: ServerResponse, msg?: string | object, contentType?: string) => {
+  handlerResponse(res, 400, '400 Bad Request', msg, contentType);
+};
+
+export const handlerResponse = (
+  res: ServerResponse,
+  statusCode: number,
+  title: string,
+  msg?: string | object,
+  contentType?: string
+) => {
+  res.statusCode = statusCode;
+  if (res.writable) {
+    try {
+      res.setHeader('content-type', contentType || 'text/html');
+    } catch (e) {
+      // errors may have been triggered when headers being sent already
+    }
+  }
+
+  const text = IsType.isObject(msg) ? JSON.stringify(msg) : (msg as string);
+  const html = contentType === 'application/json' ? msg : generateHtml(title, text || title);
+  res.end(html);
+};
+
+export const generateHtml = (title: string, message: string) => {
+  return `${[
+    '<!doctype html>',
+    '<html>',
+    '  <head>',
+    '    <meta charset="utf-8">',
+    `    <title>${title}</title>`,
+    '  </head>',
+    '  <body>',
+    `      <p>${encodeHtml(message)}</p>`,
+    '  </body>',
+    '</html>',
+  ].join('\n')}\n`;
+};

package/src/api/index.ts

@@ -0,0 +1,16 @@
+export * from './api-cache';
+export * from './api-helper';
+export * from './api-module';
+export * from './api-router';
+export * from './api-shared-storage';
+export * from './async-storage';
+export * from './debug-service';
+export * from './encode-html';
+export * from './handle-status';
+export * from './mini-web-socket';
+export * from './server-content-type';
+export * from './server-render';
+export * from './simple-storage';
+export * from './static-server';
+export * from './to-client-delivery';
+

package/src/api/mini-web-socket.ts

@@ -0,0 +1,270 @@
+import { IncomingMessage } from 'http';
+import { Duplex } from 'stream';
+import crypto from 'crypto';
+
+// https://github.com/ErickWendel/websockets-with-nodejs-from-scratch/blob/main/nodejs-raw-websocket/server.mjs
+// https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API/Writing_WebSocket_servers
+// https://github.com/websockets/ws/blob/d343a0cf7bba29a4e14217cb010446bec8fdf444/lib/receiver.js
+
+// This is a specification specific to WebSocket protocol
+const WEBSOCKET_GUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11';
+
+const MASK_KEY_BYTES_LENGTH = 4;
+// opcodes
+const OPCODE_CONTINUATION = 0x0;
+const OPCODE_TEXT = 0x1;
+const OPCODE_BINARY = 0x2;
+const OPCODE_CLOSE = 0x8;
+const OPCODE_PING = 0x9;
+const OPCODE_PONG = 0xa;
+
+// This is only used in debug mode (no clusters)
+export type MiniWebSocketMsgProps = (msg: string, socket: Duplex) => void;
+export type MiniWebSocketCloseProps = (socket: Duplex, status: string) => void;
+export class MiniWebSocket {
+  clientRefreshFlag = Date.now();
+  clients = new Set<Duplex>();
+  private _onMessage: MiniWebSocketMsgProps;
+  private _onClose?: MiniWebSocketCloseProps;
+
+  constructor(onMessage: MiniWebSocketMsgProps, onClose?: MiniWebSocketCloseProps) {
+    this._onMessage = onMessage;
+    this._onClose = onClose;
+  }
+
+  public handleUpgrade(req: IncomingMessage, socket: Duplex, head: Buffer) {
+    const key = req.headers['sec-websocket-key'];
+    const upgrade = (req.headers.upgrade || '').toString().toLowerCase();
+    const connection = (req.headers.connection || '').toString().toLowerCase();
+
+    if (!key || upgrade !== 'websocket' || !connection.includes('upgrade')) {
+      socket.write('HTTP/1.1 400 Bad Request\r\n\r\n');
+      socket.destroy();
+      return;
+    }
+
+    const cleanup = (socket: Duplex, status: string) => {
+      if (this.clients.has(socket)) this.clients.delete(socket);
+      if (!socket.destroyed) socket.destroy();
+      if (this._onClose) this._onClose(socket, status);
+    };
+    this.clients.add(socket);
+    socket.on('close', () => cleanup(socket, 'close'));
+    socket.on('error', () => cleanup(socket, 'error'));
+    socket.on('end', () => cleanup(socket, 'end'));
+    socket.write(this.handleHeaders(key));
+
+    // buffer for cumulative data, parse frame; first append head (possible residual data)
+    let buffer = head && head.length ? Buffer.from(head) : Buffer.alloc(0);
+    const onData = (chunk: Buffer) => {
+      buffer = Buffer.concat([buffer, chunk]);
+      // try to parse as many complete frames as possible
+      parseFrames();
+    };
+
+    const parseFrames = () => {
+      while (true) {
+        if (buffer.length < 2) return;
+
+        const firstByte = buffer[0];
+        const secondByte = buffer[1];
+
+        const fin = (firstByte & 0x80) !== 0; // FIN bit
+        const rsv = firstByte & 0x70; // must be 0
+        const opcode = firstByte & 0x0f;
+
+        const masked = (secondByte & 0x80) !== 0; // must be 1
+        let payloadLen = secondByte & 0x7f;
+
+        let offset = 2;
+
+        // illegal case: RSV set bits; or client not mask
+        if (rsv !== 0 || !masked) {
+          this.sendClose(socket, 1002); // protocol error
+          cleanup(socket, 'protocol error');
+          return;
+        }
+
+        // only accept single frame (FIN=1) simple message, ignore fragmented/continuation frame
+        if (!fin && opcode !== OPCODE_CONTINUATION) {
+          this.sendClose(socket, 1003); // unsupported data
+          cleanup(socket, 'unsupported opcode');
+          return;
+        }
+
+        if (payloadLen === 126) {
+          if (buffer.length < offset + 2) return;
+          payloadLen = buffer.readUInt16BE(offset);
+          offset += 2;
+        } else if (payloadLen === 127) {
+          // read 64-bit length: we only take low 32 bits (enough for your scene)
+          if (buffer.length < offset + 8) return;
+          // ignore high 32
+          payloadLen = buffer.readUInt32BE(offset + 4);
+          offset += 8;
+        }
+
+        const needed = offset + MASK_KEY_BYTES_LENGTH + payloadLen;
+        if (buffer.length < needed) return;
+
+        const maskKey = buffer.slice(offset, offset + MASK_KEY_BYTES_LENGTH);
+        offset += MASK_KEY_BYTES_LENGTH;
+
+        let payload = buffer.slice(offset, offset + payloadLen);
+        // unmask
+        for (let i = 0; i < payload.length; i++) {
+          payload[i] ^= maskKey[i % MASK_KEY_BYTES_LENGTH];
+        }
+
+        // consume this frame
+        buffer = buffer.slice(needed);
+
+        switch (opcode) {
+          case OPCODE_TEXT: {
+            const text = payload.toString('utf8');
+            try {
+              this._onMessage(text, socket);
+            } catch {
+              // ignore user callback error
+            }
+            break;
+          }
+          case OPCODE_BINARY: {
+            // don't support binary, close
+            this.sendClose(socket, 1003); // unsupported data
+            cleanup(socket, 'unsupported binary');
+            return;
+          }
+          case OPCODE_PING: {
+            // reply Pong, with same payload
+            this.safeWrite(socket, this.encodeFrame(payload, OPCODE_PONG));
+            break;
+          }
+          case OPCODE_PONG: {
+            // ignore
+            break;
+          }
+          case OPCODE_CLOSE: {
+            // parse status code
+            let code = 1000;
+            if (payload.length >= 2) {
+              code = payload.readUInt16BE(0);
+            }
+            // reply Close, then end
+            this.sendClose(socket, code);
+            setTimeout(() => {
+              try {
+                socket.end();
+              } catch {}
+              cleanup(socket, 'close');
+            }, 20);
+            return; // end loop
+          }
+          case OPCODE_CONTINUATION: {
+            // don't support continuation, close
+            this.sendClose(socket, 1003);
+            cleanup(socket, 'unsupported continuation');
+            return;
+          }
+          default: {
+            // unknown opcode
+            this.sendClose(socket, 1002);
+            cleanup(socket, 'unknown opcode');
+            return;
+          }
+        }
+      }
+    };
+    socket.on('data', onData);
+  }
+
+  private handleHeaders(key: string) {
+    const acceptKey = crypto
+      .createHash('sha1')
+      .update(key + WEBSOCKET_GUID)
+      .digest('base64');
+    const headers = [
+      'HTTP/1.1 101 Switching Protocols',
+      // 'Content-Type: text/html',
+      'Upgrade: websocket',
+      'Connection: Upgrade',
+      `Sec-WebSocket-Accept: ${acceptKey}`,
+      '\r\n',
+    ];
+    return headers.join('\r\n');
+  }
+
+  public broadcast(msg: string) {
+    const payload = Buffer.from(msg);
+    const frame = this.encodeFrame(payload, OPCODE_TEXT);
+    for (const socket of this.clients) {
+      this.safeWrite(socket, frame);
+    }
+  }
+  public sendMessage(socket: Duplex, msg: string) {
+    const payload = Buffer.from(msg);
+    this.safeWrite(socket, this.encodeFrame(payload, OPCODE_TEXT));
+  }
+
+  encodeFrame(payload: Buffer, opcode: number) {
+    const len = payload.length;
+
+    // 0x80 === 128 in binary
+    // FIN=1 + opcode
+    if (len <= 125) {
+      const header = Buffer.from([0x80 | opcode, len]);
+      return Buffer.concat([header, payload]);
+    }
+
+    if (len < 0x10000) {
+      // alloc 4 bytes
+      // [0] - 128 + 1 - 10000001  fin + opcode
+      // [1] - 126 + 0 - payload length marker + mask indicator
+      // [2] 0 - content length
+      // [3] 113 - content length
+      // [ 4 - ..] - the message itself
+      const header = Buffer.allocUnsafe(4);
+      header[0] = 0x80 | opcode;
+      header[1] = 126;
+      header.writeUInt16BE(len, 2);
+      return Buffer.concat([header, payload]);
+    }
+
+    // 64-bit length：write high 32 bits as 0, low 32 bits as len (enough for your scene)
+    const header = Buffer.allocUnsafe(10);
+    header[0] = 0x80 | opcode;
+    header[1] = 127;
+    header.writeUInt32BE(0, 2); // high 32
+    header.writeUInt32BE(len, 6); // low 32
+    return Buffer.concat([header, payload]);
+  }
+
+  sendClose(socket: Duplex, code = 1000) {
+    const payload = Buffer.allocUnsafe(2);
+    payload.writeUInt16BE(code, 0);
+    this.safeWrite(socket, this.encodeFrame(payload, OPCODE_CLOSE));
+  }
+
+  safeWrite(socket: Duplex, data: Buffer) {
+    try {
+      if (!socket.destroyed) socket.write(data);
+    } catch {
+      // ignore
+    }
+  }
+
+  close(socket: Duplex, code = 1000) {
+    this.sendClose(socket, code);
+    setTimeout(() => {
+      try {
+        socket.end();
+      } catch {}
+    }, 20);
+  }
+
+  closeAll(code = 1000) {
+    for (const s of this.clients) {
+      this.close(s, code);
+    }
+  }
+}