lupine.api 1.0.41

package/src/app/app-loader.ts

@@ -0,0 +1,62 @@
+import { Logger } from '../lib';
+import path from 'path';
+import { appCache } from './app-cache';
+import { AppCacheKeys, AppLoaderProps, HostToPathProps, IApiModule, setAppCache } from '../models';
+import { appStorage } from './app-shared-storage';
+
+class AppLoader {
+  logger: Logger = new Logger('app-loader');
+
+  constructor() {}
+
+  async loadApi(config: AppLoaderProps) {
+    // const apps: Set<string> = new Set();
+    // for (let key in config.webHostMap) {
+    //   // one app may be defined for multiple hosts, but Set only stores unique values
+    //   apps.add(config.webHostMap[key].appName);
+    //   config.webHostMap[key].webPath = path.join(config.serverRoot, config.webHostMap[key].appName + '_web');
+    //   config.webHostMap[key].dataPath = path.join(config.serverRoot, config.webHostMap[key].appName + '_data');
+    //   config.webHostMap[key].apiPath = path.join(config.serverRoot, config.webHostMap[key].appName + '_api');
+    // }
+
+    for (let appConfig of config.webHostMap) {
+      await this.callInitApi(appConfig);
+      appCache.set(appConfig.appName, AppCacheKeys.API_CONFIG, appConfig);
+    }
+  }
+
+  async callInitApi(appConfig: HostToPathProps) {
+    const apiPath = path.join(process.cwd(), 'dist/server_root', appConfig.appName + '_api/index.js');
+    try {
+      // const module = await import(apiPath);
+      const module = require(apiPath);
+      this.logger.debug(`========= ${appConfig.appName} apiModule: `, module);
+
+      if (module && module.apiModule && typeof module.apiModule.initApi === 'function') {
+        const apiModule = module.apiModule as IApiModule;
+        appCache.set(appConfig.appName, AppCacheKeys.API_MODULE, apiModule);
+
+        // getAppCache should be only called inside api scope, but set it in app scope in case it's used
+        setAppCache(appCache);
+        // setAppStorage(appStorage);
+        await apiModule.initApi(appConfig, appCache, appStorage);
+      }
+    } catch (err: any) {
+      this.logger.error(`appName: ${appConfig.appName}, load api error: `, err);
+    }
+  }
+
+  async refreshApi(appConfig: HostToPathProps) {
+    // TODO: call unloadApi?
+    const apiPath = path.join(process.cwd(), 'dist/server_root', appConfig.appName + '_api/index.js');
+    for (const path in require.cache) {
+      if (path.endsWith('.js') && path.indexOf(apiPath) <= 0) {
+        console.log(`clear cache: ${path}`);
+        delete require.cache[path];
+      }
+    }
+    await this.callInitApi(appConfig);
+  }
+}
+
+export const appLoader = /* @__PURE__ */ new AppLoader();

package/src/app/app-message.ts

@@ -0,0 +1,60 @@
+import cluster from 'cluster';
+import { Logger, LogWriter, LogWriterMessageId } from '../lib';
+import { processDebugMessage } from './process-dev-requests';
+import { appStorage } from './app-shared-storage';
+import { AppSharedStorageMessageId } from '../models';
+import { cleanupAndExit } from './cleanup-exit';
+
+const logger = new Logger('app-message');
+// send msg to all clients
+const broadcast = (msgObject: any) => {
+  for (let i in cluster.workers) {
+    if (cluster.workers[i]) cluster.workers[i].send(msgObject);
+  }
+};
+
+// this is a worker and msg is from Primary
+// when debug is on, it's in primary, but it shouldn't receive those msgs
+export const processMessageFromPrimary = (msgObject: any) => {
+  if (!msgObject || !msgObject.id) {
+    logger.warn(`Unknown message from master in work: ${cluster.worker?.id}`);
+    return;
+  }
+
+  if (msgObject.id == 'debug') {
+    processDebugMessage(msgObject);
+  } else if (msgObject.id == AppSharedStorageMessageId) {
+    appStorage.messageFromPrimaryProcess(msgObject);
+  } else {
+    logger.warn(`Unknown message: ${msgObject.id}`);
+  }
+};
+
+// this is primary, msg is from a client
+export const processMessageFromWorker = (msgObject: any) => {
+  if (!msgObject || !msgObject.id) {
+    if (msgObject['watch:require']) return;
+    logger.warn(`Unknown message from work: ${cluster.worker?.id}`);
+    return;
+  }
+
+  if (msgObject.id == LogWriterMessageId) {
+    LogWriter.messageFromSubProcess(msgObject);
+  } else if (msgObject.id == AppSharedStorageMessageId) {
+    appStorage.messageFromSubProcess(msgObject);
+  } else if (msgObject.id == 'debug') {
+    logger.debug(
+      `Message from worker ${cluster.worker?.id}, message: ${msgObject.message}, appName: ${msgObject.appName}`
+    );
+    broadcast(msgObject);
+    // if it's suspend, the primary process will exit
+    if (msgObject.message === 'suspend') {
+      setTimeout(() => {
+        console.log(`[server primary] Received suspend command.`, cluster.workers);
+        cleanupAndExit();
+      }, 100);
+    }
+  } else {
+    logger.warn(`Unknown message: ${msgObject.id}`);
+  }
+};

package/src/app/app-shared-storage.ts

@@ -0,0 +1,317 @@
+/**
+ * A persistent storage for the Api
+ */
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import cluster from 'cluster';
+import { SimpleStorage } from '../api';
+import { FsUtils, Logger } from '../lib';
+import {
+  SimpleStorageDataProps,
+  AppSharedStorageApiPrefix,
+  AppSharedStorageMessageId,
+  AppSharedStorageWebPrefix,
+  IAppSharedStorage,
+  StorageMessageFromSubProcess,
+} from '../models';
+
+// in Api scope, use ApiSharedStorage instead of this
+// storage cross clusters, loaded when start and saved before exist
+export class AppSharedStorage implements IAppSharedStorage {
+  private static instance: IAppSharedStorage;
+  configMap: { [key: string]: { fPath: string; storage: SimpleStorage } } = {};
+  logger = new Logger('server-config');
+
+  private constructor() {}
+
+  public static getInstance(): IAppSharedStorage {
+    if (!AppSharedStorage.instance) {
+      AppSharedStorage.instance = new AppSharedStorage();
+    }
+    return AppSharedStorage.instance;
+  }
+
+  private getWorker(workerId: number) {
+    for (let i in cluster.workers) {
+      if (cluster.workers[i]?.id === workerId) return cluster.workers[i];
+    }
+  }
+
+  private getStorageMap(appName: string) {
+    let storage = this.configMap[appName];
+    if (!storage) {
+      this.configMap[appName] = { fPath: '', storage: new SimpleStorage({}) };
+      storage = this.configMap[appName];
+    }
+    return storage;
+  }
+  private getStorage(appName: string): SimpleStorage {
+    return this.getStorageMap(appName).storage;
+  }
+
+  // this is primary, msg is from a client
+  messageFromSubProcess(msgObject: StorageMessageFromSubProcess) {
+    if (!cluster.isPrimary || !msgObject.action || !msgObject.key || !msgObject.appName || !msgObject.workerId) {
+      console.error('AppStorage got wrong message: ', msgObject);
+      return;
+    }
+
+    if (msgObject.action === 'get') {
+      const storage = this.getStorage(msgObject.appName);
+      const value = storage.get(msgObject.key, '');
+      // console.log(`AppStorage get value from ${msgObject.pid} in ${process.pid}, for key: ${msgObject.key}`);
+      // send message back to the worker from the worker id
+      const worker = this.getWorker(msgObject.workerId);
+      if (!worker) {
+        console.error("AppStorage can' find the worker: ", msgObject);
+        return;
+      }
+      worker.send({
+        value,
+        ...msgObject,
+      });
+    } else if (msgObject.action === 'getWithPrefix') {
+      const storage = this.getStorage(msgObject.appName);
+      const value = storage.getWithPrefix(msgObject.key);
+      // console.log(`AppStorage getWithPrefix from ${msgObject.pid} in ${process.pid}, for key: ${msgObject.key}`);
+      // send message back to the worker from the worker id
+      const worker = this.getWorker(msgObject.workerId);
+      if (!worker) {
+        console.error("AppStorage can' find the worker: ", msgObject);
+        return;
+      }
+      worker.send({
+        value: JSON.stringify(value),
+        ...msgObject,
+      });
+    } else if (msgObject.action === 'set') {
+      const storage = this.getStorage(msgObject.appName);
+      storage.set(msgObject.key, msgObject.value);
+    } else if (msgObject.action === 'save') {
+      this.save(msgObject.appName);
+    } else {
+      this.logger.warn(`Unknown message: ${msgObject.action}`);
+    }
+  }
+
+  // this is a worker and msg is from Primary
+  // when debug is on, it's in primary, but it shouldn't receive those msgs
+  // mainly for get (a worker requests a get, primaary sends the value back to here)
+  messageFromPrimaryProcess(msgObject: StorageMessageFromSubProcess) {
+    AppSharedStorageWorker.messageFromPrimaryProcess(msgObject);
+  }
+
+  // should be only called from primary when the app is starting
+  async load(appName: string, rootPath: string) {
+    if (!cluster.isPrimary) {
+      throw new Error('AppStorage.load should be only called from primary');
+    }
+
+    const map = this.getStorageMap(appName);
+    map.fPath = path.join(rootPath, 'config.json');
+
+    let tempPath = map.fPath;
+    try {
+      if (!(await FsUtils.pathExist(tempPath))) {
+        tempPath = path.join(rootPath, 'resources', 'config_default.json');
+      }
+
+      let json;
+      if ((json = await fs.readFile(tempPath, 'utf-8'))) {
+        // it will still save to map.fPath
+        map.storage.setContent(JSON.parse(json));
+        map.storage.Dirty = false;
+        this.logger.info(`Loading shared storage for [ ${appName} ] from ${tempPath}`);
+      }
+    } catch (e: any) {
+      this.logger.error('Loading json file failed: ' + tempPath, e.message);
+    }
+  }
+
+  // called from primary before exit, or from api to save changes
+  async save(appName?: string) {
+    if (!cluster.isPrimary) {
+      AppSharedStorageWorker.save(appName);
+      return;
+    }
+
+    if (appName) {
+      const map = this.configMap[appName];
+      if (map && map.fPath && map.storage.size() > 0 && map.storage.Dirty) {
+        await map.storage.saveContent(map.fPath);
+      }
+    } else {
+      // save all data
+      for (let appName in this.configMap) {
+        const map = this.configMap[appName];
+        if (map && map.fPath && map.storage.size() > 0 && map.storage.Dirty) {
+          await map.storage.saveContent(map.fPath);
+        }
+      }
+    }
+  }
+
+  // this can be called in primary or worker
+  get(appName: string, key: string): Promise<string> {
+    return new Promise((resolve, reject) => {
+      console.log(`AppStorage get value start in ${process.pid}, for key: ${key}`);
+
+      if (!cluster.isPrimary) {
+        AppSharedStorageWorker.get(appName, key, resolve, reject);
+        return;
+      }
+
+      // in primary
+      resolve(this.getStorage(appName).get(key, ''));
+    });
+  }
+  getWeb(appName: string, key: string): Promise<string> {
+    return this.get(appName, AppSharedStorageWebPrefix + key);
+  }
+  getApi(appName: string, key: string): Promise<string> {
+    return this.get(appName, AppSharedStorageApiPrefix + key);
+  }
+  getWebAll(appName: string): Promise<SimpleStorageDataProps> {
+    return this.getWithPrefix(appName, AppSharedStorageWebPrefix);
+  }
+  getWithPrefix(appName: string, prefixKey: string): Promise<SimpleStorageDataProps> {
+    return new Promise((resolve, reject) => {
+      console.log(`AppStorage getWithPrefix start in ${process.pid}, for prefixKey: ${prefixKey}`);
+
+      if (!cluster.isPrimary) {
+        AppSharedStorageWorker.getWithPrefix(appName, prefixKey, resolve, reject);
+        return;
+      }
+
+      // in primary
+      resolve(this.getStorage(appName).getWithPrefix(prefixKey));
+    });
+  }
+
+  // this can be called in primary or worker
+  set(appName: string, key: string, value: any) {
+    if (!cluster.isPrimary) {
+      AppSharedStorageWorker.set(appName, key, value);
+      return;
+    }
+
+    // in primary
+    this.getStorage(appName).set(key, value);
+  }
+  setWeb(appName: string, key: string, value: any) {
+    return this.set(appName, AppSharedStorageWebPrefix + key, value);
+  }
+  setApi(appName: string, key: string, value: any) {
+    return this.set(appName, AppSharedStorageApiPrefix + key, value);
+  }
+}
+
+type AppSharedStorageWorkerMap = { [key: string]: any };
+class AppSharedStorageWorker {
+  static handleMap: AppSharedStorageWorkerMap = {};
+  static logger = new Logger('server-config');
+
+  static messageFromPrimaryProcess(msgObject: StorageMessageFromSubProcess) {
+    if (cluster.isPrimary || !msgObject.action || !msgObject.key || !msgObject.uniqueKey) {
+      console.error('AppSharedStorageWorker got wrong message: ', msgObject);
+      return;
+    }
+
+    if (msgObject.action === 'get') {
+      console.log(`AppStorage get value end in process: ${process.pid}, for key: ${msgObject.key}`);
+
+      const value = msgObject.value;
+      // how to pass the value to the caller
+      const map = this.handleMap[msgObject.uniqueKey];
+      delete this.handleMap[msgObject.uniqueKey];
+      if (map) {
+        map.resolve(value);
+      } else {
+        throw new Error(`Unknown uniqueKey: ${msgObject.uniqueKey}`);
+      }
+    } else if (msgObject.action === 'getWithPrefix') {
+      console.log(`AppStorage get value end in process: ${process.pid}, for key: ${msgObject.key}`);
+      const value = JSON.parse(msgObject.value);
+      // how to pass the value to the caller
+      const map = this.handleMap[msgObject.uniqueKey];
+      delete this.handleMap[msgObject.uniqueKey];
+      if (map) {
+        map.resolve(value);
+      } else {
+        throw new Error(`Unknown uniqueKey: ${msgObject.uniqueKey}`);
+      }
+    } else {
+      this.logger.warn(`Unknown message: ${msgObject.action}`);
+    }
+  }
+
+  static async save(appName?: string) {
+    if (cluster.isPrimary) {
+      throw new Error('AppSharedStorageWorker should be only called from workers');
+    }
+    const obj: StorageMessageFromSubProcess = {
+      id: AppSharedStorageMessageId,
+      pid: process.pid,
+      workerId: cluster.worker?.id || 0,
+      action: 'save',
+      appName: appName || '',
+      key: '',
+    };
+    process.send!(obj);
+  }
+
+  static get(appName: string, key: string, resolve: (value: any) => void, reject: (reason: any) => void) {
+    if (cluster.isPrimary) {
+      throw new Error('AppSharedStorageWorker should be only called from workers');
+    }
+    const uniqueKey = key + ':' + crypto.randomUUID();
+    AppSharedStorageWorker.handleMap[uniqueKey] = { resolve, reject };
+    const obj: StorageMessageFromSubProcess = {
+      id: AppSharedStorageMessageId,
+      pid: process.pid,
+      workerId: cluster.worker?.id || 0,
+      action: 'get',
+      appName,
+      uniqueKey,
+      key,
+    };
+    process.send!(obj);
+  }
+
+  static getWithPrefix(appName: string, prefixKey: string, resolve: (value: any) => void, reject: (reason: any) => void) {
+    if (cluster.isPrimary) {
+      throw new Error('AppSharedStorageWorker should be only called from workers');
+    }
+    const uniqueKey = prefixKey + ':' + crypto.randomUUID();
+    AppSharedStorageWorker.handleMap[uniqueKey] = { resolve, reject };
+    const obj: StorageMessageFromSubProcess = {
+      id: AppSharedStorageMessageId,
+      pid: process.pid,
+      workerId: cluster.worker?.id || 0,
+      action: 'getWithPrefix',
+      appName,
+      uniqueKey,
+      key: prefixKey,
+    };
+    process.send!(obj);
+  }
+
+  static set(appName: string, key: string, value: any) {
+    if (cluster.isPrimary) {
+      throw new Error('AppSharedStorageWorker should be only called from workers');
+    }
+    const obj: StorageMessageFromSubProcess = {
+      id: AppSharedStorageMessageId,
+      pid: process.pid,
+      workerId: cluster.worker?.id || 0,
+      action: 'set',
+      appName,
+      key,
+      value,
+    };
+    process.send!(obj);
+  }
+}
+
+// this can be used in app, but in api, it should use getAppStorage()
+export const appStorage = /* @__PURE__ */ AppSharedStorage.getInstance();

package/src/app/app-start.ts

@@ -0,0 +1,117 @@
+import cluster from 'cluster';
+import { WebProcessor } from './web-processor';
+import { appLoader } from './app-loader';
+import { processMessageFromPrimary, processMessageFromWorker } from './app-message';
+import { WebServer } from './web-server';
+import { processDevRequests } from './process-dev-requests';
+import { appCache } from './app-cache';
+import { AppStartProps, InitStartProps, AppCacheGlobal, AppCacheKeys } from '../models';
+import { appStorage } from './app-shared-storage';
+import { HostToPath } from './host-to-path';
+import { cleanupAndExit } from './cleanup-exit';
+
+// Don't use logger before set process message
+class AppStart {
+  debug: boolean = false;
+  webServer: WebServer | undefined;
+
+  getWorkerId() {
+    return cluster.worker ? cluster.worker.id : 0;
+  }
+
+  async start(props: AppStartProps, webServer?: WebServer) {
+    this.debug = props.debug;
+    this.bindProcess();
+
+    appCache.set(AppCacheGlobal, AppCacheKeys.APP_DEBUG, props.debug);
+    // appCache.set(AppCacheGlobal, AppCacheKeys.APP_ENV_FILE, props.appEnvFile);
+    appCache.set(AppCacheGlobal, AppCacheKeys.START_TIME, new Date());
+    appCache.set(AppCacheGlobal, AppCacheKeys.RENDER_PAGE_FUNCTIONS, props.renderPageFunctions);
+    const appsList = props.apiConfig.webHostMap.map((item) => item.appName);
+    appCache.set(AppCacheGlobal, AppCacheKeys.APP_LIST, appsList);
+
+    this.webServer = webServer || new WebServer();
+
+    // call the Logger after initLog
+    console.log(
+      `${cluster.isPrimary ? 'Primary Process' : 'Worker Process'}, Starting Server - process id ${
+        process.pid
+      }, path: ${process.cwd()}`
+    );
+
+    // when it's cluster.isPrimary or props.debug, initialize the shared storage first
+    if (cluster.isPrimary) {
+      for (let appConfig of props.apiConfig.webHostMap) {
+        await appStorage.load(appConfig.appName, appConfig.dataPath);
+      }
+    }
+
+    if (props.debug || !cluster.isPrimary) {
+      console.log(`Worker id ${this.getWorkerId()}`);
+
+      process.on('message', processMessageFromPrimary);
+
+      HostToPath.setHostToPathList(props.apiConfig.webHostMap);
+      appLoader.loadApi(props.apiConfig);
+      this.initServer(props.serverConfig);
+    } else if (cluster.isPrimary) {
+      const numCPUs = require('os').cpus().length;
+      console.log(`Master Process is trying to fork ${numCPUs} processes`);
+
+      for (let i = 0; i < numCPUs; i++) {
+        let worker = cluster.fork();
+        worker.on('message', processMessageFromWorker);
+      }
+
+      cluster.on('death', (worker: any) => {
+        console.log(`Worker ${worker.pid} died; starting a new one...`);
+        cluster.fork();
+      });
+    }
+  }
+
+  bindProcess() {
+    if (cluster.isPrimary) {
+      // it looks like the child processes are hung up here
+      process.stdin.resume(); // so the program will not close instantly
+    }
+    // Emitted whenever a no-error-handler Promise is rejected
+    process.on('unhandledRejection', (reason: string, promise) => {
+      console.error(`${process.pid} - Process on unhandledRejection, promise: `, promise, ', reason: ', reason);
+    });
+
+    // do something when app is closing
+    process.on('beforeExit', async () => {
+      cleanupAndExit();
+    });
+    process.on('exit', (ret) => {
+      console.log(`${process.pid} - Process on exit, code: ${ret}`);
+    });
+    // catches uncaught exceptions
+    process.on('uncaughtException', (err: Error) => {
+      console.error(`${process.pid} - Process on uncaughtException: `, err);
+      console.error(err.stack);
+    });
+    // catches ctrl+c event and others
+    ['SIGTERM', 'SIGHUP', 'SIGINT', 'SIGINT', 'SIGBREAK'].forEach((evt) => process.on(evt, cleanupAndExit));
+  }
+
+  async initServer(config: InitStartProps) {
+    const bindIp = config.bindIp || '::';
+    const httpPort = config.httpPort;
+    const httpsPort = config.httpsPort;
+    const sslKeyPath = config.sslKeyPath || '';
+    const sslCrtPath = config.sslCrtPath || '';
+
+    console.log(`Starting Web Server, httpPort: ${httpPort}, httpsPort: ${httpsPort}`);
+    // for dev to refresh the FE or stop the server
+    if (this.debug) {
+      WebProcessor.enableDebug('/debug', processDevRequests);
+    }
+
+    httpPort && this.webServer!.startHttp(httpPort, bindIp);
+    httpsPort && this.webServer!.startHttps(httpsPort, bindIp, sslKeyPath, sslCrtPath);
+  }
+}
+
+export const appStart = /* @__PURE__ */ new AppStart();

package/src/app/cleanup-exit.ts

@@ -0,0 +1,12 @@
+import cluster from 'cluster';
+import { appStorage } from './app-shared-storage';
+
+export const cleanupAndExit = async () => {
+  console.log(`${process.pid} - Process on SIGINT, exit.`);
+  // save shared storage first
+  if (cluster.isPrimary) {
+    // save only happens once
+    await appStorage.save();
+  }
+  process.exit(0);
+};

package/src/app/host-to-path.ts

@@ -0,0 +1,38 @@
+// const request = require('request');
+import { Logger } from '../lib';
+import { HostToPathProps } from '../models';
+
+const logger = new Logger('HostToPath');
+export class HostToPath {
+  static props: HostToPathProps[] = [];
+
+  // this should be initialized before any request
+  static setHostToPathList(props: HostToPathProps[]) {
+    this.props = props;
+  }
+
+  static findHostPath(host: string): HostToPathProps | undefined {
+    for (let key in this.props) {
+      if (this.props[key].webPath && this.props[key].hosts.includes(host)) {
+        logger.debug(`Found ${host} in `, this.props[key].hosts);
+        return this.props[key];
+      }
+    }
+    for (let key in this.props) {
+      for (let domain in this.props[key].hosts) {
+        // if host is 'sub3.sub2.sub1.domain', it matches 'sub2.domain'
+        if (this.props[key].webPath && host.endsWith(this.props[key].hosts[domain])) {
+          logger.debug(`Found ${host} in `, this.props[key].hosts);
+          return this.props[key];
+        }
+      }
+    }
+    for (let key in this.props) {
+      if (this.props[key].webPath && this.props[key].hosts.length === 0) {
+        logger.debug(`Not found ${host} from any domains and use default app: ${this.props[key].appName}`);
+        return this.props[key];
+      }
+    }
+    return;
+  }
+}

package/src/app/index.ts

@@ -0,0 +1,11 @@
+export * from './app-cache';
+export * from './app-loader';
+export * from './app-message';
+export * from './app-shared-storage';
+export * from './app-start';
+export * from './cleanup-exit'
+export * from './host-to-path';
+export * from './process-dev-requests';
+export * from './web-listener';
+export * from './web-processor';
+export * from './web-server';