lupine.api 1.1.49 → 1.1.51

package/admin/admin-release.tsx

@@ -218,7 +218,7 @@ export const AdminReleasePage = () => {
     releaseUpdateBtn.disabled = false;
     if (!dataResponse || dataResponse.status !== 'ok') {
       NotificationMessage.sendMessage(
-        dataResponse.message || 'Failed to update release (timeout, possibly backend is runing, please wait!)',
+        dataResponse.message || 'Failed to update release (timeout, possibly backend is runing, please wait and click Check!)',
         NotificationColor.Error
       );
       return;
@@ -259,6 +259,9 @@ export const AdminReleasePage = () => {
 
     domUpdate.value = <ReleaseList result={result} onUpdate={onUpdate} onLogClick={onLogClick} />;
     domLog.value = <pre>{JSON.stringify(result, null, 2)}</pre>;
+    if (result.releaseProgress) {
+      NotificationMessage.sendMessage('Release progress: ' + result.releaseProgress, NotificationColor.Warning);
+    }
   };
 
   const onRefreshCacheLocal = async () => {

package/admin/admin-resources.tsx

@@ -30,11 +30,10 @@ const ResourcesList = (props: {
   const listFolder = (results: any, parentFolder = '') => {
     return results.map((result: any) => (
       <div>
-        <div
-          class={`row-box mt-m` + (typeof result.size === 'undefined' ? ' f-folder' : '')}
-          onClick={() => props.onListFolder(parentFolder + result.name)}
-        >
-          <label class='label mr-m f-name'>{result.name}</label>
+        <div class={`row-box mt-m`}>
+          <label class='label mr-m f-name' onClick={() => props.onListFolder(parentFolder + result.name)}>
+            {result.name}
+          </label>
           <label class='label mr-m f-time'>{result.time}</label>
           <label class='label mr-m f-size'>{typeof result.size !== 'undefined' && formatBytes(result.size)}</label>
           {typeof result.size !== 'undefined' && (
@@ -97,12 +96,10 @@ const ResourcesList = (props: {
     ));
   };
   const css: CssProps = {
-    '.f-folder': {
-      textDecoration: 'underline',
-      cursor: 'pointer',
-    },
     '.f-name': {
+      textDecoration: 'underline',
       width: '200px',
+      cursor: 'pointer',
     },
     '.f-size span, .f-time span': {
       color: 'red',
@@ -154,18 +151,18 @@ export const AdminResourcesPage = () => {
       fDom.click();
     };
     const onUploadLocal = (fPath: string) => {
-      MessageBox.show({
-        title: 'Override remote file',
-        buttonType: MessageBoxButtonProps.YesNo,
-        contentMinWidth: '300px',
-        handleClicked: (index: number, close) => {
-          if (index === 0) {
-            uploadFile(fPath);
-          }
-          close();
-        },
-        children: <div>Do you upload local files that may overwrite remote file [{fPath}]?</div>,
-      });
+      // MessageBox.show({
+      //   title: 'Override remote file',
+      //   buttonType: MessageBoxButtonProps.YesNo,
+      //   contentMinWidth: '300px',
+      //   handleClicked: (index: number, close) => {
+      //     if (index === 0) {
+      uploadFile(fPath);
+      //     }
+      //     close();
+      //   },
+      //   children: <div>Do you upload local files that may overwrite remote file [{fPath}]?</div>,
+      // });
     };
     const onNewFolder = async (fPath: string) => {
       let newName = '';

package/dev/cp-index-html.js

@@ -37,7 +37,7 @@ exports.cpIndexHtml = async (htmlFile, outputFile, appName, isMobile, defaultThe
   // if isMobile=true, then the last number is 1, or if isMobile=false, the last is 2
   const chgTime = Math.trunc(f1.mtime.getTime() / 10) * 10 + (isMobile ? 1 : 2);
 
-  // when it's isMobile, need to update env and configs
+  // when it's isMobile, need to update env and configs => no configs as mobile app fetches it from api
   if (!f2 || f2.mtime.getTime() !== chgTime || isMobile) {
     const inHtml = await fs.readFile(htmlFile, 'utf-8');
     let outStr = inHtml.replace(/{hash}/gi, new Date().getTime().toString(36));
@@ -48,7 +48,7 @@ exports.cpIndexHtml = async (htmlFile, outputFile, appName, isMobile, defaultThe
       const metaIndexStart = inHtml.indexOf(metaTextStart);
       const metaIndexEnd = inHtml.indexOf(metaTextEnd);
 
-      const webConfig = await readWebConfig(outdirData);
+      // const webConfig = await readWebConfig(outdirData);
       const webEnvData = webEnv.getWebEnv(appName);
 
       outStr =
@@ -56,9 +56,9 @@ exports.cpIndexHtml = async (htmlFile, outputFile, appName, isMobile, defaultThe
         '<script id="web-env" type="application/json">' +
         JSON.stringify(webEnvData) +
         '</script>\r\n' +
-        '<script id="web-setting" type="application/json">' +
-        JSON.stringify(webConfig) +
-        '</script>' +
+        // '<script id="web-setting" type="application/json">' +
+        // JSON.stringify(webConfig) +
+        // '</script>' +
         outStr.substring(metaIndexEnd + metaTextEnd.length);
       // outStr = webEnv.replaceWebEnv(inHtml, appName, true);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lupine.api",
-  "version": "1.1.49",
+  "version": "1.1.51",
   "license": "MIT",
   "author": "uuware.com",
   "homepage": "https://github.com/uuware/lupine.js",

package/src/admin-api/admin-api.ts

@@ -11,6 +11,7 @@ import { AdminConfig } from './admin-config';
 import { Logger } from '../lib';
 import { IApiBase, ServerRequest } from '../models';
 import { ApiRouter } from '../api';
+import { readWebConfig } from './web-config-api';
 
 const logger = new Logger('admin-api');
 
@@ -27,6 +28,8 @@ export class AdminApi implements IApiBase {
   }
 
   protected mountDashboard() {
+    this.router.use('/web-config', readWebConfig);
+
     const adminDb = new AdminDb();
     this.router.use('/db', needDevAdminSession, adminDb.getRouter());
 

package/src/admin-api/admin-auth.ts

@@ -10,6 +10,11 @@ export const needDevAdminSession = async (req: ServerRequest, res: ServerRespons
   const devAdminSession = await adminApiHelper.getDevAdminFromCookie(req, res, true);
   if (!devAdminSession) {
     // return true to skip the rest of the middleware
+    const response = {
+      status: 'error',
+      message: langHelper.getLang('shared:permission_denied'),
+    };
+    ApiHelper.sendJson(req, res, response);
     return true;
   }
   return false;

package/src/admin-api/admin-csv.ts

@@ -54,6 +54,8 @@ export const loadCSV = async (db: Db, lines: string[]) => {
   let table = '';
   let insSql = '';
   const result: any = {};
+  let toFields: string[] = [];
+  let toFieldsIndex: number[] = [];
   for (const i in lines) {
     if (!lines[i] || lines[i].startsWith('#')) {
       continue;
@@ -61,14 +63,25 @@ export const loadCSV = async (db: Db, lines: string[]) => {
     if (lines[i].startsWith('@TABLE,')) {
       table = lines[i].substring(7);
       result[table] = { succeeded: 0, failed: 0, errorMessage: [] };
+
+      const toTableInfo = await db.getTableInfo(table);
+      toFields = toTableInfo.map((item: any) => item.name);
     } else if (lines[i].startsWith('@FIELD,')) {
-      const fields = lines[i].substring(7); //.split(',');
-      const values = Array(fields.split(',').length).fill('?').join(',');
-      insSql = `INSERT INTO ${table} (${fields} ) VALUES (${values})`;
+      const fromFields = lines[i]
+        .substring(7)
+        .split(',')
+        .filter((item: string) => toFields.includes(item));
+      toFieldsIndex = toFields.map((item: string) => fromFields.indexOf(item));
+      const values = Array(fromFields.length).fill('?').join(',');
+      insSql = `INSERT INTO ${table} (${fromFields.join(',')} ) VALUES (${values})`;
     } else {
+      if (toFields.length === 0 || !insSql) {
+        throw new Error('Invalid CSV format (no @TABLE or @FIELD)');
+      }
       try {
         const row = JSON.parse(lines[i]);
-        const r = await db.execute(insSql, row);
+        const values = toFieldsIndex.map((index: number) => row[index]);
+        await db.execute(insSql, values);
         result[table].succeeded++;
       } catch (error: any) {
         result[table].failed++;

package/src/admin-api/admin-release.ts

@@ -10,11 +10,13 @@ import {
   FsUtils,
   adminApiHelper,
   processRefreshCache,
+  apiStorage,
 } from 'lupine.api';
 import path from 'path';
 import { needDevAdminSession } from './admin-auth';
 import { adminTokenHelper } from './admin-token-helper';
 
+const releaseProgress = 'admin-release-progress';
 export class AdminRelease implements IApiBase {
   private logger = new Logger('release-api');
   protected router = new ApiRouter();
@@ -30,7 +32,7 @@ export class AdminRelease implements IApiBase {
   protected mountDashboard() {
     // called by FE
     this.router.use('/check', needDevAdminSession, this.check.bind(this));
-    this.router.use('/update', needDevAdminSession, this.update.bind(this));
+    this.router.use('/update', needDevAdminSession, this.callUpdate.bind(this));
     this.router.use('/view-log', needDevAdminSession, this.viewLog.bind(this));
     // called online or by clients
     this.router.use('/refresh-cache', needDevAdminSession, this.refreshCache.bind(this));
@@ -197,6 +199,7 @@ export class AdminRelease implements IApiBase {
     // const webSub = webSubFolders.filter(i => i.isDirectory()).map(i => path.join(i.parentPath.substring(appData.webPath.length + 1), i.name).replace(/\\/g, '/')).sort();
 
     const response = {
+      releaseProgress: await apiStorage.get(releaseProgress),
       status: 'ok',
       message: 'check.',
       appsFrom: apps,
@@ -261,6 +264,24 @@ export class AdminRelease implements IApiBase {
     return true;
   }
 
+  async callUpdate(req: ServerRequest, res: ServerResponse) {
+    // when remote server is slow, then local update call may be timeout.
+    // so we set a flag to prevent multiple update calls
+    apiStorage.set(releaseProgress, 'update started: ' + new Date().toLocaleString());
+    let result = true;
+    try {
+      result = await this.update(req, res);
+    } catch (e: any) {
+      const response = {
+        status: 'error',
+        message: e.message,
+      };
+      ApiHelper.sendJson(req, res, response);
+    }
+    apiStorage.set(releaseProgress, undefined);
+    return result;
+  }
+
   async update(req: ServerRequest, res: ServerResponse) {
     const jsonData = req.locals.json();
     const data = await this.chkData(jsonData, req, res, false);
@@ -391,6 +412,7 @@ export class AdminRelease implements IApiBase {
       this.logger.error(`updateSendFile, not found: ${sendFile}`);
       return { status: 'error', message: 'Client file not found: ' + sendFile };
     }
+    apiStorage.set(releaseProgress, 'updateSendFile: ' + sendFile);
     const fileContent = (await FsUtils.readFile(sendFile))!;
     // const compressedContent = await new Promise<Buffer>((resolve, reject) => {
     //     zlib.gzip(fileContent, (err, buffer) => {
@@ -417,6 +439,12 @@ export class AdminRelease implements IApiBase {
           fileContent.length
         }), f: ${sendFile}`
       );
+      apiStorage.set(
+        releaseProgress,
+        `updateSendFile, index: ${cnt}, sending: ${chunk.length} (${i + chunk.length} / ${
+          fileContent.length
+        }), f: ${sendFile}`
+      );
       i > 0 && (await new Promise((resolve) => setTimeout(resolve, 1000)));
       const remoteData = await fetch(targetUrl + '/api/admin/release/byClientUpdate', postData);
       const resultText = await remoteData.text();

package/src/admin-api/web-config-api.ts

@@ -0,0 +1,19 @@
+import { ServerResponse } from 'http';
+import { Logger } from '../lib';
+import { ServerRequest } from '../models';
+import { langHelper } from '../lang';
+import { ApiHelper, apiStorage } from '../api';
+
+// only used by mobile app. For web, it's injected in the html by server-side render
+const logger = new Logger('web-cfg-api');
+export const readWebConfig = async (req: ServerRequest, res: ServerResponse) => {
+  logger.info('readWebConfig');
+
+  const response = {
+    status: 'ok',
+    result: await apiStorage.getWebAll(),
+    message: langHelper.getLang('shared:operation_success'),
+  };
+  ApiHelper.sendJson(req, res, response);
+  return true;
+};

package/src/api/api-shared-storage.ts

@@ -1,11 +1,11 @@
 /**
- * A persistent storage for the Api
+ * A persistent storage for the Api (stored in primary process)
  */
 import { IAppSharedStorage } from '../models';
 import { SimpleStorageDataProps } from '../models';
 import { apiCache } from './api-cache';
 
-// ApiSharedStorage is used in api module to store variables inside an api scope
+// ApiSharedStorage is used in api module to store variables under appName scope, but is stored in one place - primary process
 export class ApiSharedStorage {
   private storage: IAppSharedStorage | undefined;
 

package/src/api/server-render.ts

@@ -9,7 +9,6 @@ import { IToClientDelivery } from '../models/to-client-delivery-props';
 import { JsonObject } from '../models/json-object';
 import { getTemplateCache } from './api-cache';
 import { apiStorage } from './api-shared-storage';
-import { SimpleStorageDataProps } from '../models';
 import { RuntimeRequire } from '../lib/runtime-require';
 
 const logger = new Logger('StaticServer');
@@ -159,13 +158,13 @@ export const serverSideRenderPage = async (
   const _lupineJs = cachedHtml[nearRoot]._lupineJs;
   const currentCache = cachedHtml[nearRoot] as CachedHtmlProps;
   const webSetting = await apiStorage.getWebAll();
-  const webSettingShortKey: SimpleStorageDataProps = {};
-  for (let item of Object.keys(webSetting)) {
-    const newItem = item.substring(4);
-    webSettingShortKey[newItem] = webSetting[item];
-  }
+  // const webSettingShortKey: SimpleStorageDataProps = {};
+  // for (let item of Object.keys(webSetting)) {
+  //   const newItem = item.substring(4);
+  //   webSettingShortKey[newItem] = webSetting[item];
+  // }
   // const webSetting = AppConfig.get(AppConfig.WEB_SETTINGS_KEY) || {};
-  const clientDelivery = new ToClientDelivery(currentCache.webEnv, webSettingShortKey, req.locals.cookies());
+  const clientDelivery = new ToClientDelivery(currentCache.webEnv, webSetting, req.locals.cookies());
   const page = await _lupineJs.generatePage(props, clientDelivery);
   // console.log(`=========load lupin: `, content);
 
@@ -199,7 +198,7 @@ export const serverSideRenderPage = async (
   res.write(page.metaData);
   res.write(page.globalCss);
   res.write('<script id="web-env" type="application/json">' + JSON.stringify(currentCache.webEnv) + '</script>');
-  res.write('<script id="web-setting" type="application/json">' + JSON.stringify(webSettingShortKey) + '</script>');
+  res.write('<script id="web-setting" type="application/json">' + JSON.stringify(webSetting) + '</script>');
   res.write(
     currentCache.content.substring(
       currentCache.metaIndexEnd + metaTextEnd.length,

package/src/app/app-message.ts

@@ -1,13 +1,33 @@
 import cluster from 'cluster';
 import { Logger, LogWriter, LogWriterMessageId } from '../lib';
 import { processDebugMessage } from './process-dev-requests';
-import { appStorage } from './app-shared-storage';
-import { AppSharedStorageMessageId } from '../models';
 import { cleanupAndExit } from './cleanup-exit';
 
+export type AppMessageProps = {
+  id: string;
+  message: string;
+  appName?: string;
+  [id: string]: any;
+};
+const _savedMessageHandler: {
+  fromPrimary: { [messageId: string]: (msgObject: AppMessageProps) => void };
+  fromWorker: { [messageId: string]: (msgObject: AppMessageProps) => void };
+} = {
+  fromPrimary: {},
+  fromWorker: {},
+};
+
+export const registerMessageHandlerFromPrimary = (messageId: string, handler: (msgObject: AppMessageProps) => void) => {
+  _savedMessageHandler.fromPrimary[messageId] = handler;
+};
+
+export const registerMessageHandlerFromWorker = (messageId: string, handler: (msgObject: AppMessageProps) => void) => {
+  _savedMessageHandler.fromWorker[messageId] = handler;
+};
+
 const logger = new Logger('app-message');
 // send msg to all clients
-const broadcast = (msgObject: any) => {
+const broadcast = (msgObject: AppMessageProps) => {
   for (let i in cluster.workers) {
     if (cluster.workers[i]) cluster.workers[i].send(msgObject);
   }
@@ -15,33 +35,45 @@ const broadcast = (msgObject: any) => {
 
 // this is a worker and msg is from Primary
 // when debug is on, it's in primary, but it shouldn't receive those msgs
-export const processMessageFromPrimary = (msgObject: any) => {
+export const processMessageFromPrimary = (msgObject: AppMessageProps) => {
   if (!msgObject || !msgObject.id) {
     logger.warn(`Unknown message from master in work: ${cluster.worker?.id}`);
     return;
   }
 
+  const primaryFn = _savedMessageHandler.fromPrimary[msgObject.id];
+  if (primaryFn) {
+    primaryFn(msgObject);
+    return;
+  }
+
   if (msgObject.id == 'debug') {
     processDebugMessage(msgObject);
-  } else if (msgObject.id == AppSharedStorageMessageId) {
-    appStorage.messageFromPrimaryProcess(msgObject);
+    // } else if (msgObject.id == AppSharedStorageMessageId) {
+    //   appStorage.messageFromPrimaryProcess(msgObject);
   } else {
     logger.warn(`Unknown message: ${msgObject.id}`);
   }
 };
 
 // this is primary, msg is from a client
-export const processMessageFromWorker = (msgObject: any) => {
+export const processMessageFromWorker = (msgObject: AppMessageProps) => {
   if (!msgObject || !msgObject.id) {
     if (msgObject['watch:require']) return;
     logger.warn(`Unknown message from work: ${cluster.worker?.id}`);
     return;
   }
 
+  const workerFn = _savedMessageHandler.fromWorker[msgObject.id];
+  if (workerFn) {
+    workerFn(msgObject);
+    return;
+  }
+
   if (msgObject.id == LogWriterMessageId) {
-    LogWriter.messageFromSubProcess(msgObject);
-  } else if (msgObject.id == AppSharedStorageMessageId) {
-    appStorage.messageFromSubProcess(msgObject);
+    LogWriter.messageFromSubProcess(msgObject as any);
+    // } else if (msgObject.id == AppSharedStorageMessageId) {
+    //   appStorage.messageFromSubProcess(msgObject);
   } else if (msgObject.id == 'debug') {
     logger.debug(
       `Message from worker ${cluster.worker?.id}, message: ${msgObject.message}, appName: ${msgObject.appName}`

package/src/app/app-shared-storage.ts

@@ -1,5 +1,7 @@
 /**
- * A persistent storage for the Api
+ * A persistent storage to store data in primary process and share to all workers
+ *
+ * You should use apiStorage in api module
  */
 import * as fs from 'fs/promises';
 import * as path from 'path';
@@ -14,6 +16,7 @@ import {
   IAppSharedStorage,
   StorageMessageFromSubProcess,
 } from '../models';
+import { registerMessageHandlerFromPrimary, registerMessageHandlerFromWorker } from './app-message';
 
 // in Api scope, use ApiSharedStorage instead of this
 // storage cross clusters, loaded when start and saved before exist
@@ -27,6 +30,15 @@ export class AppSharedStorage implements IAppSharedStorage {
   public static getInstance(): IAppSharedStorage {
     if (!AppSharedStorage.instance) {
       AppSharedStorage.instance = new AppSharedStorage();
+      // register app message handlers
+      registerMessageHandlerFromPrimary(
+        AppSharedStorageMessageId,
+        AppSharedStorage.instance.messageFromPrimaryProcess.bind(AppSharedStorage.instance)
+      );
+      registerMessageHandlerFromWorker(
+        AppSharedStorageMessageId,
+        AppSharedStorage.instance.messageFromSubProcess.bind(AppSharedStorage.instance)
+      );
     }
     return AppSharedStorage.instance;
   }
@@ -129,9 +141,9 @@ export class AppSharedStorage implements IAppSharedStorage {
   }
 
   // called from primary before exit, or from api to save changes
-  async save(appName?: string) {
+  async save(appName?: string, exit?: boolean) {
     if (!cluster.isPrimary) {
-      AppSharedStorageWorker.save(appName);
+      await AppSharedStorageWorker.save(appName);
       return;
     }
 
@@ -171,8 +183,16 @@ export class AppSharedStorage implements IAppSharedStorage {
   getApi(appName: string, key: string): Promise<string> {
     return this.get(appName, AppSharedStorageApiPrefix + key);
   }
-  getWebAll(appName: string): Promise<SimpleStorageDataProps> {
-    return this.getWithPrefix(appName, AppSharedStorageWebPrefix);
+  async getWebAll(appName: string): Promise<SimpleStorageDataProps> {
+    const webAll = await this.getWithPrefix(appName, AppSharedStorageWebPrefix);
+
+    const webSettingShortKey: SimpleStorageDataProps = {};
+    for (let item of Object.keys(webAll)) {
+      const newItem = item.substring(AppSharedStorageWebPrefix.length);
+      webSettingShortKey[newItem] = webAll[item];
+    }
+
+    return webSettingShortKey;
   }
   getWithPrefix(appName: string, prefixKey: string): Promise<SimpleStorageDataProps> {
     return new Promise((resolve, reject) => {
@@ -255,7 +275,7 @@ class AppSharedStorageWorker {
       workerId: cluster.worker?.id || 0,
       action: 'save',
       appName: appName || '',
-      key: '',
+      key: 'save',
     };
     process.send!(obj);
   }
@@ -278,7 +298,12 @@ class AppSharedStorageWorker {
     process.send!(obj);
   }
 
-  static getWithPrefix(appName: string, prefixKey: string, resolve: (value: any) => void, reject: (reason: any) => void) {
+  static getWithPrefix(
+    appName: string,
+    prefixKey: string,
+    resolve: (value: any) => void,
+    reject: (reason: any) => void
+  ) {
     if (cluster.isPrimary) {
       throw new Error('AppSharedStorageWorker should be only called from workers');
     }

package/src/app/app-start.ts

@@ -46,7 +46,7 @@ class AppStart {
       }
     }
 
-    if (props.debug || !cluster.isPrimary) {
+    if (!cluster.isPrimary) {
       console.log(`Worker id ${this.getWorkerId()}`);
 
       process.on('message', processMessageFromPrimary);
@@ -55,7 +55,7 @@ class AppStart {
       appLoader.loadApi(props.apiConfig);
       this.initServer(props.serverConfig);
     } else if (cluster.isPrimary) {
-      const numCPUs = require('os').cpus().length;
+      const numCPUs = props.debug ? 1 : require('os').cpus().length;
       console.log(`Master Process is trying to fork ${numCPUs} processes`);
 
       for (let i = 0; i < numCPUs; i++) {
@@ -73,7 +73,7 @@ class AppStart {
   bindProcess() {
     if (cluster.isPrimary) {
       // it looks like the child processes are hung up here
-      process.stdin.resume(); // so the program will not close instantly
+      process.stdin.resume(); // so the program will not close instantly, keep isPrimary process running
     }
     // Emitted whenever a no-error-handler Promise is rejected
     process.on('unhandledRejection', (reason: string, promise) => {

package/src/app/cleanup-exit.ts

@@ -6,7 +6,7 @@ export const cleanupAndExit = async () => {
   // save shared storage first
   if (cluster.isPrimary) {
     // save only happens once
-    await appStorage.save();
+    await appStorage.save('', true);
   }
   process.exit(0);
 };

package/src/app/web-listener.ts

@@ -39,7 +39,47 @@ export const setServerName = (serverName: string) => {
   SERVER_NAME = serverName;
 };
 
-export type RawMiddleware = (req: IncomingMessage, res: ServerResponse, next: () => void) => void;
+export type RawMiddleware = (req: IncomingMessage, res: ServerResponse, next: () => void) => Promise<void>;
+
+/*
+IP limit
+
+let IP_LIMIT_ENABLED = false;
+export const setIpLimitEnabled = (enabled: boolean) => {
+  IP_LIMIT_ENABLED = enabled;
+};
+
+let IP_LIMIT_RATE = 60;
+let IP_LIMIT_DURATION = 1000 * 30;
+export const setIpLimitRateAndDuration = (rate: number, durationSeconds: number) => {
+  IP_LIMIT_RATE = rate;
+  IP_LIMIT_DURATION = durationSeconds * 1000;
+};
+
+
+class TTLMap<K, V> extends Map<K, V> {
+  set(key: K, value: V): this;
+  set(key: K, value: V, ttl: number): this;
+  set(key: K, value: V, ttl?: number): this {
+    const isNew = !this.has(key);
+    super.set(key, value);
+    if (isNew && ttl) {
+      setTimeout(() => this.delete(key), ttl).unref?.();
+    }
+    return this;
+  }
+}
+const counts = new TTLMap<string, number>();
+
+function checkFixedWindow(ip: string, limit = IP_LIMIT_RATE) {
+  const now = Date.now();
+  const key = `${ip}:${Math.floor(now / IP_LIMIT_DURATION)}`;
+  const count = (counts.get(key) || 0) + 1;
+  counts.set(key, count, IP_LIMIT_DURATION * 2); // key自动过期
+  return count > limit;
+}
+
+*/
 
 let lastRequestTime = new Date().getTime();
 
@@ -57,16 +97,14 @@ export class WebListener {
   addRawMiddlewareChain(middleware: RawMiddleware) {
     this.rawMiddlewares.push(middleware);
   }
-  async handleRawMiddlewares(req: IncomingMessage, res: ServerResponse) {
-    const runChain = (list: RawMiddleware[], context: { req: IncomingMessage; res: ServerResponse }) => {
-      const dispatch = async (i: number) => {
-        const fn = list[i];
-        if (!fn) return;
-        await fn(context.req, context.res, () => dispatch(i + 1));
-      };
-      return dispatch(0);
+
+  runMiddlewareChain(list: RawMiddleware[], context: { req: IncomingMessage; res: ServerResponse }) {
+    const dispatch = async (i: number) => {
+      const fn = list[i];
+      if (!fn) return;
+      await fn(context.req, context.res, () => dispatch(i + 1));
     };
-    await runChain(this.rawMiddlewares, { req, res });
+    return dispatch(0);
   }
 
   async listener(reqOrigin: IncomingMessage, res: ServerResponse) {
@@ -87,7 +125,7 @@ export class WebListener {
       return;
     }
 
-    await this.handleRawMiddlewares(reqOrigin, res);
+    await this.runMiddlewareChain(this.rawMiddlewares, { req: reqOrigin, res });
     if (res.writableEnded || res.headersSent) {
       return;
     }

package/src/lib/db/db-sqlite.ts

@@ -95,7 +95,7 @@ export class DbSqlite extends Db {
 
   public async getTableInfo(table: string): Promise<any> {
     const query = `PRAGMA table_info(${table});`;
-    const result = await this.execute(query);
+    const result = await this.select(query);
     return result;
   }
 }

package/src/lib/db/db.ts

@@ -2,7 +2,12 @@ import { Logger } from '../logger';
 import { DbConfig } from '../../models/db-config';
 
 // Instead, Boolean values are stored as integers 0 (false) and 1 (true).
+export type DbFieldExprssionProps = { exprssion: string; params?: (string | number)[] };
 export type DbFieldValue = { [key: string]: string | number };
+export type DbFieldExpression = { [key: string]: string | number | DbFieldExprssionProps };
+const isDbFieldExprssion = (value: any): value is DbFieldExprssionProps => {
+  return value && typeof value === 'object' && 'exprssion' in value;
+};
 
 const logger = new Logger('db');
 export class Db {
@@ -252,12 +257,28 @@ export class Db {
     return await this.execute(sql, params);
   }
 
-  public async updateObject(table: string, updateFieldValues: DbFieldValue, whereFieldValues: DbFieldValue) {
+  public async updateObject(table: string, updateFieldValues: DbFieldExpression, whereFieldValues: DbFieldValue) {
     table = this.replacePrefix(table);
     const fields = Object.keys(updateFieldValues);
-    let sql = 'UPDATE ' + table + ' SET ' + fields.map((item) => `${item}=?`).join(',');
-    const params = Object.values(updateFieldValues);
+    const setClauseParts: string[] = [];
+    const params: (string | number)[] = [];
+    // let sql = 'UPDATE ' + table + ' SET ' + fields.map((item) => `${item}=?`).join(',');
+    // const params = Object.values(updateFieldValues);
+    for (const field of fields) {
+      const value = updateFieldValues[field];
+
+      // expression
+      if (isDbFieldExprssion(value)) {
+        setClauseParts.push(`${field} = ${value.exprssion}`);
+        if (value.params) params.push(...value.params);
+      } else {
+        // static value
+        setClauseParts.push(`${field} = ?`);
+        params.push(value);
+      }
+    }
 
+    let sql = `UPDATE ${table} SET ${setClauseParts.join(', ')}`;
     if (whereFieldValues && Object.keys(whereFieldValues).length > 0) {
       sql +=
         ' WHERE ' +

package/src/lib/logger.ts

@@ -25,10 +25,17 @@ export const LogWriterMessageId = 'LogWriter';
 logger.info('log test. numer: %d, string: %s, json: %j', 11, 'p2', {key: 'value', n: 22});
 */
 export class LogWriter {
-  static instance = new LogWriter();
+  static instance?: LogWriter;
 
   private constructor() {}
 
+  public static getInstance(): LogWriter {
+    if (!LogWriter.instance) {
+      LogWriter.instance = new LogWriter();
+    }
+    return LogWriter.instance;
+  }
+
   private _config: LogConfig | undefined;
   getConfig = (): LogConfig => {
     return this._config || getDefaultLogConfig();
@@ -138,7 +145,7 @@ export class LogWriter {
       console.error('Logger got wrong message: ', msgObject);
       return;
     }
-    LogWriter.instance._log(
+    LogWriter.getInstance()._log(
       msgObject.level,
       msgObject.namespace,
       msgObject.color,
@@ -234,41 +241,41 @@ export class Logger {
   }
 
   isDebug(): boolean {
-    return LogWriter.instance.level <= 4;
+    return LogWriter.getInstance().level <= 4;
   }
 
   debug(...message: (string | object | number)[]) {
-    if (LogWriter.instance.level < 4) {
+    if (LogWriter.getInstance().level < 4) {
       return;
     }
-    LogWriter.instance._log('DEBUG', this.namespace, this.color, message);
+    LogWriter.getInstance()._log('DEBUG', this.namespace, this.color, message);
   }
 
   info(...message: (string | object | number)[]) {
-    if (LogWriter.instance.level < 3) {
+    if (LogWriter.getInstance().level < 3) {
       return;
     }
-    LogWriter.instance._log('INFO', this.namespace, this.color, message);
+    LogWriter.getInstance()._log('INFO', this.namespace, this.color, message);
   }
 
   warn(...message: (string | object | number)[]) {
-    if (LogWriter.instance.level < 2) {
+    if (LogWriter.getInstance().level < 2) {
       return;
     }
-    LogWriter.instance._log('WARN', this.namespace, this.color, message);
+    LogWriter.getInstance()._log('WARN', this.namespace, this.color, message);
   }
 
   error(...message: (string | object | number)[]) {
-    if (LogWriter.instance.level < 1) {
+    if (LogWriter.getInstance().level < 1) {
       return;
     }
-    LogWriter.instance._log('ERROR', this.namespace, this.color, message);
+    LogWriter.getInstance()._log('ERROR', this.namespace, this.color, message);
   }
 
   fatal(...message: (string | object | number)[]) {
-    if (LogWriter.instance.level < 0) {
+    if (LogWriter.getInstance().level < 0) {
       return;
     }
-    LogWriter.instance._log('FATAL', this.namespace, this.color, message);
+    LogWriter.getInstance()._log('FATAL', this.namespace, this.color, message);
   }
 }

package/src/models/app-shared-storage-props.ts

@@ -25,7 +25,7 @@ export interface IAppSharedStorage {
   // this is primary, msg is from a client
   messageFromSubProcess(msgObject: any): void;
   load(appName: string, rootPath: string): Promise<void>;
-  save(appName?: string): Promise<void>;
+  save(appName?: string, exit?: boolean): Promise<void>;
   get(appName: string, key: string): Promise<string>;
   getWeb(appName: string, key: string): Promise<string>;
   getApi(appName: string, key: string): Promise<string>;

package/src/app/web-listener copy.ts

@@ -1,274 +0,0 @@
-import { IncomingMessage, ServerResponse } from 'http';
-import { Logger } from '../lib/logger';
-import crypto from 'crypto';
-import { parseCookies } from '../lib/utils/cookie-util';
-import { WebProcessor } from './web-processor';
-import { handler403, handler404, handler500, handler503, SimpleStorage } from '../api';
-import { JsonObject, AsyncStorageProps, ServerRequest, SetCookieProps } from '../models';
-import { HostToPath } from './host-to-path';
-import { serializeCookie } from '../lib/utils/cookie-util';
-const logger = new Logger('listener');
-
-let MAX_REQUEST_SIZE = 1024 * 1024 * 5;
-export const setMaxRequestSize = (size: number) => {
-  MAX_REQUEST_SIZE = size;
-};
-
-// The maximum number of requests being processed. If there are no requests for 10 minutes, this number will be reset to 0.
-let MAX_REQUEST_COUNT = 100;
-let REQUEST_COUNT = 0;
-export const setMaxRequestCount = (count: number) => {
-  MAX_REQUEST_COUNT = count;
-};
-export const getRequestCount = () => {
-  return REQUEST_COUNT;
-};
-
-let REQUEST_TIMEOUT = 1000 * 30;
-export const setRequestTimeout = (timeout: number) => {
-  REQUEST_TIMEOUT = timeout;
-};
-
-let accessControlAllowHosts: string[] = [];
-export const setAccessControlAllowHost = (allowHosts: string[]) => {
-  accessControlAllowHosts = allowHosts;
-};
-
-let SERVER_NAME: string = 'nginx/1.19.2';
-export const setServerName = (serverName: string) => {
-  SERVER_NAME = serverName;
-};
-
-let IP_LIMIT_RATE_PER_SECOND = 3;
-export const setIpLimitRatePerSecond = (rate: number) => {
-  IP_LIMIT_RATE_PER_SECOND = rate;
-};
-let IP_LIMIT_RATE_PER_MINUTE = 60;
-export const setIpLimitRatePerMinute = (rate: number) => {
-  IP_LIMIT_RATE_PER_MINUTE = rate;
-};
-
-let IP_BLOCK_TIME = 60_000;
-export const setIpBlockTime = (time: number) => {
-  IP_BLOCK_TIME = time;
-};
-
-export const rawMiddlewareIpLimit = (req: IncomingMessage, res: ServerResponse, next: () => void) => {
-  const clientIp = req.socket.remoteAddress || 'unknown';
-};
-
-export type RawMiddleware = (req: IncomingMessage, res: ServerResponse, next: () => void) => void;
-
-let lastRequestTime = new Date().getTime();
-
-// type ProcessRequest = (req: ServerRequest, res: ServerResponse) => void;
-export class WebListener {
-  // process requests before business logic, for example IP filter, rate limit, etc.
-  rawMiddlewares: RawMiddleware[];
-  processor: WebProcessor;
-
-  constructor(processRequest: WebProcessor) {
-    this.rawMiddlewares = [];
-    this.processor = processRequest;
-  }
-
-  addRawMiddlewareChain(middleware: RawMiddleware) {
-    this.rawMiddlewares.push(middleware);
-  }
-  async handleRawMiddlewares(req: IncomingMessage, res: ServerResponse) {
-    const runChain = (list: RawMiddleware[], context: { req: IncomingMessage; res: ServerResponse }) => {
-      const dispatch = async (i: number) => {
-        const fn = list[i];
-        if (!fn) return;
-        await fn(context.req, context.res, () => dispatch(i + 1));
-      };
-      return dispatch(0);
-    };
-    await runChain(this.rawMiddlewares, { req, res });
-  }
-
-  async listener(reqOrigin: IncomingMessage, res: ServerResponse) {
-    // If there is no request in the last 10 minutes, reset the request count.
-    if (new Date().getTime() - lastRequestTime > 1000 * 60 * 10) {
-      if (REQUEST_COUNT != 0) {
-        // in case any errors skipped (--REQUEST_COUNT)
-        logger.warn(`!!!!!!!!!! ========== REQUEST_COUNT is not counted properly: ${REQUEST_COUNT}`);
-      }
-      REQUEST_COUNT = 0;
-      lastRequestTime = new Date().getTime();
-    }
-
-    // back-pressure
-    if (REQUEST_COUNT > MAX_REQUEST_COUNT) {
-      logger.warn(`Too many requests, count: ${REQUEST_COUNT} > ${MAX_REQUEST_COUNT}`);
-      handler503(res, 'Server is busy, please retry later.');
-      return;
-    }
-
-    await this.handleRawMiddlewares(reqOrigin, res);
-    if (res.writableEnded || res.headersSent) {
-      return;
-    }
-
-    // const requestStart = process.hrtime.bigint();
-    const uuid = crypto.randomUUID();
-    const url = reqOrigin.url || '';
-    const requestInfo = `uuid: ${uuid}, Access url: ${url}`;
-    const req = reqOrigin as ServerRequest;
-
-    const host = (req.headers.host || '').split(':')[0]; // req.headers.host contains port
-    const hostPath = HostToPath.findHostPath(host);
-    if (!hostPath || !hostPath.webPath || !hostPath.appName) {
-      const msg = `Web root is not defined properly for host: ${host}.`;
-      logger.error(msg);
-      handler404(res, msg);
-      return;
-    }
-
-    REQUEST_COUNT++;
-    logger.debug(
-      `Request started. Count: ${REQUEST_COUNT}, Log uuid: ${uuid}, access: ${
-        req.headers.host
-      }, url: ${url}, time: ${new Date().toISOString()}, from: ${req.socket.remoteAddress}`
-    );
-
-    const urlSplit = url.split('?');
-    req.setTimeout(REQUEST_TIMEOUT);
-    req.on('timeout', () => {
-      REQUEST_COUNT--;
-      logger.warn('timeout');
-      req.destroy(new Error('timeout handling'));
-    });
-
-    const jsonFn = (): JsonObject | undefined => {
-      if (!req.locals._json && req.locals.body) {
-        const sBody = req.locals.body.toString();
-        if (!sBody) {
-          req.locals._json = undefined;
-        } else {
-          try {
-            req.locals._json = JSON.parse(sBody);
-          } catch (err: any) {
-            logger.warn(`JSON.parse error: ${err.message}`);
-          }
-        }
-      }
-      return req.locals._json;
-    };
-    const cookiesFn = (): SimpleStorage => {
-      if (!req.locals._cookies) {
-        req.locals._cookies = new SimpleStorage(req.headers ? parseCookies(req.headers.cookie) : {});
-      }
-      return req.locals._cookies;
-    };
-    const setCookieFn = (name: string, value: string, options: SetCookieProps): void => {
-      const cookies: string[] = [];
-      const cookiesOld = res.getHeader('Set-Cookie');
-      if (cookiesOld) {
-        if (!Array.isArray(cookiesOld)) {
-          cookies.push(cookiesOld as any);
-        } else {
-          cookies.push(...cookiesOld);
-        }
-      }
-
-      const cookiePair = serializeCookie(name, value, options);
-      cookies.push(cookiePair);
-      res.setHeader('Set-Cookie', cookies);
-
-      const localCookies = req.locals.cookies();
-      localCookies.set(name, value);
-    };
-
-    req.locals = {
-      uuid,
-      host,
-      url,
-      hostPath,
-      // urlSections: urlSplit[0].split('/').filter((i) => !!i),
-      query: new URLSearchParams(urlSplit[1] || ''),
-      urlWithoutQuery: urlSplit[0],
-      urlParameters: new SimpleStorage({}),
-      body: undefined,
-      json: jsonFn,
-      cookies: cookiesFn,
-      setCookie: setCookieFn,
-      clearCookie: (name: string) => {
-        res.setHeader('Set-Cookie', `${name}=; max-age=0`);
-      },
-    };
-
-    let bigRequest = false;
-    let totalLength = 0;
-    const bodyData: any[] = [];
-    req.on('error', (err: any) => {
-      REQUEST_COUNT--;
-      logger.error(`${requestInfo}, count: ${REQUEST_COUNT}, Request Error: `, err);
-      handler500(res, `listener error: ${err && err.message}`);
-    });
-
-    req.on('data', (chunk: any) => {
-      totalLength += chunk.length;
-      logger.debug(`${requestInfo}, Request data length: ${chunk.length}, total: ${totalLength}`);
-      // Limit Request Size
-      if (!bigRequest && totalLength < MAX_REQUEST_SIZE) {
-        bodyData.push(chunk);
-      } else {
-        if (!bigRequest) {
-          bigRequest = true;
-          logger.warn(`Warn, request data is too big: ${totalLength} > ${MAX_REQUEST_SIZE}`);
-        }
-        req.socket.destroy();
-      }
-    });
-
-    req.on('end', async () => {
-      try {
-        if (bigRequest) {
-          logger.warn(`Request data is too big to process, url: ${req.locals.url}`);
-          handler403(res, `Request data is too big to process, url: ${req.locals.url}`);
-          return;
-        }
-
-        const body = Buffer.concat(bodyData);
-        const contentType = req.headers['content-type'];
-        logger.debug(`url: ${url}, Request body length: ${body.length}, contentType: ${contentType}`);
-        req.locals.body = body;
-
-        res.setHeader('Server', SERVER_NAME);
-        if (accessControlAllowHosts.includes(host)) {
-          const allowOrigin = req.headers.origin && req.headers.origin !== 'null' ? req.headers.origin : '*';
-          res.setHeader('Access-Control-Allow-Origin', allowOrigin);
-          res.setHeader('Access-Control-Allow-Credentials', 'true');
-        }
-
-        const store: AsyncStorageProps = {
-          uuid: uuid,
-          hostPath: hostPath,
-          appName: hostPath.appName,
-          locals: req.locals,
-          lang: req.locals.cookies().get('lang', 'en') || 'en',
-        };
-        await this.processor.processRequest(store, req, res);
-      } finally {
-        REQUEST_COUNT--;
-      }
-      // await new Promise(resolve => setTimeout(resolve, 3000));
-
-      // asyncLocalStorage.run(store, async () => {
-      //   try {
-      //     await onEnd();
-      //   } catch (error: any) {
-      //     logger.error(`url: ${url}, Request end error: `, error.message);
-      //   }
-
-      //   lastRequestTime = new Date().getTime();
-      //   const requestEnd = process.hrtime.bigint();
-      //   REQUEST_COUNT--;
-      //   logger.debug(
-      //     `Request finished. Count: ${REQUEST_COUNT}, url: ${url}, time: ${new Date().toISOString()}, duration: ${Number(requestEnd - requestStart) / 1000000} ms`
-      //   );
-      // });
-    });
-  }
-}