miqro 6.1.2 → 6.1.4

package/src/services/utils/cluster-cache.ts

@@ -0,0 +1,230 @@
+import { Logger } from "@miqro/core";
+import { CacheInterface } from "../../types.js";
+
+const ClusterCacheType = "$$$$$$$$$$$ClusterCacheType$$$$$$$$$$$";
+
+interface ClusterCacheMessage {
+  type: typeof ClusterCacheType;
+  target: string;
+  action: "set" | "unset" | "set_add" | "set_delete" | "array_push";
+  fromPID: number;
+  key: string;
+  value?: string;
+}
+
+export class ClusterCache implements CacheInterface {
+  private localCache = new Map<string, unknown>();
+  //private logger: Logger;
+  listener: (data: any) => Promise<void>;
+
+  constructor(public name: string, public logger?: Logger) {
+    this.listener = async (data) => {
+      try {
+        const msg = (data as ClusterCacheMessage);
+
+        //console.dir(msg);
+        if (
+          msg &&
+          msg.key &&
+          msg.action &&
+          msg.type === ClusterCacheType &&
+          msg.fromPID !== process.pid,
+          (msg.action === "set_add" || msg.action === "set" || msg.action === "unset" || msg.action === "set_delete" || msg.action === "array_push") &&
+          msg.target === this.name) {
+          this.logger?.debug("remote cluster cache message from [%s] [%s] [%s] [%s]", msg.fromPID, msg.target, msg.action, msg.key);
+          switch (msg.action) {
+            case "unset":
+              this.localCache.delete(msg.key);
+              break;
+            case "set":
+              this.localCache.set(msg.key, msg.value);
+              break;
+            case "set_add": {
+              //this.localCache.set(msg.key, msg.value);
+              const list = this.localCache.has(msg.key) ? this.localCache.get(msg.key) : new Set<string>();
+              if (!(list instanceof Set)) {
+                throw new Error("cannot apply push on non array");
+              }
+              if (list.has(msg.value)) {
+                list.add(msg.value);
+              }
+              this.localCache.set(msg.key, list);
+              break;
+            }
+            case "set_delete": {
+              //this.localCache.set(msg.key, msg.value);
+              const list = this.localCache.has(msg.key) ? this.localCache.get(msg.key) : new Set<string>();
+              if (!(list instanceof Set)) {
+                throw new Error("cannot apply push on non array");
+              }
+              if (list.has(msg.value)) {
+                list.delete(msg.value);
+              }
+              this.localCache.set(msg.key, list);
+              break;
+            }
+            case "array_push": {
+              //this.localCache.set(msg.key, msg.value);
+              const list = this.localCache.has(msg.key) ? this.localCache.get(msg.key) : [];
+              if (!(list instanceof Array)) {
+                throw new Error("cannot apply push on non array");
+              }
+              list.push(msg.value);
+              this.localCache.set(msg.key, list);
+              break;
+            }
+          }
+        }
+      } catch (e) {
+        this.logger?.error(e);
+      }
+    };
+    this.connect();
+  }
+
+  connect() {
+    if (process.send) {
+      process.removeListener("message", this.listener);
+      process.on("message", this.listener);
+    }
+  }
+
+  disconnect() {
+    process.removeListener("message", this.listener);
+  }
+
+  get<T = any>(key: string): T | undefined {
+    this.logger?.trace("get(%s)", key);
+    return this.localCache.get(key) as T;
+  }
+  set(key: string, value: unknown): void {
+    this.localCache.set(key, value);
+    this.logger?.trace("set(%s, ...)", key);
+    if (process.send) {
+      setTimeout(() => {
+        process.send({
+          type: ClusterCacheType,
+          action: "set",
+          target: this.name,
+          fromPID: process.pid,
+          key,
+          value
+        } as ClusterCacheMessage);
+      }, 10);
+    }
+  }
+  unset(key: string): void {
+    this.logger?.trace("unset(%s)", key);
+    this.localCache.delete(key);
+    if (process.send) {
+      setTimeout(() => {
+        process.send({
+          type: ClusterCacheType,
+          target: this.name,
+          action: "unset",
+          fromPID: process.pid,
+          key
+        } as ClusterCacheMessage);
+      }, 10);
+    }
+  }
+  has(key: string): boolean {
+    this.logger?.trace("has(%s)", key);
+    return this.localCache.has(key);
+  }
+
+  set_add(key: string, value: unknown): void {
+    this.logger?.trace("push(%s)", key);
+    const list = this.localCache.has(key) ? this.localCache.get(key) : new Set<string>();
+    if (!(list instanceof Set)) {
+      throw new Error("cannot apply on non Set");
+    }
+    if (list.has(value)) {
+      list.add(value);
+    }
+    this.localCache.set(key, list);
+
+    if (process.send) {
+      setTimeout(() => {
+        process.send({
+          type: ClusterCacheType,
+          target: this.name,
+          action: "set_add",
+          fromPID: process.pid,
+          key,
+          value
+        } as ClusterCacheMessage);
+      }, 10);
+    }
+  }
+  set_delete(key: string, value: unknown): void {
+    this.logger?.trace("delete(%s)", key);
+    const list = this.localCache.has(key) ? this.localCache.get(key) : new Set<string>();
+    if (!(list instanceof Set)) {
+      throw new Error("cannot apply on non Set");
+    }
+    if (list.has(value)) {
+      list.delete(value);
+    }
+    this.localCache.set(key, list);
+    if (process.send) {
+      setTimeout(() => {
+        process.send({
+          type: ClusterCacheType,
+          target: this.name,
+          action: "set_delete",
+          fromPID: process.pid,
+          key,
+          value
+        } as ClusterCacheMessage);
+      }, 10);
+    }
+  }
+  set_has(key: string, value: unknown): boolean {
+    this.logger?.trace("set_has(%s)", key);
+    const list = this.localCache.has(key) ? this.localCache.get(key) : new Set<string>();
+    if (!(list instanceof Set)) {
+      throw new Error("cannot apply on non Set");
+    }
+    const ret = list.has(value);
+    this.localCache.set(key, list);
+    return ret;
+  }
+  set_clear(key: string): void {
+    this.logger?.trace("set_clear(%s)", key);
+    const list = this.localCache.has(key) ? this.localCache.get(key) : new Set<string>();
+    if (!(list instanceof Set)) {
+      throw new Error("cannot apply on non Set");
+    }
+    list.clear();
+    this.localCache.set(key, list);
+  }
+  array_push(key: string, value: unknown): void {
+    this.logger?.trace("array_push(%s)", key);
+    const list = this.localCache.has(key) ? this.localCache.get(key) : [];
+    if (!(list instanceof Array)) {
+      throw new Error("cannot apply on non Array");
+    }
+    list.push(value);
+    this.localCache.set(key, list);
+    if (process.send) {
+      setTimeout(() => {
+        process.send({
+          type: ClusterCacheType,
+          target: this.name,
+          action: "array_push",
+          fromPID: process.pid,
+          key,
+          value
+        } as ClusterCacheMessage);
+      }, 10);
+    }
+  }
+  array_clear(key: string): void {
+    this.logger?.trace("array_clear(%s)", key);
+    if (this.localCache.has(key) && !(this.localCache.get(key) instanceof Array)) {
+      throw new Error("cannot apply on non Array");
+    }
+    this.localCache.set(key, []);
+  }
+}

package/src/services/utils/cluster-ws.ts

@@ -0,0 +1,197 @@
+import { Logger, WebSocketServer, WebSocketServerOptions } from "@miqro/core";
+import { Serializable } from "node:child_process";
+
+const ClusterWebSocketServer2MessageType = "$$$$ClusterWebSocketServer2Message$$$$$";
+
+export interface ClusterWebSocketServer2Message {
+  type: typeof ClusterWebSocketServer2MessageType;
+  target: string;
+  action: "connection" | "disconnection" | "sendMessage" | "error" | "sync";
+  clientUUID?: string;
+  errorMessage?: string;
+  fromUUID?: string;
+  fromPID: string,
+  payload?: Serializable;
+}
+
+export class ClusterWebSocketServer2 extends WebSocketServer {
+  public remoteClients: Set<string> = new Set();
+  listener: (data: any) => Promise<void>;
+  public constructor(protected name: string, options: WebSocketServerOptions) {
+    super({
+      ...options,
+      validate: (req) => {
+        if (
+          this.options.maxConnections !== undefined &&
+          this.clients.size + this.remoteClients.size >= this.options.maxConnections
+        ) {
+          return false;
+        } else {
+          return options.validate ? options.validate(req) : true;
+        }
+      },
+      onError: (req, error) => {
+        if (process.send) {
+          process.send({
+            type: ClusterWebSocketServer2MessageType,
+            action: "error",
+            target: this.name,
+            clientUUID: req.uuid,
+            fromPID: String(process.pid),
+            errorMessage: error.message
+          } as ClusterWebSocketServer2Message);
+        }
+        if (options.onError) {
+          options.onError(req, error);
+        }
+      },
+      onConnection: (req) => {
+        if (process.send) {
+          process.send({
+            type: ClusterWebSocketServer2MessageType,
+            action: "connection",
+            target: this.name,
+            fromPID: String(process.pid),
+            clientUUID: req.uuid
+          } as ClusterWebSocketServer2Message);
+        }
+        if (options.onConnection) {
+          options.onConnection(req);
+        }
+      },
+      onDisconnect: (req) => {
+        if (process.send) {
+          process.send({
+            type: ClusterWebSocketServer2MessageType,
+            action: "disconnection",
+            target: this.name,
+            fromPID: String(process.pid),
+            clientUUID: req.uuid
+          } as ClusterWebSocketServer2Message);
+        }
+        if (options.onDisconnect) {
+          options.onDisconnect(req);
+        }
+      }
+    });
+    this.listener = async (data) => {
+      try {
+        const msg = (data as ClusterWebSocketServer2Message);
+        if (
+          msg &&
+          msg.type === ClusterWebSocketServer2MessageType &&
+          msg.target === this.name &&
+          msg.action) {
+          // receive message from cluster workers
+          //console.dir(data);
+          //this.logger?.debug("remote web socket server message from [%s] [%s] [%s]", msg.fromPID, msg.target, msg.action);
+          switch (msg.action) {
+            case "sync":
+              if (process.send) {
+                for (const clientUUID of this.clients.keys()) {
+                  process.send({
+                    type: ClusterWebSocketServer2MessageType,
+                    action: "connection",
+                    target: this.name,
+                    fromPID: String(process.pid),
+                    clientUUID
+                  } as ClusterWebSocketServer2Message);
+                }
+              }
+              break;
+            case "connection":
+              if (!msg.clientUUID) {
+                throw new Error(`action [${msg.action}] without clientUUID`);
+              }
+              if (!this.remoteClients.has(msg.clientUUID)) {
+                this.remoteClients.add(msg.clientUUID);
+              }
+              break;
+            case "disconnection":
+              if (!msg.clientUUID) {
+                throw new Error(`action [${msg.action}] without clientUUID`);
+              }
+              if (this.remoteClients.has(msg.clientUUID)) {
+                this.remoteClients.delete(msg.clientUUID);
+              }
+              break;
+            case "sendMessage": {
+              const payload = String(msg.payload);
+              if (!msg.clientUUID) {
+                // broadcast to local clients
+                await super.broadcast(payload, msg.fromUUID);
+              } else if (this.isConnected(msg.clientUUID)) {
+                // write if local client
+                await super.writeTo(msg.clientUUID, payload);
+              }
+              break;
+            }
+            default:
+              throw new Error(`action [${msg.action}] not supported`);
+          }
+        }
+      } catch (e) {
+        console.error(e);
+      }
+    };
+
+    this.connect();
+  }
+
+  public connect() {
+    if (process.send) {
+      process.removeListener("message", this.listener);
+      process.on("message", this.listener);
+      process.send({
+        type: ClusterWebSocketServer2MessageType,
+        action: "sync",
+        fromPID: String(process.pid),
+        target: this.name
+      } as ClusterWebSocketServer2Message);
+    }
+  }
+
+  public dispose() {
+    process.removeListener("message", this.listener);
+  }
+
+  public async broadcast(payload: string, fromUUID?: string): Promise<void> {
+    if (process.send) {
+      process.send({
+        type: ClusterWebSocketServer2MessageType,
+        action: "sendMessage",
+        payload,
+        target: this.name,
+        fromPID: String(process.pid),
+        fromUUID
+      } as ClusterWebSocketServer2Message);
+    }
+    return super.broadcast(payload, fromUUID);
+  }
+
+  public async writeTo(clientUUID: string, payload: string): Promise<void> {
+    if (this.remoteClients.has(clientUUID)) {
+      // write to remote client via IPC
+      if (process.send) {
+        process.send({
+          type: ClusterWebSocketServer2MessageType,
+          action: "sendMessage",
+          clientUUID,
+          target: this.name,
+          fromPID: String(process.pid),
+          payload
+        } as ClusterWebSocketServer2Message);
+        return;
+      }
+    }
+    // due to the implementation of isUUIDValid depends on this.remoteClients to be sync there can be two clients registered with the same UUID in the cluster
+    if (this.clients.has(clientUUID)) {
+      // write to local client
+      return super.writeTo(clientUUID, payload);
+    }
+  }
+
+  public isUUIDValid(uuid: string) {
+    return !this.clients.has(uuid) && !this.remoteClients.has(uuid);
+  }
+}

package/src/services/utils/db-manager.ts

@@ -0,0 +1,92 @@
+import cluster from "node:cluster";
+import { Database } from "@miqro/query";
+import { DBConfig } from "../../types.js";
+import { LogProvider } from "./log.js";
+import { Logger } from "@miqro/core";
+
+export class DBManager {
+  private map = new Map<string, Database>();
+  public options: {
+    loggerProvider?: LogProvider;
+    logger?: Logger;
+  };
+
+  constructor(options?: {
+    loggerProvider?: LogProvider;
+    logger?: Logger;
+  }) {
+    this.options = options ? options : {};
+  }
+
+  async setupDB(config: DBConfig) {
+    if (this.getDB(config.name)) {
+      throw new Error("cannot override db");
+    }
+    if (!config.disabled) {
+      const DB_IDENTIFIER = cluster.isPrimary ? "DATABASE" : process.env["CLUSTER_NODE_NUMBER"] ? `WORKER_${process.env["CLUSTER_NODE_NUMBER"]}_DATABASE` : "WORKER_DATABASE";
+      this.options?.logger?.debug("setting up db connection [%s]", config.name);
+      this.options?.logger?.trace("creating db connection [%s]", config.name);
+      const db = new Database({
+        dialect: config.dialect ? config.dialect as any : "node:sqlite",
+        storage: config.storage ? config.storage : "./db.sqlite3",
+        connectionString: config.url,
+        logger: this.options?.loggerProvider?.getLogger(`${DB_IDENTIFIER}_${config.name}`)
+      });
+
+      this.options?.logger?.trace("connecting db connection [%s]", config.name);
+      await db.connect();
+      this.options?.logger?.trace("db connection [%s] connected", config.name);
+
+      /*if (config.name && config.name !== name) {
+        this.setDB(config.name, db);
+      }*/
+
+      this.setDB(config.name, db);
+
+      return db;
+    }
+  }
+
+  async connectAll() {
+    this.options?.logger?.debug("connection all db connections");
+    const tR = [];
+    for (const name of this.map.keys()) {
+      const db = this.map.get(name);
+      if (db && db.status === "disconnected") {
+        this.options?.logger?.debug("connecting db connection [%s]", name);
+        tR.push(db.connect());
+      }
+    }
+    await Promise.all(tR);
+  }
+
+  async closeAll() {
+    this.options?.logger?.debug("closing all db connections");
+    for (const name of this.map.keys()) {
+      const db = this.map.get(name);
+      if (db && db.status === "connected") {
+        this.options?.logger?.debug("disconnecting db connection [%s]", name);
+        await db.disconnect();
+      }
+    }
+  }
+
+  async deleteAll() {
+    await this.closeAll();
+    this.options?.logger?.debug("clear all db connections");
+    this.map.clear();
+  }
+
+  getDB(name: string): Database | null {
+    const db = this.map.get(name);
+    return db ? db : null;
+  }
+
+  setDB(name: string, db: Database): void {
+    if (this.getDB(name)) {
+      throw new Error("cannot override db connection [" + name + "]");
+    }
+    this.map.set(name, db);
+  }
+
+}

package/src/services/utils/get-route.ts

@@ -0,0 +1,70 @@
+import { APIRoute, normalizePath, RouterHandlerOptions } from "@miqro/core";
+import { join } from "node:path";
+
+export interface RouterUseCall {
+  path?: string;
+  method?: string;
+  inflatePath: string;
+  defaultInflatePath: string;
+  options?: RouterHandlerOptions;
+}
+
+export function getRoutes(prePath: string, defaultPath: string, apiOptions?: Partial<APIRoute>): RouterUseCall[] {
+  const ret: RouterUseCall[] = [];
+  const path = apiOptions && apiOptions.path ? apiOptions.path : defaultPath;
+  const defaultInflatePath = join(prePath, defaultPath);
+  const method = apiOptions && apiOptions.method ? apiOptions.method : "GET";
+
+  const methods = method instanceof Array ? method : [method];
+  const paths = path instanceof Array ? path : [path];
+
+  if (apiOptions && apiOptions.path === null) {
+    for (const m of methods) {
+      ret.push({
+        options: {
+          apiName: apiOptions?.apiName,
+          description: apiOptions?.description,
+          identifier: apiOptions?.identifier,
+          middleware: apiOptions?.middleware,
+          name: apiOptions?.name,
+          parser: apiOptions?.parser,
+          policy: apiOptions?.policy,
+          request: apiOptions?.request,
+          response: apiOptions?.response,
+          session: apiOptions?.session
+        },
+        inflatePath: defaultInflatePath,
+        defaultInflatePath, 
+        method: m.toLocaleLowerCase() !== "use" ? m.toUpperCase() : undefined,
+        path: undefined
+      });
+    }
+  } else {
+    for (const p of paths) {
+      for (const m of methods) {
+        ret.push({
+          options: {
+            apiName: apiOptions?.apiName,
+            description: apiOptions?.description,
+            identifier: apiOptions?.identifier,
+            middleware: apiOptions?.middleware,
+            name: apiOptions?.name,
+            parser: apiOptions?.parser,
+            policy: apiOptions?.policy,
+            request: apiOptions?.request,
+            response: apiOptions?.response,
+            session: apiOptions?.session
+          },
+          inflatePath: p !== "/" ? join(prePath, p) : defaultInflatePath,
+          defaultInflatePath,
+          method: m.toLocaleLowerCase() !== "use" ? m.toUpperCase() : undefined,
+          path: normalizePath(join(prePath, p))
+        });
+      }
+    }
+  }
+
+
+
+  return ret;
+}

package/src/services/utils/log-transport.ts

@@ -0,0 +1,81 @@
+import { ConsoleTransport, FileTransport, LoggerTransport, LoggerTransportWriteArgs, LogLevel, WriteArgs } from "@miqro/core";
+import { format } from "node:util";
+import { LOG_SOCKET_PATH, LOG_WRITE_EVENT } from "../../../editor/common/constants.js";
+import { Miqro } from "../app.js";
+
+export function createLogProviderOptions(app: Miqro) {
+  const defaultConsole = ConsoleTransport();
+  //console.log("app.options.logFile [%s]", app.options.logFile);
+  const defaultFile: LoggerTransport | undefined =
+    app.options.logFile !== true && app.options.logFile !== false && String(app.options.logFile).toUpperCase() !== "TRUE" && String(app.options.logFile).toUpperCase() !== "FALSE" &&
+      app.options.logFile ? FileTransport(app.options.logFile) :
+      String(app.options.logFile).toUpperCase() === "TRUE" || app.options.logFile === true || app.options.logFile === undefined ?
+        FileTransport() :
+        undefined;
+  const defaultWrite = async (args: LoggerTransportWriteArgs, level?: LogLevel) => {
+    try {
+      const serviceNamesWithLogConfigReplaceConsole = level === undefined && app.inflated ?
+        Object.keys(app.inflated.logConfigMap).filter(serviceName => app.inflated.logConfigMap[serviceName].replaceConsoleTransport) : [];
+      const serviceNamesWithLogConfigReplaceFile = level === undefined && app.inflated ?
+        Object.keys(app.inflated.logConfigMap).filter(serviceName => app.inflated.logConfigMap[serviceName].replaceFileTransport) : [];
+      await Promise.allSettled((level === undefined ?
+        [
+          level === undefined && serviceNamesWithLogConfigReplaceConsole.length === 0 && defaultConsole ?
+            defaultConsole.write(args) : Promise.resolve(),
+          level === undefined && serviceNamesWithLogConfigReplaceFile.length === 0 && defaultFile ?
+            defaultFile.write(args) : Promise.resolve()
+        ] : []).concat(app.inflated ?
+          Object.keys(app.inflated.logConfigMap).map(serviceName => app.inflated.logConfigMap[serviceName]).filter(c => c.level === level).map(c => c.write(args)) : []
+        ));
+    } catch (e) {
+      console.error(e);
+    }
+  }
+  return {
+    name: app.options.name,
+    formatter: (args: WriteArgs) => {
+      const params: string[] = args.optionalParams;
+      return format(
+        `${new Date().toISOString()} PID[${process.pid}] ${args.identifier ? `[${args.identifier}] ` : ""}${args.level !== "info" ? (args.level === "error" || args.level === "warn" ? `[${args.level.toUpperCase()}] ` : `[${args.level}] `) : ""}${args.message}`,
+        ...params)
+    },
+    transports: [
+      ...(([undefined, "error", "warn", "info", "debug", "trace"] as LogLevel[]).map(level => {
+        return level ? {
+          level,
+          write: async (args) => {
+            await defaultWrite(args, level);
+          }
+        } : {
+          write: async (args) => {
+            await defaultWrite(args, undefined);
+          }
+        }
+      })), {
+        level: "trace" as LogLevel,
+        write: async (args) => {
+          try {
+            if (app.options.editor) {
+              try {
+                const ws = app.webSocketManager.getWS(LOG_SOCKET_PATH);
+                if (ws) {
+                  //console.log("\n\n" + process.pid + " broadcasting " + LOG_SOCKET_PATH + "\n\n\n")
+                  await ws.broadcast(JSON.stringify({
+                    type: LOG_WRITE_EVENT,
+                    level: args.level,
+                    identifier: args.identifier,
+                    out: args.out
+                  }));
+                }
+              } catch (e) {
+                console.error(e);
+              }
+            }
+          } catch (e) {
+            console.error(e);
+          }
+        }
+      }],
+    ...(app.options.logProviderOptions ? app.options.logProviderOptions : {})
+  };
+}