miqro 6.2.12 → 6.3.0

package/src/inflate/setup-http.ts

@@ -1,5 +1,5 @@
 import { Router, newURL, Response, Request, RouterHandlerOptions, Logger, APIRoute, normalizePath, HandlerWithOptions, Handler } from "@miqro/core";
-import { existsSync, mkdirSync, readFile, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
+import { existsSync, mkdir, readFile, readFileSync, readdirSync, statSync } from "node:fs";
 import { dirname, join, relative, resolve as pathResolve } from "node:path";
 
 import { CONTENT_TYPE_MAP, DEFAULT_CONTENT_TYPE } from "../common/content-type.js";
@@ -13,7 +13,7 @@ import { getHTTPRouterPath, getStaticFilesPath } from "../common/paths.js";
 import { setupCORS } from "./setup-cors.js";
 import { setupAUTH } from "./setup-auth.js";
 import { getRoutes } from "../services/utils/get-route.js";
-import { describeFilePath } from "../common/fs.js";
+import { describeFilePath, mkdirASync, writeFileASync } from "../common/fs.js";
 import { inflateMD2HTML } from "./md.js";
 import { MiddlewareConfig, ServerInterface, ServerRequest } from "../types.js";
 import { setupMiddleware } from "./setup-middleware.js";
@@ -45,7 +45,7 @@ export interface StaticFileMap {
   }
 }
 
-export async function setupHTTPRouter(server: ServerInterface, logger: Logger, hotreload: boolean, servicePath: string, service: string, routeFileMap: RouteFileMap, staticFileMap: StaticFileMap | null, inflateDir: string | undefined | false, inflateSea: boolean, errors: InflateError[]) {
+export async function setupHTTPRouter(server: ServerInterface, logger: Logger, hotreload: boolean, servicePath: string, service: string, routeFileMap: RouteFileMap, staticFileMap: StaticFileMap | null, inflateDir: string | undefined | false, inflateSea: boolean, errors: InflateError[], inflateParallel?: number) {
   const mainRouter = new Router();
   const apiRouterPath = getHTTPRouterPath(servicePath); //resolve(process.cwd(), service, "http");
   let middlewareConfig: MiddlewareConfig | null = null;
@@ -58,14 +58,14 @@ export async function setupHTTPRouter(server: ServerInterface, logger: Logger, h
 
   if (apiRouterPath) {
     logger.trace("setting up http routes from [%s]", service);
-    const { router: httpRouter } = await createRouterFromDirectory(server, hotreload, service, logger, apiRouterPath, errors, routeFileMap, staticFileMap, inflateDir, inflateSea);
+    const { router: httpRouter } = await createRouterFromDirectory(server, hotreload, service, logger, apiRouterPath, errors, routeFileMap, staticFileMap, inflateDir, inflateSea, inflateParallel);
     mainRouter.use(httpRouter);
   }
 
   const staticFilesPath = getStaticFilesPath(servicePath); //resolve(process.cwd(), service, "static");
   if (staticFilesPath) {
     logger.trace("setting up static file routes from [%s]", service);
-    const staticRouter = createStaticRouterFromDirectory(service, logger, staticFilesPath, inflateDir, routeFileMap, staticFileMap);
+    const staticRouter = await createStaticRouterFromDirectory(service, logger, staticFilesPath, inflateDir, routeFileMap, staticFileMap, inflateParallel);
     mainRouter.use(staticRouter);
   }
 
@@ -78,95 +78,123 @@ export async function setupHTTPRouter(server: ServerInterface, logger: Logger, h
   return mainRouter;
 }
 
-function createStaticRoute(service: string, logger: Logger, router: Router, dir: string, file: ScannedFile, inflateDir?: string | undefined | false, routeFileMap?: RouteFileMap, staticFileMap?: StaticFileMap) {
-  logger.trace("creating static route for [%s]", file.filePath);
-  logger.trace("[%o]", {
-    file,
-    dir
-  });
-  const contentType = CONTENT_TYPE_MAP[String(file.ext).toLocaleLowerCase()];
-  const path = join("/", relative(dir, file.filePath));
-
-
-  routeFileMap[file.filePath] = {
-    routes: [{
-      method: "GET",
-      path: normalizePath(path)
-    }],
-    service,
-    filePath: file.filePath,
-    previewMethod: "html"
-  };
-
-  if (inflateDir) {
-    const inflatePath = join(inflateDir, service, "static", path);
-    mkdirSync(dirname(inflatePath), {
-      recursive: true
-    });
-    logger.log("writing [%s]", relative(cwd(), inflatePath));
-    const body = readFileSync(file.filePath);
-    writeFileSync(inflatePath, body);
-    if (staticFileMap) {
-      staticFileMap[file.filePath] = {
-        contentType,
+async function createStaticRoute(service: string, logger: Logger, router: Router, dir: string, file: ScannedFile, inflateDir?: string | undefined | false, routeFileMap?: RouteFileMap, staticFileMap?: StaticFileMap) {
+  return new Promise<void>(async (resolve, reject) => {
+    try {
+      logger.trace("creating static route for [%s]", file.filePath);
+      logger.trace("[%o]", {
+        file,
+        dir
+      });
+      const contentType = CONTENT_TYPE_MAP[String(file.ext).toLocaleLowerCase()];
+      const path = join("/", relative(dir, file.filePath));
+
+
+      routeFileMap[file.filePath] = {
+        routes: [{
+          method: "GET",
+          path: normalizePath(path)
+        }],
+        service,
         filePath: file.filePath,
-        previewMethod: "html",
-        method: "GET",
-        path: normalizePath(path),
-        body: Buffer.from(body),
-        inflatePath: inflateDir ? join(inflateDir, service, "static", path) : undefined
+        previewMethod: "html"
+      };
+
+      if (inflateDir) {
+        const inflatePath = join(inflateDir, service, "static", path);
+        mkdir(dirname(inflatePath), {
+          recursive: true
+        }, (err) => {
+
+        })
+        await mkdirASync(dirname(inflatePath), {
+          recursive: true
+        });
+        logger.log("writing [%s]", relative(cwd(), inflatePath));
+        const body = readFileSync(file.filePath);
+        await writeFileASync(inflatePath, body);
+        if (staticFileMap) {
+          staticFileMap[file.filePath] = {
+            contentType,
+            filePath: file.filePath,
+            previewMethod: "html",
+            method: "GET",
+            path: normalizePath(path),
+            body: Buffer.from(body),
+            inflatePath: inflateDir ? join(inflateDir, service, "static", path) : undefined
+          }
+        }
       }
-    }
-  }
 
 
 
-  router.use(assertGlobalTampered);
-  router.get(path, async function (_req, res) {
-    await new Promise<void>((resolve, reject) => {
-      try {
-        readFile(file.filePath, async (err, body) => {
-          if (err) {
-            reject(err);
-          } else {
-            try {
-              await res.asyncEnd({
-                status: 200,
-                headers: {
-                  ["Content-Type"]: contentType ? contentType : DEFAULT_CONTENT_TYPE
-                },
-                body
-              });
-              resolve();
-            } catch (e) {
-              reject(e);
-            }
+      router.use(assertGlobalTampered);
+      router.get(path, async function (_req, res) {
+        await new Promise<void>((resolve, reject) => {
+          try {
+            readFile(file.filePath, async (err, body) => {
+              if (err) {
+                reject(err);
+              } else {
+                try {
+                  await res.asyncEnd({
+                    status: 200,
+                    headers: {
+                      ["Content-Type"]: contentType ? contentType : DEFAULT_CONTENT_TYPE
+                    },
+                    body
+                  });
+                  resolve();
+                } catch (e) {
+                  reject(e);
+                }
+              }
+            })
+          } catch (e) {
+            reject(e);
           }
-        })
-      } catch (e) {
-        reject(e);
-      }
-    });
+        });
+      });
+      resolve();
+    } catch (e) {
+      reject(e);
+    }
   });
 }
 
-function createStaticRouterFromDirectory(service: string, logger: Logger, dir: string, inflateDir: string | false | undefined, routeFileMap: RouteFileMap | undefined, staticFileMap: StaticFileMap | null): Router {
+async function createStaticRouterFromDirectory(service: string, logger: Logger, dir: string, inflateDir: string | false | undefined, routeFileMap: RouteFileMap | undefined, staticFileMap: StaticFileMap | null, inflateParallel?: number): Promise<Router> {
   const router = new Router();
-  scanFiles(dir).forEach(file => {
-    createStaticRoute(service, logger, router, dir, file, inflateDir, routeFileMap, staticFileMap);
-  });
+  const maxParallel = inflateParallel ? inflateParallel : 1;
+  logger.debug("loading static directory with parallel [%s]", maxParallel);
+  let tR = [];
+  const files = scanFiles(dir);
+  for (const file of files) {
+    tR.push(await createStaticRoute(service, logger, router, dir, file, inflateDir, routeFileMap, staticFileMap));
+    if (tR.length >= maxParallel) {
+      await Promise.all(tR);
+      tR = [];
+    }
+  }
+  if (tR.length > 0) {
+    await Promise.all(tR);
+    tR = [];
+  }
   return router;
 }
 
-async function createRouterFromDirectory(server: ServerInterface, hotreload: boolean, service: string, logger: Logger, dir: string, errors: InflateError[] = [], routeFileMap: RouteFileMap = {}, staticFileMap: StaticFileMap | null = null, inflateDir: string | undefined | false, inflateSea: boolean): Promise<{
+async function createRouterFromDirectory(server: ServerInterface, hotreload: boolean, service: string, logger: Logger, dir: string, errors: InflateError[] = [], routeFileMap: RouteFileMap = {}, staticFileMap: StaticFileMap | null = null, inflateDir: string | undefined | false, inflateSea: boolean, inflateParallel?: number): Promise<{
   router: Router;
   errors: InflateError[];
   routeFileMap: RouteFileMap;
 }> {
   const router = new Router();
+  const maxParallel = inflateParallel ? inflateParallel : 1;
+  server.logger.debug("loading http directory with parallel [%s]", maxParallel);
+  let tR = [];
   router.use(assertGlobalTampered);
-  for (const file of scanFiles(dir)) {
-    await new Promise<void>(async (resolve) => {
+  const files = scanFiles(dir);
+  for (const file of files) {
+    tR.push(new Promise<void>(async (resolve) => {
       try {
         switch (file.ext) {
           case ".jsx":
@@ -216,11 +244,11 @@ async function createRouterFromDirectory(server: ServerInterface, hotreload: boo
                   if (inflateDir && r.defaultInflatePath && inflateSea) {
                     const rPath = r.defaultInflatePath;
                     const inflatePath = join(inflateDir, service, "http", rPath + ".api.cjs");
-                    mkdirSync(dirname(inflatePath), {
+                    await mkdirASync(dirname(inflatePath), {
                       recursive: true
                     });
                     logger.log("writing [%s]", relative(cwd(), inflatePath));
-                    writeFileSync(inflatePath, inflatedCode);
+                    await writeFileASync(inflatePath, inflatedCode);
                   }
 
 
@@ -252,7 +280,7 @@ async function createRouterFromDirectory(server: ServerInterface, hotreload: boo
                       //if (r.method === "GET" || r.method === "get") {
                       const rPath = r.inflatePath;
                       const inflatePath = join(inflateDir, service, "static", rPath);
-                      mkdirSync(dirname(inflatePath), {
+                      await mkdirASync(dirname(inflatePath), {
                         recursive: true
                       });
                       if (existsSync(inflatePath) && statSync(inflatePath).isDirectory()) {
@@ -262,7 +290,7 @@ async function createRouterFromDirectory(server: ServerInterface, hotreload: boo
                       const JSON_STATIC = await getJSON({ server } as ServerRequest, null, newURL(r.path), module.apiOptions?.basePath, module.default);
                       //const JSON = await getJSON({ server } as ServerRequest, null, newURL(r.path), module.apiOptions?.basePath, module.default);
                       logger.log("writing [%s]", relative(cwd(), inflatePath));
-                      writeFileSync(inflatePath, JSON_STATIC);
+                      await writeFileASync(inflatePath, JSON_STATIC);
                       //}
 
                       if (staticFileMap && inflateSea) {
@@ -319,7 +347,7 @@ async function createRouterFromDirectory(server: ServerInterface, hotreload: boo
                       //if (r.method === "GET" || r.method === "get") {
                       const rPath = r.inflatePath;
                       const inflatePath = join(inflateDir, service, "static", rPath);
-                      mkdirSync(dirname(inflatePath), {
+                      await mkdirASync(dirname(inflatePath), {
                         recursive: true
                       });
                       if (existsSync(inflatePath) && statSync(inflatePath).isDirectory()) {
@@ -330,7 +358,7 @@ async function createRouterFromDirectory(server: ServerInterface, hotreload: boo
                       const HTML_STATIC = await getHTML(hotreload, { server } as ServerRequest, null, newURL(r.path), module.apiOptions?.basePath, await toRender);
 
                       logger.log("writing [%s]", relative(cwd(), inflatePath));
-                      writeFileSync(inflatePath, HTML_STATIC);
+                      await writeFileASync(inflatePath, HTML_STATIC);
                       //}
 
 
@@ -390,11 +418,11 @@ async function createRouterFromDirectory(server: ServerInterface, hotreload: boo
 
                   if (inflateDir) {
                     const inflatePath = join(inflateDir, service, "static", path);
-                    mkdirSync(dirname(inflatePath), {
+                    await mkdirASync(dirname(inflatePath), {
                       recursive: true
                     });
                     logger.log("writing [%s]", relative(cwd(), inflatePath));
-                    writeFileSync(inflatePath, code);
+                    await writeFileASync(inflatePath, code);
 
 
                     if (staticFileMap && inflateSea) {
@@ -443,11 +471,11 @@ async function createRouterFromDirectory(server: ServerInterface, hotreload: boo
 
                 if (inflateDir) {
                   const inflatePath = join(inflateDir, service, "static", path);
-                  mkdirSync(dirname(inflatePath), {
+                  await mkdirASync(dirname(inflatePath), {
                     recursive: true
                   });
                   logger.log("writing [%s]", relative(cwd(), inflatePath));
-                  writeFileSync(inflatePath, code);
+                  await writeFileASync(inflatePath, code);
                   if (staticFileMap && inflateSea) {
                     staticFileMap[file.filePath] = {
                       contentType,
@@ -504,11 +532,11 @@ async function createRouterFromDirectory(server: ServerInterface, hotreload: boo
 
                   if (inflateDir) {
                     const inflatePath = join(inflateDir, service, "static", path);
-                    mkdirSync(dirname(inflatePath), {
+                    await mkdirASync(dirname(inflatePath), {
                       recursive: true
                     });
                     logger.log("writing [%s]", relative(cwd(), inflatePath));
-                    writeFileSync(inflatePath, code);
+                    await writeFileASync(inflatePath, code);
                     if (staticFileMap && inflateSea) {
                       staticFileMap[file.filePath] = {
                         contentType,
@@ -562,11 +590,11 @@ async function createRouterFromDirectory(server: ServerInterface, hotreload: boo
 
                   if (inflateDir) {
                     const inflatePath = join(inflateDir, service, "static", path);
-                    mkdirSync(dirname(inflatePath), {
+                    await mkdirASync(dirname(inflatePath), {
                       recursive: true
                     });
                     logger.log("writing [%s]", relative(cwd(), inflatePath));
-                    writeFileSync(inflatePath, code);
+                    await writeFileASync(inflatePath, code);
                     if (staticFileMap && inflateSea) {
                       staticFileMap[file.filePath] = {
                         contentType,
@@ -594,7 +622,7 @@ async function createRouterFromDirectory(server: ServerInterface, hotreload: boo
                 }
               }
             }
-            createStaticRoute(service, logger, router, dir, file, inflateDir, routeFileMap, staticFileMap);
+            await createStaticRoute(service, logger, router, dir, file, inflateDir, routeFileMap, staticFileMap);
             return resolve();
 
         }
@@ -608,7 +636,15 @@ async function createRouterFromDirectory(server: ServerInterface, hotreload: boo
       } finally {
         return resolve();
       }
-    });
+    }));
+    if (tR.length >= maxParallel) {
+      await Promise.all(tR);
+      tR = [];
+    }
+  }
+  if (tR.length > 0) {
+    await Promise.all(tR);
+    tR = [];
   }
   router.use(assertGlobalTampered);
   return {

package/src/main.ts

@@ -23,6 +23,7 @@ async function main(args: Arguments) {
       browser: args.browser,
       logFile: args.logFile,
       hotreload: args.test ? false : args.hotreload,
+      watch: args.test ? false : args.watch,
       https: args.test ? false : args.https,
       serverOptions: args.serverOptions,
       httpRedirect: args.test ? undefined : args.httpsRedirect
@@ -30,7 +31,9 @@ async function main(args: Arguments) {
     // check arguments
     if (args.generateDoc) {
       // --generate-doc
-      const inflated = await app.inflate();
+      const inflated = await app.inflate({
+        inflateParallel: args.inflateParallel
+      });
       await generateDocs(args, app.logger, inflated);
       process.exit(EXIT_CODES.NORMAL_EXIT);
     } else if (args.migrateUp || args.migrateDown) {
@@ -44,7 +47,9 @@ async function main(args: Arguments) {
       process.exit(EXIT_CODES.NORMAL_EXIT);
     } else if (args.test) {
       // --test
-      await app.inflate();
+      await app.inflate({
+        inflateParallel: args.inflateParallel
+      });
       await app.start();
       await testMain(app);
       await app.stop();
@@ -55,7 +60,8 @@ async function main(args: Arguments) {
       // --compile
       await app.inflate({
         inflateDir: args.inflateDir,
-        inflateSea: true
+        inflateSea: true,
+        inflateParallel: args.inflateParallel
       });
       await app.dbManager.closeAll();
       await app.webSocketManager.disconnectAll();
@@ -66,14 +72,17 @@ async function main(args: Arguments) {
       // loadApp with inflateDir arg to inflate inflatable files
       await app.inflate({
         inflateDir: args.inflateDir,
-        inflateSea: args.inflateSEA
+        inflateSea: args.inflateSEA,
+        inflateParallel: args.inflateParallel
       });
       await app.dbManager.closeAll();
       await app.webSocketManager.disconnectAll();
       process.exit(EXIT_CODES.NORMAL_EXIT);
     } else {
       // server
-      await app.inflate();
+      await app.inflate({
+        inflateParallel: args.inflateParallel
+      });
       await app.start();
     }
   }

package/src/services/app.ts

@@ -43,6 +43,7 @@ export interface MiqroOptions {
   browser?: string | boolean;
   logFile?: string | boolean;
   hotreload?: boolean;
+  watch?: boolean;
   serverOptions?: ServerOptions<any, any>;
   https?: boolean;
   httpRedirect?: number;
@@ -51,6 +52,7 @@ export interface MiqroOptions {
 export interface InflateOptions {
   inflateDir?: string;
   inflateSea?: boolean;
+  inflateParallel?: number;
 }
 
 export interface InflatedResult {
@@ -185,6 +187,7 @@ export class Miqro {
     });
     await app.inflate({
       inflateDir: miqroJSON.inflateDir ? String(miqroJSON.inflateDir) : undefined,
+      inflateParallel: miqroJSON.inflateParallel ? miqroJSON.inflateParallel : undefined,
       ...(inflate ? inflate : {}),
     });
     return app;
@@ -370,7 +373,8 @@ export class Miqro {
         inflateDir: options?.inflateDir,
         inflateSea: options?.inflateSea ? true : false,
         //inflateTests: options?.inflateTests ? true : false,
-        hotreload: this.options?.hotreload ? true : false
+        hotreload: this.options?.hotreload ? true : false,
+        inflateParallel: options?.inflateParallel
       });
 
       wsConfigList.push(...serviceWSConfigList);
@@ -491,7 +495,7 @@ export class Miqro {
 
     await notifiyServerConfig(this.logger, this.serverInterface, this.adminInterface, this.inflated.serverConfigMap, "start");
 
-    if (this.options.hotreload && (cluster.isPrimary || process.env["CLUSTER_NODE_NUMBER"] === "0")) {
+    if (this.options.watch && (cluster.isPrimary || process.env["CLUSTER_NODE_NUMBER"] === "0")) {
       this.watcher = await watchAndServer(this);
     }