noxt-server 0.1.13 → 0.1.15

package/noxt-server.js

@@ -5,31 +5,56 @@ import MLM from 'mlm-core';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
-export async function startServer({ config, recipe }) {
+function resolveModule(name) {
+  if (!name.includes('/')) {
+    return `${__dirname}/units/${name}.js`;
+  }
+  if (name.startsWith('app/')) {
+    return `${process.cwd()}/units/${name.slice(4)}.js`;
+  }
+  if (name.startsWith('contrib/')) {
+    const m = name.match(/^contrib\/([^/]+)(?:\/(.*))?$/);
+    if (!m) throw new Error(`Invalid contrib module name: ${name}`);
+    if (!m[2]) return `noxt-contrib-${m[1]}`;
+    return `noxt-contrib-${m[1]}/${m[2]}`;
+  }
+  throw new Error(`Invalid module name: ${name}`);
+}
+
+export async function startServer({ config, recipe, import: _import}) {
  
-  const mlmInstance = MLM({
-    import:
-      p => import(p),
-    resolveModule:
-      name => name.startsWith('app/')
-        ? `${process.cwd()}/units/${name.slice(4)}.js`
-        : `${__dirname}/units/${name}.js`
+  const beforeBoot = process.hrtime.bigint();
+  const mlmInstance = await MLM({
+    import: _import ?? (p => import(p)),
+    resolveModule
   });
   try {
-
+    const afterBoot = process.hrtime.bigint();
     const report = await mlmInstance.analyze(recipe ?? 'noxt-dev');
     if (!report.success) {
       console.log('Bad recipe: ' + recipe + '\n' + report.order.join(', '));
       console.log(report.errors.join('\n'));
       process.exit(1);
     }
-    
+    const afterAnalyze = process.hrtime.bigint();
     await mlmInstance.install(recipe ?? 'noxt-dev');
+    const afterInstall = process.hrtime.bigint();
     const mlm = mlmInstance.context;
     await mlm.services.config.merge(config);
-    console.log(mlm.config);
+    const afterConfig = process.hrtime.bigint();
     await mlmInstance.start();
+    const afterStart = process.hrtime.bigint();
+
+    console.log(
+      '[noxt-server] Total time', Number((afterStart - beforeBoot) / 1_000_000n), 'ms',
+      '| Boot', Number((afterBoot - beforeBoot) / 1_000_000n), 'ms',
+      '| Load', Number((afterAnalyze - afterBoot) / 1_000_000n), 'ms',
+      '| Install', Number((afterInstall - afterAnalyze) / 1_000_000n), 'ms',
+      '| Config', Number((afterConfig - afterInstall) / 1_000_000n), 'ms',
+      '| Start', Number((afterStart - afterConfig) / 1_000_000n), 'ms',
+    );
 
+    mlmInstance.repl();
   } catch (e) {
     console.log(await mlmInstance.analyze(recipe ?? 'noxt-dev'));
     console.error(e);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "noxt-server",
-  "version": "0.1.13",
+  "version": "0.1.15",
   "description": "Server for noxt-js-middleware with CLI and config support",
   "type": "module",
   "main": "noxt-server.js",
@@ -23,13 +23,15 @@
   "author": "Zoran Obradović [https://github.com/zocky]",
   "license": "GPL-3.0-or-later",
   "dependencies": {
-    "noxt-js-middleware": "^1.0.5",
-    "mlm-core": "^1.0.5",
-    "express": "^5.0.0",
+    "noxt-js-middleware": "^1.0.7",
+    "mlm-core": "^1.0.7",
+    "express": "^5.1.0",
     "cookie-parser": "^1.4.6",
-    "js-yaml": "^4.1.0",
-    "minimist": "^1.2.8",
-    "node-fetch-cache": "^5.0.2"
+    "js-yaml": "^4.1.0"
+  },
+  "peerDependencies": {
+    "compression": "^1.8.1",
+    "sharp": "^0.34.5"
   },
   "devDependencies": {
     "noxt-js-middleware": "../noxt-js-middleware",

package/units/bundler.js

@@ -0,0 +1,46 @@
+export const info = {
+  name: 'bundler',
+  version: '1.0.0',
+  description: 'Bundle js and css from component exports',
+  requires: ['noxt-plugin'],
+}
+
+export default mlm => {
+  const styles = {};
+  const scripts = {};
+  return {
+    'config.bundler': {
+      is: {
+        style: 'string|false',
+        script: 'string|false',
+      },
+      default: {
+        style: '_style',
+        script: '_script',
+      },
+    },
+    'services.bundler': () => ({ styles, scripts }),
+    'registerComponent.bundler': async ({ name, component, module }) => {
+      styles[component.name] = module.style ? `/* ${name} */ \n ${module.style}\n` : '';
+      scripts[component.name] = module.script ? `/* ${name} */ \n ${module.script}\n` : '';
+    },
+    'middleware.bundler': () => {
+      const router = mlm.services.express.Router();
+      if (mlm.config.bundler.style) {
+        router.get(`/${mlm.config.bundler.style}`, (req, res) => {
+          res.logGroup = 'bundler';
+          res.set('Content-Type', 'text/css');
+          res.send( Object.values(styles).join('') );
+        });
+      }
+      if (mlm.config.bundler.script) {
+        router.get(`/${mlm.config.bundler.script}`, (req, res) => {
+          res.logGroup = 'bundler';
+          res.set('Content-Type', 'text/javascript');
+          res.send( Object.values(scripts).join('') );
+        });
+      }
+      return router
+    }
+  }
+}

package/units/compression.js

@@ -0,0 +1,33 @@
+// units/compression.js
+export const info = {
+  name: 'compression',
+  version: '1.0.0',
+  description: 'Response compression',
+  requires: ['express'],
+  npm: {
+    'compression': '^1.8.0'
+  },
+
+}
+
+export default mlm => ({
+  'config.compression': {
+    is: {
+      enabled: 'boolean',
+      level: 'number',  // 0-9, default 6
+      threshold: 'number',  // bytes, default 1024
+    },
+    default: {
+      enabled: true,
+      level: 6,
+      threshold: 1024,  // only compress responses > 1kb
+    },
+  },
+  
+  'middleware.compression': async () => {
+    const { enabled, level, threshold } = mlm.config.compression;
+    if (!enabled) return null;
+    const compression = (await mlm.import('compression')).default;
+    return compression({ level, threshold });
+  },
+});

package/units/config.js

@@ -5,6 +5,7 @@ export const info = {
   requires: ['utils','services'],
 }
 
+const config = {};
 const config_is = {};
 const config_defaults = {};
 const config_defs = {}
@@ -54,7 +55,11 @@ function isPlainObject(value) {
 }
 
 export default mlm => ({
-  'define.config': () => ({}),
+  'define.config': {
+    get: () => config,
+    enumerable: true,
+    configurable: false
+  },
   'register.config': (defs, unit) => {
     for (const key in defs) {
       const def = defs[key];
@@ -78,8 +83,8 @@ export default mlm => ({
   },
   'utils.merge_deep': merge_deep,
   'services.config': () => new class ConfigService {
-    process(config) {
-      const merged = merge_deep({}, config_defaults, config);
+    process(userConfig) {
+      const merged = merge_deep({}, config_defaults, userConfig);
       const ret = {};
       for (const key in config_defs) {
         const { is: type, normalize } = config_defs[key];
@@ -94,10 +99,10 @@ export default mlm => ({
       }
       return ret;
     }
-    merge(config) {
-      mlm.log('config', config,mlm.config);
-      Object.assign(mlm.config, this.process(config));
-      
+    merge(userConfig) {
+//      console.log('initial',config_defaults,userConfig)
+      Object.assign(config, this.process(userConfig));
+//      console.log(userConfig,'final',config)
     }
     get_defs() {
       return {

package/units/env.js

@@ -3,12 +3,12 @@ export const info = {
   version: '1.0.0',
   description: 'Environment',
 }
-/*
+
 export default mlm => ({
   'define.DEV': () => process.env.NODE_ENV !== 'production',
   'define.PROD': () => process.env.NODE_ENV === 'production',
   'onBeforeLoad': () => {
-    //process.env.NODE_ENV ||= 'development'
+    process.env.NODE_ENV ||= 'development'
   }
 })
-  */
+  

package/units/express.js

@@ -8,54 +8,92 @@ export const info = {
   },
 }
 
-let app;
-const middlewareNames = new Set();
-const middlewares = [];
-export default mlm => ({
-  'define.express': { get: () => app },
-  'register.middleware': (conf, unit) => {
-    for (const key in conf) {
-      mlm.assert.not(key in middlewareNames, 'Duplicate middleware ' + key);
-      middlewareNames.add(key);
-      let c = conf[key];
-      if (mlm.is.function(c)) {
-        c = { create: c };
+export default async mlm => {
+  let app;
+  const middlewareNames = new Set();
+  const middlewares = [];
+  const { default: express } = await mlm.import('express');
+  return {
+    'services.express': () => new class {
+      get app () {
+        return app;
       }
-      mlm.assert.is({
-        path: 'string|none',
-        create: 'function'
-      }, c, 'middleware');
-      mlm.assert.is.function(c.create, 'middleware');
-      middlewares.push({ ...c, unit: unit.name });
-    }
-  },
-  'config.port': {
-    is: v => Number.isInteger(v) && v > 0 && v < 65536,
-    default: 3000
-  },
-  'config.host': {
-    is: 'string',
-    default: 'localhost'
-  },
-  'middleware.json': async (app) => {
-    const { default: express } = await mlm.import('express');
-    return express.json();
-  },
-  'middleware.urlencoded': async (app) => {
-    const { default: express } = await mlm.import('express');
-    return express.urlencoded({ extended: true });
-  },
-  async onStart() {
-    const { default: express } = await mlm.import('express');
-    app = express();
-    for (const middleware of middlewares) {
-      const mw = await middleware.create(app);
-      if (middleware.path) {
-        app.use(middleware.path, mw);
-      } else {
-        app.use(mw);
+      get Router() {
+        return express.Router;
       }
+      get express () {
+        return express;
+      }
+    },
+    'register.middleware': (conf, unit) => {
+      for (const key in conf) {
+        mlm.assert.not(key in middlewareNames, 'Duplicate middleware ' + key);
+        middlewareNames.add(key);
+        let c = conf[key];
+        if (mlm.is.function(c)) {
+          c = { create: c };
+        }
+        mlm.assert.is({
+          path: 'string|none',
+          create: 'function'
+        }, c, 'middleware');
+        mlm.assert.is.function(c.create, 'middleware');
+        middlewares.push({ ...c, unit: unit.name });
+      }
+    },
+    'config.port': {
+      is: v => Number.isInteger(v) && v > 0 && v < 65536,
+      default: 3000
+    },
+    'config.host': {
+      is: 'string',
+      default: 'localhost'
+    },
+    'middleware.json': async (app) => {
+      return express.json();
+    },
+    'middleware.urlencoded': async (app) => {
+      return express.urlencoded({ extended: true });
+    },
+    async onStart() {
+      app = express();
+      for (const middleware of middlewares) {
+        const mw = await middleware.create(app);
+        if (!mw) {
+          continue;
+        }
+        if (middleware.path) {
+          app.use(middleware.path, mw);
+        } else {
+          app.use(mw);
+        }
+      }
+      app.use('/.well-known', (req, res, next) => {
+        res.logGroup = 'well-known';
+        res.status(404).send('Not found');
+        res.end();
+      });
+      app.use((req, res, next) => {
+        res.logGroup = '404';
+        res.status(404).send('Not found');
+      })
+      app.use((error, req, res, next,) => {
+        mlm.log('error', error);
+        res.status(500).send(error.stack);
+      })
+      const server = app.listen(mlm.config.port, mlm.config.host, (error) => {
+        if (error) {
+          mlm.throw(error);
+        }
+        mlm.log(`Listening on ${mlm.config.host}:${mlm.config.port}`);
+      });
+      server.on('close', () => {
+        mlm.log('Server closed');
+      });
+      app.on('error', (error) => {
+        mlm.error(error);
+        app.throw(error);
+      });
     }
-    app.listen(mlm.config.port, mlm.config.host);
-  },
-})
+  }
+}

package/units/fetch-cache-fs.js

@@ -1,54 +1,181 @@
 export const info = {
   name: 'fetch-cache-fs',
   version: '1.0.0',
-  description: 'Cached fetch with filesystem cache',
-  requires: ['noxt-plugin'],
   provides: ['#fetch'],
-  npm: {
-    'node-fetch-cache': '^1.0.0',
-  },
+  description: 'Cached fetch with stale-while-revalidate',
 }
 
-let fetchCache = null;
+import { createHash } from 'crypto';
+import { readFile, writeFile, mkdir, stat } from 'fs/promises';
+import { join } from 'path';
 
-export default mlm => ({
-  'config.fetch': {
-    is: {
-      cacheDir: 'string',
-      ttl: 'integer|none',
-      enabled: 'boolean',
-      retry: 'positiveInteger',
-      timeout: 'positiveInteger'
-    },
-    default: {
-      enabled: true,
-      cacheDir: '.cache/fetch-cache',
-      ttl: 60 * 60 * 1000,
-      retry: 2,
-      timeout: 5000
-    },
-  },
-  'pageContext.fetch': ({ ctx }) => globalFetch.bind(null, ctx),
-  'serverContext.fetch': () => globalFetch,
-  'onStart': async () => {
-    const { default: nodeFetchCache, FileSystemCache } = await mlm.import('node-fetch-cache');
-    const cfg = mlm.config.fetch;
-    fetchCache = nodeFetchCache.create({
-      cache: cfg.enabled ? new FileSystemCache({ cacheDirectory: cfg.cacheDir }) : undefined,
-      ttl: cfg.ttl,
-      retry: cfg.retry,
-      timeout: cfg.timeout
-    });
+export default mlm => {
+  const inflight = new Map();
+  const refreshing = new Map();
+
+  async function cachedFetch(url, options = {}) {
+    const forceFresh = options.cache === 'no-cache';
+    const isGet = !options.method || options.method === 'GET';
+    
+    if (!isGet) return fetchWithRetry(url, options);
+    
+    const cacheKey = createHash('sha256').update(url).digest('hex');
+    const cachePath = join(mlm.config.fetch.cacheDir, cacheKey + '.json');
+    
+    if (inflight.has(cacheKey)) {
+      return inflight.get(cacheKey);
+    }
+    
+    if (!forceFresh) {
+      const cached = await readCache(cachePath);
+      if (cached) {
+        const age = Date.now() - cached.cachedAt;
+        
+        if (age <= mlm.config.fetch.stale) {
+          return toResponse(cached);
+        }
+        
+        if (age <= mlm.config.fetch.ttl) {
+          refreshInBackground(url, cachePath, options, cacheKey);
+          return toResponse(cached);
+        }
+      }
+    }
+    
+    const promise = (async () => {
+      try {
+        const res = await fetchWithRetry(url, options);
+        
+        if (res.ok) {
+          const body = await res.text();
+          await writeCache(cachePath, res, body);
+          return new Response(body, { status: res.status, headers: res.headers });
+        }
+      } catch (error) {
+        // Fetch failed completely - fall through to stale cache check
+        mlm.log('fetch failed, checking for stale cache:', error.message);
+      }
+      
+      // Fallback to stale cache on error or non-ok response
+      const cached = await readCache(cachePath);
+      if (cached && Date.now() - cached.cachedAt <= mlm.config.fetch.ttl) {
+        mlm.log('returning stale cache for', url);
+        return toResponse(cached);
+      }
+      
+      // No cache available, throw or return error response
+      throw new Error(`Failed to fetch ${url} and no cache available`);
+    })();
+    
+    inflight.set(cacheKey, promise);
+    
+    try {
+      return await promise;
+    } finally {
+      inflight.delete(cacheKey);
+    }
+  }
+
+  async function refreshInBackground(url, cachePath, options, cacheKey) {
+    if (refreshing.has(cacheKey)) {
+      return;
+    }
+    
+    const promise = (async () => {
+      try {
+        const res = await fetchWithRetry(url, options);
+        if (res.ok) {
+          const body = await res.text();
+          await writeCache(cachePath, res, body);
+        }
+      } catch {}
+    })();
+    
+    refreshing.set(cacheKey, promise);
+    
+    try {
+      await promise;
+    } finally {
+      refreshing.delete(cacheKey);
+    }
+  }
+
+  async function fetchWithRetry(url, options) {
+    let lastError;
+    
+    for (let i = 0; i <= mlm.config.fetch.retry; i++) {
+      const controller = new AbortController();
+      const timeout = setTimeout(() => controller.abort(), mlm.config.fetch.timeout);
+      
+      try {
+        mlm.log('fetching', url);
+        const res = await fetch(url, { ...options, signal: controller.signal });
+        clearTimeout(timeout);
+        return res;
+      } catch (e) {
+        mlm.log('failed',url,e.message)
+        clearTimeout(timeout);
+        lastError = e;
+        if (i < mlm.config.fetch.retry) {
+          await new Promise(r => setTimeout(r, 100 * (i + 1)));
+        }
+      }
+    }
+    
+    throw lastError;
+  }
+
+  async function readCache(path) {
+    try {
+      const stats = await stat(path);
+      const data = JSON.parse(await readFile(path, 'utf8'));
+      data.cachedAt = stats.mtimeMs;
+      return data;
+    } catch {
+      return null;
+    }
+  }
+
+  async function writeCache(path, res, body) {
+    await writeFile(path, JSON.stringify({
+      url: res.url,
+      status: res.status,
+      headers: Object.fromEntries(res.headers),
+      body,
+    }));
   }
-});
-
-async function globalFetch(ctx, url, options = {}) {
-  let res = await fetchCache(url, options);
-  if (ctx.req.headers.pragma === 'no-cache' && !res.isCacheMiss) {
-    console.log('reload', url);
-    res.ejectFromCache();
-    res = await fetch(url, options);
+
+  function toResponse(cached) {
+    return new Response(cached.body, {
+      status: cached.status,
+      headers: new Headers(cached.headers),
+    });
   }
-  if (!res.ok) throw new Error(res.statusText);
-  return res;
-}
+
+  return {
+    'config.fetch': {
+      is: {
+        cacheDir: 'string',
+        ttl: 'integer',
+        stale: 'integer',
+        retry: 'positiveInteger',
+        timeout: 'positiveInteger'
+      },
+      default: {
+        cacheDir: '.cache/fetch',
+        ttl: 24 * 60 * 60 * 1000,
+        stale: 60 * 60 * 1000,
+        retry: 2,
+        timeout: 5000
+      }
+    },
+
+    'services.fetch': () => ({ 
+      fetch: cachedFetch,
+    }),
+
+    async onStart() {
+      await mkdir(mlm.config.fetch.cacheDir, { recursive: true });
+    }
+  };
+};

package/units/fetch-cache.js

@@ -25,7 +25,7 @@ export default mlm => ({
     }
   },
 
-  'pageContext.fetch': (props, ctx) => globalFetch.bind(null, ctx),
+  'requestContext.fetch': (props, ctx) => globalFetch.bind(null, ctx),
   'serverContext.fetch': () => globalFetch,
 
   async onStart() {

package/units/fetch-node.js

@@ -11,19 +11,6 @@ export default mlm => ({
   requires: ['noxt-plugin'],
   provides: ['#fetch'],
   description: 'Native fetch',
-  'config.fetch': {
-    is: {
-      ttl: 'integer|none',
-      retry: 'positiveInteger|none',
-      timeout: 'positiveInteger|none'
-    },
-    default: {
-      ttl: 60 * 60 * 1000,
-      retry: 2,
-      timeout: 5000
-    }
-  },
-
   'serverContext.fetch': () => fetchOrThrow,
 });