noxt-server 0.1.14 → 0.1.16

package/units/fetch-cache-fs.js

@@ -1,54 +1,181 @@
 export const info = {
   name: 'fetch-cache-fs',
   version: '1.0.0',
-  description: 'Cached fetch with filesystem cache',
-  requires: ['noxt-plugin'],
   provides: ['#fetch'],
-  npm: {
-    'node-fetch-cache': '^1.0.0',
-  },
+  description: 'Cached fetch with stale-while-revalidate',
 }
 
-let fetchCache = null;
+import { createHash } from 'crypto';
+import { readFile, writeFile, mkdir, stat } from 'fs/promises';
+import { join } from 'path';
 
-export default mlm => ({
-  'config.fetch': {
-    is: {
-      cacheDir: 'string',
-      ttl: 'integer|none',
-      enabled: 'boolean',
-      retry: 'positiveInteger',
-      timeout: 'positiveInteger'
-    },
-    default: {
-      enabled: true,
-      cacheDir: '.cache/fetch-cache',
-      ttl: 60 * 60 * 1000,
-      retry: 2,
-      timeout: 5000
-    },
-  },
-  'pageContext.fetch': ({ ctx }) => globalFetch.bind(null, ctx),
-  'serverContext.fetch': () => globalFetch,
-  'onStart': async () => {
-    const { default: nodeFetchCache, FileSystemCache } = await mlm.import('node-fetch-cache');
-    const cfg = mlm.config.fetch;
-    fetchCache = nodeFetchCache.create({
-      cache: cfg.enabled ? new FileSystemCache({ cacheDirectory: cfg.cacheDir }) : undefined,
-      ttl: cfg.ttl,
-      retry: cfg.retry,
-      timeout: cfg.timeout
-    });
+export default mlm => {
+  const inflight = new Map();
+  const refreshing = new Map();
+
+  async function cachedFetch(url, options = {}) {
+    const forceFresh = options.cache === 'no-cache';
+    const isGet = !options.method || options.method === 'GET';
+    
+    if (!isGet) return fetchWithRetry(url, options);
+    
+    const cacheKey = createHash('sha256').update(url).digest('hex');
+    const cachePath = join(mlm.config.fetch.cacheDir, cacheKey + '.json');
+    
+    if (inflight.has(cacheKey)) {
+      return inflight.get(cacheKey);
+    }
+    
+    if (!forceFresh) {
+      const cached = await readCache(cachePath);
+      if (cached) {
+        const age = Date.now() - cached.cachedAt;
+        
+        if (age <= mlm.config.fetch.stale) {
+          return toResponse(cached);
+        }
+        
+        if (age <= mlm.config.fetch.ttl) {
+          refreshInBackground(url, cachePath, options, cacheKey);
+          return toResponse(cached);
+        }
+      }
+    }
+    
+    const promise = (async () => {
+      try {
+        const res = await fetchWithRetry(url, options);
+        
+        if (res.ok) {
+          const body = await res.text();
+          await writeCache(cachePath, res, body);
+          return new Response(body, { status: res.status, headers: res.headers });
+        }
+      } catch (error) {
+        // Fetch failed completely - fall through to stale cache check
+        mlm.log('fetch failed, checking for stale cache:', error.message);
+      }
+      
+      // Fallback to stale cache on error or non-ok response
+      const cached = await readCache(cachePath);
+      if (cached && Date.now() - cached.cachedAt <= mlm.config.fetch.ttl) {
+        mlm.log('returning stale cache for', url);
+        return toResponse(cached);
+      }
+      
+      // No cache available, throw or return error response
+      throw new Error(`Failed to fetch ${url} and no cache available`);
+    })();
+    
+    inflight.set(cacheKey, promise);
+    
+    try {
+      return await promise;
+    } finally {
+      inflight.delete(cacheKey);
+    }
+  }
+
+  async function refreshInBackground(url, cachePath, options, cacheKey) {
+    if (refreshing.has(cacheKey)) {
+      return;
+    }
+    
+    const promise = (async () => {
+      try {
+        const res = await fetchWithRetry(url, options);
+        if (res.ok) {
+          const body = await res.text();
+          await writeCache(cachePath, res, body);
+        }
+      } catch {}
+    })();
+    
+    refreshing.set(cacheKey, promise);
+    
+    try {
+      await promise;
+    } finally {
+      refreshing.delete(cacheKey);
+    }
+  }
+
+  async function fetchWithRetry(url, options) {
+    let lastError;
+    
+    for (let i = 0; i <= mlm.config.fetch.retry; i++) {
+      const controller = new AbortController();
+      const timeout = setTimeout(() => controller.abort(), mlm.config.fetch.timeout);
+      
+      try {
+        mlm.log('fetching', url);
+        const res = await fetch(url, { ...options, signal: controller.signal });
+        clearTimeout(timeout);
+        return res;
+      } catch (e) {
+        mlm.log('failed',url,e.message)
+        clearTimeout(timeout);
+        lastError = e;
+        if (i < mlm.config.fetch.retry) {
+          await new Promise(r => setTimeout(r, 100 * (i + 1)));
+        }
+      }
+    }
+    
+    throw lastError;
+  }
+
+  async function readCache(path) {
+    try {
+      const stats = await stat(path);
+      const data = JSON.parse(await readFile(path, 'utf8'));
+      data.cachedAt = stats.mtimeMs;
+      return data;
+    } catch {
+      return null;
+    }
+  }
+
+  async function writeCache(path, res, body) {
+    await writeFile(path, JSON.stringify({
+      url: res.url,
+      status: res.status,
+      headers: Object.fromEntries(res.headers),
+      body,
+    }));
   }
-});
-
-async function globalFetch(ctx, url, options = {}) {
-  let res = await fetchCache(url, options);
-  if (ctx.req.headers.pragma === 'no-cache' && !res.isCacheMiss) {
-    console.log('reload', url);
-    res.ejectFromCache();
-    res = await fetch(url, options);
+
+  function toResponse(cached) {
+    return new Response(cached.body, {
+      status: cached.status,
+      headers: new Headers(cached.headers),
+    });
   }
-  if (!res.ok) throw new Error(res.statusText);
-  return res;
-}
+
+  return {
+    'config.fetch': {
+      is: {
+        cacheDir: 'string',
+        ttl: 'integer',
+        stale: 'integer',
+        retry: 'positiveInteger',
+        timeout: 'positiveInteger'
+      },
+      default: {
+        cacheDir: '.cache/fetch',
+        ttl: 24 * 60 * 60 * 1000,
+        stale: 60 * 60 * 1000,
+        retry: 2,
+        timeout: 5000
+      }
+    },
+
+    'services.fetch': () => ({ 
+      fetch: cachedFetch,
+    }),
+
+    async onStart() {
+      await mkdir(mlm.config.fetch.cacheDir, { recursive: true });
+    }
+  };
+};

package/units/fetch-cache.js

@@ -25,7 +25,7 @@ export default mlm => ({
     }
   },
 
-  'pageContext.fetch': (props, ctx) => globalFetch.bind(null, ctx),
+  'requestContext.fetch': (props, ctx) => globalFetch.bind(null, ctx),
   'serverContext.fetch': () => globalFetch,
 
   async onStart() {

package/units/fetch.js

@@ -1,27 +1,43 @@
 export const info = {
+  name: 'fetch',
   version: '1.0.0',
-  description: 'Preload props from urls',
-  requires: ['noxt-plugin','#fetch'],
+  description: 'Preload props from URLs',
+  requires: ['noxt-plugin', '#fetch'],
 }
 
 export default mlm => ({
-  'componentExports.fetch': async ({exported, props, ctx}) => {
+  'requestContext.fetch': ({ ctx }) => {
+    // Wrapper that respects request cache headers
+    return (url, options = {}) => {
+      const forceFresh = ctx.req.headers.pragma === 'no-cache';
+      return mlm.services.fetch.fetch(url, {
+        ...options,
+        cache: forceFresh ? 'no-cache' : options.cache,
+        cors: true
+      });
+    };
+  },
+
+  'componentExports.fetch': async ({ component, exported, props, ctx }) => {
+    const promises = [];
+    const ids = [];
+    const urls = [];
+    
     for (const id in exported) {
-      let url = await mlm.utils.eval(exported[id], props, ctx);
-     // mlm.log('fetching', id, url);
       try {
-        const now = performance.now();
-        const res = await ctx.fetch(url);
-        mlm.log('fetched', id, url, (performance.now() - now).toFixed(3) +'ms');
-        props[id] = await res.json();
-        //mlm.log('fetched', url, JSON.stringify(props[id]).length);
+        const url = await mlm.utils.eval(exported[id], props, ctx);
+        ids.push(id);
+        urls.push(url);
+        promises.push(ctx.fetch(url).then(res => res.json()));
       } catch (e) {
-        if (process.env.NODE_ENV !== 'production') {
-          mlm.throw(new Error('error fetching ' + url + ': ' + e));
-        }
-        mlm.error('fetch', url, e);
         props[id] = null;
       }
     }
+
+    const results = await Promise.allSettled(promises);
+
+    results.forEach((result, index) => {
+      props[ids[index]] = result.status === 'fulfilled' ? result.value : null;
+    });
   },
-})
+});

package/units/hooks.js

@@ -2,7 +2,7 @@ export const info = {
   name: 'hooks',
   version: '1.0.0',
   description: 'Hooks',
-  requires: ['utils','services']
+  requires: ['utils', 'services']
 }
 const hooks = {}
 const hook_handlers = {}
@@ -10,18 +10,31 @@ const hook_handlers = {}
 export default mlm => {
   return ({
     'define.hooks': () => mlm.utils.readOnly(hooks, 'hooks'),
-    'register.hooks': mlm.utils.collector(hooks, {
-      is: 'function',
+    'collect.hooks': ({
+      target: hooks,
+      is: 'function|boolean',
       mode: 'object',
-      map: (fn, key) => (...args) => {
-        for (const handler of hook_handlers[key] ?? []) {
-          fn(handler, ...args);
+      map: (fn, key) => {
+        if (fn === false) return () => { };
+        if (fn === true) return async (...args) => {
+          for (const handler of hook_handlers[key] ?? []) {
+            await handler(...args);
+          }
+        }
+        if (mlm.is.function(fn)) {
+          return async (...args) => {
+            for (const handler of hook_handlers[key] ?? []) {
+              await fn(handler, ...args);
+            }
+          }
         }
+        mlm.throw('Cosmic ray: Invalid hook');
       }
     }),
-    'register.on': mlm.utils.collector(hook_handlers, {
+    'collect.on': {
+      target: hook_handlers,
       is: 'function',
-      mode: 'array'
-    })
+      mode: 'arrays'
+    }
   })
 }

package/units/image-resizer.js

@@ -0,0 +1,263 @@
+export const info = {
+  name: 'image-resizer',
+  description: 'Image resizing and caching service',
+  requires: ['express'],
+  provides: ['#image-resizer'],
+  packages: ['sharp','express']
+}
+
+import crypto from 'crypto';
+import fs from 'fs/promises';
+import path from 'path';
+
+// Pure helper functions (don't need mlm)
+function generateHash(options, maxLength = 16) {
+  const str = JSON.stringify(options);
+  const fullHash = crypto.createHash('sha256').update(str).digest('base64url');
+  return fullHash.substring(0, maxLength);
+}
+
+function normalizeFormat(format) {
+  if (!format) return 'jpeg';
+  if (format === 'jpg') return 'jpeg';
+  return format;
+}
+
+function getShardedPath(hash, ext, depth, segmentLength) {
+  const segments = [];
+  for (let i = 0; i < depth; i++) {
+    segments.push(hash.substring(i * segmentLength, (i + 1) * segmentLength));
+  }
+  segments.push(`${hash}.${ext}`);
+  return segments.join('/');
+}
+
+export default async mlm => {
+  const {default:sharp} = mlm.packages.sharp;
+
+  async function getMetadata(hash) {
+    const shardPath = getShardedPath(hash, 'json', mlm.config.imageResizer.shardDepth, mlm.config.imageResizer.shardSegmentLength);
+    const metaPath = path.join(path.resolve(mlm.config.imageResizer.storage), shardPath);
+    try {
+      const data = await fs.readFile(metaPath, 'utf8');
+      return JSON.parse(data);
+    } catch (err) {
+      return null;
+    }
+  }
+
+  async function saveMetadata(hash, metadata) {
+    const shardPath = getShardedPath(hash, 'json', mlm.config.imageResizer.shardDepth, mlm.config.imageResizer.shardSegmentLength);
+    const metaPath = path.join(path.resolve(mlm.config.imageResizer.storage), shardPath);
+    await fs.mkdir(path.dirname(metaPath), { recursive: true });
+    await fs.writeFile(metaPath, JSON.stringify(metadata, null, 2));
+  }
+
+  async function processImage(hash, metadata) {
+    const format = metadata.format;
+    const ext = format === 'jpeg' ? 'jpg' : format;
+    const shardPath = getShardedPath(hash, ext, mlm.config.imageResizer.shardDepth, mlm.config.imageResizer.shardSegmentLength);
+    const filePath = path.join(path.resolve(mlm.config.imageResizer.storage), shardPath);
+
+    // Ensure directory exists
+    await fs.mkdir(path.dirname(filePath), { recursive: true });
+
+    // Download image
+    const response = await fetch(metadata.url);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch image: ${response.statusText} ${metadata.url}`);
+    }
+
+    const buffer = await response.arrayBuffer();
+    const imageBuffer = Buffer.from(buffer);
+
+    // Process with sharp
+    let image = sharp(imageBuffer);
+    const options = metadata.options;
+
+    // Resize if dimensions provided
+    if (options.width || options.height) {
+      const resizeOptions = {
+        width: +options.width || null,
+        height: +options.height || null,
+        fit: options.fit || 'inside',
+      };
+
+      if (options.background) {
+        resizeOptions.background = options.background;
+      }
+      image = image.resize(resizeOptions);
+    }
+
+    // Set format and quality
+    switch (format) {
+      case 'jpeg':
+        image = image.jpeg({ quality: options.quality });
+        break;
+      case 'png':
+        image = image.png();
+        break;
+      case 'webp':
+        image = image.webp({ quality: options.quality });
+        break;
+      case 'avif':
+        image = image.avif({ quality: options.quality });
+        break;
+    }
+
+    // Process and save
+    const processedBuffer = await image.toBuffer();
+    await fs.writeFile(filePath, processedBuffer);
+
+    // Update metadata
+    metadata.lastProcessed = new Date().toISOString();
+    await saveMetadata(hash, metadata);
+
+    return processedBuffer;
+  }
+
+  async function resizeImage(url, options = {}) {
+    if (!url) {
+      throw new Error('URL is required');
+    }
+
+    // Normalize format
+    const format = normalizeFormat(options.format);
+
+    // Merge with defaults
+    const opts = {
+      url,
+      width: options.width || null,
+      height: options.height || null,
+      fit: options.fit || 'cover',
+      format,
+      quality: options.quality || 80,
+      ...options
+    };
+
+    const hash = generateHash(opts, mlm.config.imageResizer.maxHashLength);
+
+    // Save metadata
+    const metadata = {
+      url,
+      format,
+      options: opts,
+      created: new Date().toISOString(),
+      hash,
+    };
+
+    await saveMetadata(hash, metadata);
+
+    // Return URL with extension and sharded path
+    const ext = format === 'jpeg' ? 'jpg' : format;
+    const shardPath = getShardedPath(hash, ext, mlm.config.imageResizer.shardDepth, mlm.config.imageResizer.shardSegmentLength);
+    return `${mlm.config.imageResizer.route}/${shardPath}`;
+  }
+
+  async function refreshImage(hash) {
+    const metadata = await getMetadata(hash);
+    if (!metadata) {
+      throw new Error('Image not found');
+    }
+    return processImage(hash, metadata);
+  }
+
+  return {
+    'config.imageResizer': {
+      is: {
+        route: 'string',
+        storage: 'string',
+        timeout: 'positiveInteger|none',
+        maxFileSize: 'positiveInteger|none',
+        maxHashLength: 'positiveInteger',
+        shardDepth: 'positiveInteger',
+        shardSegmentLength: 'positiveInteger',
+      },
+      default: {
+        route: '/_images',
+        storage: '.cache/images',
+        timeout: 10000,
+        maxFileSize: 10 * 1024 * 1024, // 10MB
+        maxHashLength: 16,
+        shardDepth: 2,
+        shardSegmentLength: 2,
+      }
+    },
+
+    'services.imageResizer': () => ({
+      resize: (url, options = {}) => resizeImage(url, options),
+      getInfo: (hash) => getMetadata(hash),
+      refresh: (hash) => refreshImage(hash),
+    }),
+
+    'middleware.imageResizerServe': async () => {
+      const { default: express } = mlm.packages.express;
+      const router = express.Router();
+
+      // Static middleware runs FIRST, mounted at the configured route
+      router.use(mlm.config.imageResizer.route, express.static(path.resolve(mlm.config.imageResizer.storage), {
+        maxAge: '1y',
+        immutable: true,
+        setHeaders: res => res.logGroup = 'static',
+      }));
+
+      return (req,res,next) => {
+        if (req.bustCache) return next();
+        router(req, res, next);
+      }
+    },
+
+    'middleware.imageResizerProcess': async () => {
+      const router = mlm.packages.express.Router();
+
+      // This runs SECOND, only if static didn't find the file
+      router.get(`${mlm.config.imageResizer.route}/*path`, async (req, res, next) => {
+        res.logGroup = 'image-resizer';
+        try {
+          const fullPath = req.params.path.join('/');
+
+          // Extract hash from path (remove extension and any directory segments)
+          const filename = path.basename(fullPath);
+          const hash = filename.replace(/\.[^.]+$/, '');
+
+          const metadata = await getMetadata(hash);
+          if (!metadata) {
+            return res.status(404).send('Image not found');
+          }
+
+
+          // Process and serve the image
+          try {
+            const imageBuffer = await processImage(hash, metadata);
+
+            // Set content type
+            const mimeTypes = {
+              'jpeg': 'image/jpeg',
+              'png': 'image/png',
+              'webp': 'image/webp',
+              'avif': 'image/avif',
+            };
+
+            res.set('Content-Type', mimeTypes[metadata.format] || 'image/jpeg');
+            res.set('Cache-Control', 'public, max-age=31536000, immutable');
+            res.send(imageBuffer);
+          } catch (err) {
+            mlm.log('Image processing failed:', err.message);
+            return res.status(502).send('Failed to process image');
+          }
+        } catch (err) {
+          next(err);
+        }
+      });
+
+      return router;
+    },
+
+    async onStart() {
+      // Ensure storage directory exists
+      await fs.mkdir(path.resolve(mlm.config.imageResizer.storage), { recursive: true });
+
+      // Import dependencies
+    }
+  };
+};

package/units/logger.js

@@ -1,15 +1,66 @@
 export const info = {
   name: 'logger',
-  version: '1.0.0',
-  description: 'Minimal logger',
+  version: '1.0.1',
+  description: 'Noxt logger',
   requires: ['express'],
 }
 
 export default mlm => ({
-  'middleware.logger': async (app) => {
-     return (req, res, next) => {
-        mlm.log(req.method, req.url);
-        next();
-     }
+  'config.logger': {
+    is: {
+      level: 'string',
+      format: 'string',
+      exclude: { $any: [['string'], 'string', 'null'] },
+    },
+    default: {
+      level: 'info',
+      format: 'combined',
+      exclude: null
+    },
+
+  },
+  'middleware.logger': async () => {
+    let { level, format, exclude } = mlm.config.logger;
+    exclude = exclude
+      ? [].concat(exclude)
+      : null;
+
+    return (req, res, next) => {
+      const start = process.hrtime.bigint();
+      const time = new Date();
+      const ip = req.headers['x-forwarded-for']?.split(',')[0] || req.socket.remoteAddress;
+      const ua = req.headers['user-agent'] || '-';
+
+      const _send = res.send.bind(res);
+      res.send = (body) => {
+        try {
+          res._realBodySize = Buffer.byteLength(body);
+        } catch (e) {
+          res._realBodySize = 0;
+        };
+        _send(body);
+      }
+
+      res.on('finish', () => {
+
+        const used = process.memoryUsage();
+        const memory = used.heapUsed / 1024 / 1024;
+        const dur = Number(process.hrtime.bigint() - start) / 1e6; // ms
+        if (exclude && exclude.includes(res.logGroup)) return;
+        const parts = [
+          time.toISOString().slice(0, 19).replace('T', ' '),
+          memory.toFixed(0).padStart(3, ' ') + 'M',
+          ip,
+          dur.toFixed(0).padStart(4, ' ') + 'ms',
+          (res._realBodySize / 1024).toFixed(0).padStart(4, ' ') + 'K',
+          res.statusCode,
+          req.method + ' ' +
+          (req.originalUrl || req.url),
+          //ua
+        ]
+        mlm.log(parts.join(' | '));
+      });
+      next();
+    }
   },
-})
+});

package/units/noxt-dev.js

@@ -4,10 +4,12 @@ export const info = {
   requires: [
     'express',
     'logger',
-    'static',
-    'noxt-router-dev',
+    'bundler',
     'reload',
+    'compression',
+    'static',
     'fetch-cache-fs',
+    'noxt-router',
     'fetch',
   ],
 }