miolo 3.0.0-beta.21 → 3.0.0-beta.210

package/src/middleware/context/cache/options.mjs

@@ -1,17 +1,18 @@
-
 function _miolo_cacher_options_merge(def, opt, logger) {
-
-  const redis_host = opt?.redis?.host || def?.redis?.host || process.env?.REDIS_HOST || '127.0.0.1'
+  const redis_host = opt?.redis?.host || def?.redis?.host || process.env?.REDIS_HOST || "127.0.0.1"
   const redis_port = opt?.redis?.port || def?.redis?.port || process.env?.REDIS_PORT || 6379
 
-  const redis_username = opt?.redis?.username || def?.redis?.username || process.env?.REDIS_USERNAME || ''
-  const redis_password = opt?.redis?.password || def?.redis?.password || process.env?.REDIS_PASSWORD || ''
-  let redis_credentials = ''
+  const redis_username =
+    opt?.redis?.username || def?.redis?.username || process.env?.REDIS_USERNAME || ""
+  const redis_password =
+    opt?.redis?.password || def?.redis?.password || process.env?.REDIS_PASSWORD || ""
+  let redis_credentials = ""
   if (redis_username) {
-    redis_credentials= `${redis_username}${redis_password ? `:${redis_password}`: ''}@`
+    redis_credentials = `${redis_username}${redis_password ? `:${redis_password}` : ""}@`
   }
 
-  const redis_url = opt?.redis?.url || def?.redis?.url || `redis://${redis_credentials}${redis_host}:${redis_port}`
+  const redis_url =
+    opt?.redis?.url || def?.redis?.url || `redis://${redis_credentials}${redis_host}:${redis_port}`
 
   return {
     type: opt?.type || def?.type,
@@ -20,47 +21,44 @@ function _miolo_cacher_options_merge(def, opt, logger) {
     },
     namespace: opt?.namespace || def?.namespace,
     version: opt?.version || def?.version,
-    clean: opt?.clean===true || def?.clean===true,
+    clean: opt?.clean === true || def?.clean === true,
     ttl: opt?.ttl || def?.ttl,
     log: logger
   }
 }
 
+export function miolo_cacher_options_for_fly(config, options, logger) {
+  const d = config.cache.default
+
+  return _miolo_cacher_options_merge(d, options, logger)
+}
 
 export function miolo_cacher_options_for_calustra(config, logger) {
-
   const d = config.cache.default
   const c = config.cache.calustra
 
   return _miolo_cacher_options_merge(d, c, logger)
 }
 
-
 export function miolo_cacher_options_for_session(config, logger) {
-
   const d = config.cache.default
   const s = config.cache.session
 
   return _miolo_cacher_options_merge(d, s, logger)
 }
 
-
 export function miolo_cacher_options_for_custom(config, logger) {
-
   const d = config.cache.default
   const cs = config.cache.custom || {}
 
-  let custom = {}
+  const custom = {}
 
-  for(const [name, c] of Object.entries(cs)) {
-  
-    let opt = _miolo_cacher_options_merge(d, c, logger)
+  for (const [name, c] of Object.entries(cs)) {
+    const opt = _miolo_cacher_options_merge(d, c, logger)
     opt.namespace = opt.namespace || `miolo-${name}`
 
-   custom[name] = opt
+    custom[name] = opt
   }
 
   return custom
 }
-
-

package/src/middleware/context/db.mjs

@@ -1,20 +1,32 @@
-import { getConnection, dropConnections } from 'calustra/conn-postgres'
-import {  miolo_cacher_options_for_calustra } from './cache/options.mjs'
-
-export function init_context_db (config, logger) {
+import {
+  getConnection as miolo_db_connection_pg,
+  dropConnections as miolo_db_drop_connections_pg
+} from "calustra/conn-postgres"
+// import { getConnection as miolo_db_connection_sqlite, dropConnections as miolo_db_drop_connections_sqlite } from 'calustra/conn-sqlite'
+import { miolo_cacher_options_for_calustra } from "./cache/options.mjs"
 
+export function init_context_db(config, logger) {
   const get_connection_wrap = async (options) => {
-
-    const dbOptions= {
+    const dbOptions = {
       ...config.db.options,
-      ...options || {},
-      
+      ...(options || {}),
+
       log: logger,
       cache: miolo_cacher_options_for_calustra(config, logger)
     }
-    
 
-    let conn = await getConnection(config.db.config, dbOptions)
+    //    let conn = undefined
+    //    if (config.db.config.dialect === 'sqlite') {
+    //      conn = await miolo_db_connection_sqlite(config.db.config, dbOptions)
+    //    } else {
+    //      conn = await miolo_db_connection_pg(config.db.config, dbOptions)
+    //    }
+
+    const conn = await miolo_db_connection_pg(config.db.config, dbOptions)
+    if (config.db.config.dialect === "sqlite") {
+      logger.error(`[miolo] SQLite is not supported yet`)
+    }
+
     conn.get_model = conn.getModel
 
     // Maybe we need to force some conn is retrieved?
@@ -29,30 +41,43 @@ export function init_context_db (config, logger) {
   }
 
   const get_model_wrap = async (name, options) => {
-
-    const dbOptions= {
+    const dbOptions = {
       ...config.db.options,
-      ...options || {},
+      ...(options || {}),
       log: logger
     }
 
-    const conn = await getConnection(config.db.config, dbOptions)
+    //    let conn = undefined
+    //    if (config.db.config.dialect === 'sqlite') {
+    //      conn = await miolo_db_connection_sqlite(config.db.config, dbOptions)
+    //    } else {
+    //      conn = await miolo_db_connection_pg(config.db.config, dbOptions)
+    //    }
+
+    const conn = await miolo_db_connection_pg(config.db.config, dbOptions)
+    if (config.db.config.dialect === "sqlite") {
+      logger.error(`[miolo] SQLite is not supported yet`)
+    }
+
     return conn.get_model(name)
-  }  
+  }
 
-  const db= {
+  const db = {
     init_connection: async () => {
-      const conn = await get_connection_wrap({reset: true})
+      const conn = await get_connection_wrap({ reset: true })
       return conn
     },
     fly_connection: async () => {
-      const conn = await get_connection_wrap({cache: false})
+      const conn = await get_connection_wrap({ cache: false })
       return conn
     },
     get_connection: get_connection_wrap,
     get_model: get_model_wrap,
-    drop_connections: dropConnections
+    drop_connections: miolo_db_drop_connections_pg
+    // drop_connections: (config.db.config.dialect === 'sqlite')
+    //   ? miolo_db_drop_connections_sqlite
+    //   : miolo_db_drop_connections_pg
   }
 
   return db
-}
+}

package/src/middleware/context/index.mjs

@@ -1,35 +1,35 @@
-import { init_emailer_transporter } from '../../engines/emailer/index.mjs'
-import { init_logger } from '../../engines/logger/index.mjs'
-import { init_parser } from '../../engines/parser/index.mjs'
-import {init_context_db} from './db.mjs'
-import {init_context_cache} from './cache/index.mjs'
+import { init_emailer_transporter } from "../../engines/emailer/index.mjs"
+import { init_logger } from "../../engines/logger/index.mjs"
+import { init_parser } from "../../engines/parser/index.mjs"
+import { init_context_cache } from "./cache/index.mjs"
+import { init_context_db } from "./db.mjs"
 
-const init_context_middleware = ( app, config ) => {
+const init_context_middleware = (app, config) => {
   const emailer = init_emailer_transporter(config.mail)
   const logger = init_logger(config.log, emailer, config?.name)
   const parser = init_parser()
 
-  const db= init_context_db(config, logger)
+  const db = init_context_db(config, logger)
   const cache = init_context_cache(config, logger)
 
   // Assign miolo stuff to ctx
-  const mioloContext= {
-    config: {...config},
+  const mioloContext = {
+    config: { ...config },
     emailer,
     logger,
     parser,
     db,
     cache
-  } 
+  }
 
   async function context_middleware(ctx, next) {
     // Assign miolo stuff to ctx
-    ctx.miolo= mioloContext 
+    ctx.miolo = mioloContext
     await next()
   }
 
   app.use(context_middleware)
-  app.context.miolo= mioloContext
+  app.context.miolo = mioloContext
 }
 
 export { init_context_middleware }

package/src/middleware/extra.mjs

@@ -1,12 +1,11 @@
-
-const init_extra_middlewares = ( app, middlewares  ) => {
-  if (middlewares==undefined || middlewares.length==0) {
+const init_extra_middlewares = (app, middlewares) => {
+  if (middlewares === undefined || middlewares.length === 0) {
     return
   }
 
-  middlewares.map(midw => { 
+  middlewares.forEach((midw) => {
     app.use(midw)
   })
 }
 
-export { init_extra_middlewares }
+export { init_extra_middlewares }

package/src/middleware/http/body.mjs

@@ -1,31 +1,31 @@
-import { bodyParser as koa_body_parser } from '@koa/bodyparser'
-import koa_compress from 'koa-compress'
-import {constants} from 'node:zlib'
+import { constants } from "node:zlib"
+import { bodyParser as koa_body_parser } from "@koa/bodyparser"
+import koa_compress from "koa-compress"
 
-const init_body_middleware = ( app ) => {
+const init_body_middleware = (app) => {
   // Compress
-  
-  app.use(koa_compress({
-    filter: function (content_type) {
-      return content_type=='application/json' || content_type=='text/html'
-    },
-    //threshold: 2048,
-    gzip: {
-      flush: constants.Z_SYNC_FLUSH
-    },
-    deflate: {
-      flush: constants.Z_SYNC_FLUSH,
-    },
-    br: false // disable brotli
 
-  }))
-
-  app.use(koa_body_parser({
-    formLimit: '100mb',
-    jsonLimit: '100mb',
-    bufferLimit: '100mb',
-  }))
+  app.use(
+    koa_compress({
+      filter: (content_type) => content_type === "application/json" || content_type === "text/html",
+      //threshold: 2048,
+      gzip: {
+        flush: constants.Z_SYNC_FLUSH
+      },
+      deflate: {
+        flush: constants.Z_SYNC_FLUSH
+      },
+      br: false // disable brotli
+    })
+  )
 
+  app.use(
+    koa_body_parser({
+      formLimit: "100mb",
+      jsonLimit: "100mb",
+      bufferLimit: "100mb"
+    })
+  )
 }
 
-export { init_body_middleware }
+export { init_body_middleware }

package/src/middleware/http/catcher.mjs

@@ -3,23 +3,95 @@
  * @param ctx
  */
 
-import http from 'http'
-const _ONLY_WARN= [401, 403]
+import util from "node:util"
+// import http from 'http'
+import statuses from "statuses"
+
+const _ONLY_WARN = [401, 403]
 
 function init_catcher_middleware(app) {
-  const logger= app.context.miolo.logger
-  
+  const logger = app.context.miolo.logger
+
   async function catcher_middleware(err) {
+    // don't do anything if there is no error.
+    // this allows you to pass `this.onerror`
+    // to node-style callbacks.
+    if (err == null) return
+
+    // Ignore this error
+    // Appearing since Koa 3.1 + koa-static 6.x + koa-send 7.x
+    if (err.code === "ERR_STREAM_PREMATURE_CLOSE") return
+
+    // When dealing with cross-globals a normal `instanceof` check doesn't work properly.
+    // See https://github.com/koajs/koa/issues/1466
+    // We can probably remove it once jest fixes https://github.com/facebook/jest/issues/2549.
+    const isNativeError =
+      Object.prototype.toString.call(err) === "[object Error]" || err instanceof Error
+    if (!isNativeError) err = new Error(util.format("non-error thrown: %j", err))
+
+    let headerSent = false
+    if (this.headerSent || !this.writable) {
+      headerSent = err.headerSent = true
+    }
+
+    // delegate
+    this.app.emit("error", err, this)
+
+    let statusCode = err.status || err.statusCode
+    // default to 500
+    if (typeof statusCode !== "number" || !statuses.message[statusCode]) statusCode = 500
+
+    // Log the error depending on the status
+    const ip = this.headers["x-real-ip"] || this.headers["x-orig-ip"] || this.ip || "127.0.0.1"
+    if (_ONLY_WARN.indexOf(statusCode) >= 0 || err.message.indexOf("Premature close") >= 0) {
+      logger.warn(`[${ip}] ${this.method} ${this.url} - ${statusCode}: ${err.message}`)
+    } else {
+      logger.error(`[${ip}] ${this.method} ${this.url} - ${statusCode}: ${err.message}`)
+    }
+
+    // nothing we can do here other
+    // than delegate to the app-level
+    // handler and log.
+    if (headerSent) {
+      return
+    }
+
+    const { res } = this
+
+    // first unset all headers
+    /* istanbul ignore else */
+    if (typeof res.getHeaderNames === "function") {
+      res.getHeaderNames().forEach((name) => {
+        res.removeHeader(name)
+      })
+    } else {
+      res._headers = {} // Node < 7.7
+    }
+
+    // then set those specified
+    this.set(err.headers)
+
+    // force text/plain
+    this.type = "text"
+
+    // respond
+    const code = statuses.message[statusCode]
+    const msg = err.expose ? err.message : code
+    this.status = err.status = statusCode
+    this.length = Buffer.byteLength(msg)
+    res.end(msg)
+
+    /*
     if (!err) return
 
     // wrap non-error object
-    if (!(err instanceof Error)) {
+    const isNativeError = (Object.prototype.toString.call(err) === '[object Error]') || (err instanceof Error)
+    if (!isNativeError) {
       let errMsg = err;
       if (typeof err === 'object') {
         try {
           errMsg = JSON.stringify(err);
-          // eslint-disable-next-line no-empty
-        } catch (e) {}
+        } catch (_) {}
       }
       const newError = new Error('non-error thrown: ' + errMsg);
       // err maybe an object, try to copy the name, message and stack to the new error instance
@@ -73,9 +145,10 @@ function init_catcher_middleware(app) {
     this.body = JSON.stringify(this.body || '', null, 2)
     this.length = Buffer.byteLength(this.body)
     this.res.end(this.body)
+    */
   }
 
   app.context.onerror = catcher_middleware
 }
 
-export {init_catcher_middleware}
+export { init_catcher_middleware }

package/src/middleware/http/custom_blacklist.mjs

@@ -1,17 +1,20 @@
-export const CUSTOM_BLACKLIST_IPS= [
-  '52.212.247.108', // Amazon Data Services Ireland
-  '54.218.32.58',   // Amazon.com, Inc. AMAZO-ZPDX4
-  '170.106.82.193', // Asia Pacific Network Information Centre (APNIC)
-  '110.166.71.39',  // Asia Pacific Network Information Centre (APNIC)
-  '39.107.54.8',    // Asia Pacific Network Information Centre (APNIC)
-  '205.210.31.240 ',// Palo Alto Networks, Inc (PAN-22)
-  '134.122.52.203', // DigitalOcean, LLC (DO-13)
-  '165.232.105.25', // DigitalOcean, LLC (DO-13)
-  '66.249.73.200',  // Google
-  '35.206.153.204', // Google
-  '199.244.88.227', // Sundance International LLC (SIL-58)
-  '198.12.222.107', // GoDaddy.com, LLC (GODAD)
-  '212.128.118.10', // RIPE Network Coordination Centre (RIPE)
-  '103.218.240.46', // Asia Pacific Network Information Centre (APNIC)
-  '45.15.167.150'   // ??
+export const CUSTOM_BLACKLIST_IPS = [
+  "52.212.247.108", // Amazon Data Services Ireland
+  "54.218.32.58", // Amazon.com, Inc. AMAZO-ZPDX4
+  "170.106.82.193", // Asia Pacific Network Information Centre (APNIC)
+  "110.166.71.39", // Asia Pacific Network Information Centre (APNIC)
+  "39.107.54.8", // Asia Pacific Network Information Centre (APNIC)
+  "205.210.31.240 ", // Palo Alto Networks, Inc (PAN-22)
+  "134.122.52.203", // DigitalOcean, LLC (DO-13)
+  "165.232.105.25", // DigitalOcean, LLC (DO-13)
+  "66.249.73.200", // Google
+  "35.206.153.204", // Google
+  "199.244.88.227", // Sundance International LLC (SIL-58)
+  "198.12.222.107", // GoDaddy.com, LLC (GODAD)
+  "212.128.118.10", // RIPE Network Coordination Centre (RIPE)
+  "103.218.240.46", // Asia Pacific Network Information Centre (APNIC)
+  "45.15.167.150", // ??
+  "165.227.173.41", // Frankfurt am Main, DE
+  "147.135.220.53", //
+  "47.237.23.78" // SG
 ]

package/src/middleware/http/headers.mjs

@@ -1,73 +1,76 @@
-import koa_cors from '@koa/cors'
-import koa_proxy from 'koa-proxies'
+import koa_cors from "@koa/cors"
+import koa_proxy from "koa-proxies"
 
 const _if_options = (http, field, callback) => {
   try {
-    const val= http[field]
-    if ((val!=undefined) && (val!=false)) {
+    const val = http[field]
+    if (val !== undefined && val !== false) {
       return callback(val)
     }
-  } catch(_) {}
+  } catch (_) {}
 }
 
 const _cors_options = (options) => {
-  if (typeof options == 'object') {
+  if (typeof options === "object") {
     return options
   }
   return {}
 }
 
-
 const _proxy_options = (options) => {
-  if (options==undefined) {
-    options= {}
+  if (options === undefined) {
+    options = {}
   }
 
-  const tpath= options?.path || '/'
+  const tpath = options?.path || "/"
 
   const target = options?.target || "https://proxy.miolo.com"
-  const changeOrigin = options?.changeOrigin!=undefined ? options.changeOrigin : true
-  const logs = options?.logs!=undefined ? options.logs : true
-
+  const changeOrigin = options?.changeOrigin !== undefined ? options.changeOrigin : true
+  const logs = options?.logs !== undefined ? options.logs : true
 
-  const toptions= {
-    target, changeOrigin, logs
+  const toptions = {
+    target,
+    changeOrigin,
+    logs
   }
 
   return [tpath, toptions]
 }
 
+const init_headers_middleware = (app, http) => {
+  const logger = app.context.miolo.logger
 
-
-
-const init_headers_middleware = ( app, http) => {
-  const logger= app.context.miolo.logger
-  
-  _if_options(http, 'cors', (options) => {
-    if (options=='simple') {
+  _if_options(http, "cors", (options) => {
+    if (options === "simple") {
       logger.debug(`[http] Setting CORS the simple way`)
       app.use(async (ctx, next) => {
-        ctx.set('Access-Control-Allow-Origin', '*')
-        ctx.set('Access-Control-Expose-Headers', 'SourceMap,X-SourceMap')
+        ctx.set("Access-Control-Allow-Origin", "*")
+        ctx.set("Access-Control-Expose-Headers", "SourceMap,X-SourceMap")
         await next()
       })
     } else {
-      const coptions= _cors_options(options)
+      const coptions = _cors_options(options)
       logger.debug(`[http] Setting CORS headers for ${JSON.stringify(coptions)}`)
       app.use(koa_cors(coptions))
     }
-
-
   })
 
-  _if_options(http, 'proxy', (options) => {
-    const [tpath, toptions]= _proxy_options(options)
-    logger.debug(`[http] Setting Proxy for ${tpath} to ${toptions.target} `)
+  _if_options(http, "proxy", (options) => {
+    // Ensure NGINX is setting headers as:
+    // proxy_set_header Host $host;
+    // proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    // proxy_set_header X-Forwarded-Proto $scheme;
+
+    if (options === true) {
+      app.proxy = true
+      logger.debug(`[http] Setting Proxy to true`)
+    } else {
+      const [tpath, toptions] = _proxy_options(options)
+      logger.debug(`[http] Setting Proxy for ${tpath} to ${toptions.target} `)
 
-    app.use(
-      koa_proxy(tpath, toptions)
-    );
-  })  
+      app.use(koa_proxy(tpath, toptions))
+    }
+  })
 }
 
 export { init_headers_middleware }

package/src/middleware/http/ratelimit.mjs

@@ -1,13 +1,10 @@
-import ratelimit from 'koa-ratelimit'
-import { yellow, magenta } from 'tinguir'
-import { ipsum_read_ips } from '../../engines/cron/ipsum.mjs'
-import { CUSTOM_BLACKLIST_IPS } from './custom_blacklist.mjs'
-
-
+import ratelimit from "koa-ratelimit"
+import { magenta, yellow } from "tinguir"
+import { ipsum_read_ips } from "../../engines/cron/ipsum.mjs"
+import { CUSTOM_BLACKLIST_IPS } from "./custom_blacklist.mjs"
 
 function init_rate_limit_middleware(app, config) {
-  /* eslint-disable no-unused-vars */
-  const miolo= app.context.miolo
+  const miolo = app.context.miolo
 
   //const _get_ip = (ctx) => ctx.headers["x-real-ip"] || ctx.headers["x-orig-ip"] || ctx.ip || '127.0.0.1'
 
@@ -16,23 +13,19 @@ function init_rate_limit_middleware(app, config) {
   const _def_whitelist = (ctx) => {
     const ips = config?.whitelist_ips || []
     if (ips) {
-      return ips.indexOf(ctx.request.ip)>=0
+      return ips.indexOf(ctx.request.ip) >= 0
     }
     return false
   }
 
   const _def_blacklist = (ctx) => {
     const ip = ctx.request.ip
-    const ips = [
-      ... ipsum_ips,
-      ... CUSTOM_BLACKLIST_IPS,
-      ... config?.blacklist_ips || []
-    ]
-    
-    const doit = ips.indexOf(ip)>=0
+    const ips = [...ipsum_ips, ...CUSTOM_BLACKLIST_IPS, ...(config?.blacklist_ips || [])]
+
+    const doit = ips.indexOf(ip) >= 0
 
     if (doit) {
-      ctx.miolo.logger.debug(`Rejecting ${yellow('blacklisted')} ${magenta(ip)}`)
+      ctx.miolo.logger.debug(`Rejecting ${yellow("blacklisted")} ${magenta(ip)}`)
     }
 
     return doit
@@ -41,20 +34,20 @@ function init_rate_limit_middleware(app, config) {
   const rateLimitDB = new Map()
 
   const rateLimitConfig = {
-    driver: 'memory',
+    driver: "memory",
     db: rateLimitDB,
 
     id: (ctx) => ctx.request.ip,
     headers: {
-      remaining: 'Rate-Limit-Remaining',
-      reset: 'Rate-Limit-Reset',
-      total: 'Rate-Limit-Total'
+      remaining: "Rate-Limit-Remaining",
+      reset: "Rate-Limit-Reset",
+      total: "Rate-Limit-Total"
     },
     disableHeader: false,
 
     max: config?.max || 1000,
     duration: config?.duration || 60 * 1000,
-    errorMessage: config?.errorMessage || 'Rate Limit reached',
+    errorMessage: config?.errorMessage || "Rate Limit reached",
 
     whitelist: config?.whitelist || _def_whitelist,
     blacklist: config?.blacklist || _def_blacklist
@@ -63,4 +56,4 @@ function init_rate_limit_middleware(app, config) {
   app.use(ratelimit(rateLimitConfig))
 }
 
-export {init_rate_limit_middleware}
+export { init_rate_limit_middleware }