miolo 2.0.0-beta.1 → 2.0.0-beta.3

package/src/server/engines/cron/index.mjs

@@ -0,0 +1,153 @@
+import {
+  init_cron_job
+} from './init.mjs'
+import {
+  sys_check_config
+} from './syscheck.mjs'
+import {
+  ipsum_config
+} from './ipsum.mjs'
+import { cyan, green_bold, yellow_bold } from 'tinguir'
+import { sys_email_queue_config } from './emails.mjs'
+
+
+export function init_cron(app, custom) {
+  const miolo = app.context.miolo
+  const logger= miolo.logger
+  
+  const jobConfigs= [
+    sys_check_config(),
+    ipsum_config(),
+    sys_email_queue_config(),
+    ...custom || []
+  ]
+  
+  // Keep trace of conr jobs to be accessible later
+  const jobInfos= [
+    // {name: 'name', job: <job>, isActive: true/false} 
+  ]
+  
+  jobConfigs.map(config => {
+    const name = config.name
+    const job= init_cron_job(miolo, config)
+    jobInfos.push({
+      name, 
+      job,
+      isActive: false
+    })  
+  })
+  
+  const _find_job_by_idx_or_name = (idxOrName) => {
+    let jobInfo
+    if (typeof idxOrName == 'number') {
+      jobInfo= jobInfos[idxOrName]
+    } else {
+      jobInfo= jobInfos.filter(j => j.name == idxOrName)[0]
+    }
+
+    if (!jobInfo) {
+      logger.error(`[cron] Job ${cyan(idxOrName)} Not Found`)
+    }
+
+    return jobInfo
+  }
+
+  const _start_job = (jobInfo) => {
+    try {
+      jobInfo.job.start()
+      jobInfo.isActive= true
+      logger.debug(`[cron][Job ${cyan(jobInfo.name)}] ${green_bold('started!')}`)
+      return 1
+    } catch(e) {
+      logger.error(`[cron][Job ${cyan(jobInfo.name)}] Error starting it`)
+      logger.error(e)
+      return 0
+    }
+  }  
+
+  const _start_job_by_idx_or_name = (idxOrName) => {
+    const jobInfo = _find_job_by_idx_or_name(idxOrName)
+    if (jobInfo) {
+      const done = _start_job(jobInfo)
+      return [done, jobInfo.name]
+    }
+    return [0, '']
+  }
+
+  const _start_all_jobs = () => {
+    try {
+      let started= [], errors= []
+      jobInfos.map(jobInfo => {
+        const done= _start_job(jobInfo)
+        if (done == 1) {
+          started.push(jobInfo.name)
+        } else {
+          errors.push(jobInfo.name)
+        }
+      })
+      if (started.length > 0) {
+        logger.info(`[cron] Started ${started.length} jobs: ${started}`)
+      }
+      if (errors.length > 0) {
+        logger.warn(`[cron] Could not start ${errors.length} jobs: ${errors}`)
+      }
+    } catch(error) {
+      logger.error(`[cron] start() error: ${error}`)
+    }
+  
+  }
+
+  const _stop_job = (jobInfo) => {
+    try {
+      jobInfo.job.stop()
+      jobInfo.isActive= false
+      logger.debug(`[cron][Job ${cyan(jobInfo.name)}] ${yellow_bold('stopped!')}`) 
+      return 1
+    } catch(e) {
+      logger.error(`[cron][Job ${cyan(jobInfo.name)}] Error stopping it`)
+      logger.error(e)
+      return 0
+    }
+  }
+
+  const _stop_job_by_idx_or_name = (idxOrName) => {
+    const jobInfo = _find_job_by_idx_or_name(idxOrName)
+    if (jobInfo) {
+      const done = _stop_job(jobInfo)
+      return [done, jobInfo.name]
+    }
+    return [0, '']
+  }
+
+  const _stop_all_jobs = () => {
+    try {
+      let stopped= [], errors= []
+      jobInfos.map(jobInfo => {
+        const done= _stop_job(jobInfo)
+        if (done == 1) {
+          stopped.push(jobInfo.name)
+        } else {
+          errors.push(jobInfo.name)
+        }
+      })
+      if (stopped.length > 0) {
+        logger.info(`[cron] Stopped ${stopped.length} jobs: ${stopped}`)
+      }
+      if (errors.length > 0) {
+        logger.warn(`[cron] Could not stop ${errors.length} jobs: ${errors}`)
+      }
+    } catch(error) {
+      logger.error(`[cron] stop() error: ${error}`)
+    }
+  }
+
+  app.cron= {
+    jobs:      jobInfos,
+    start_one: _start_job_by_idx_or_name,
+    start:     _start_all_jobs,
+    stop_one : _stop_job_by_idx_or_name,
+    stop:      _stop_all_jobs
+  }
+
+  return app
+}

package/src/server/engines/cron/init.mjs

@@ -0,0 +1,53 @@
+
+import {CronJob} from 'cron'
+import { cyan, green_bold } from 'tinguir'
+
+export function init_cron_job(miolo, config) {
+  const logger= miolo.logger
+  const onTickName = config?.onTick?.name
+    ? config.onTick.name != 'onTick'
+      ? config.onTick.name
+      : 'custom'
+      : 'custom'
+
+  const name = config?.name || onTickName
+
+  const job= new CronJob(
+    // cronTime
+    config?.cronTime || '*/5 * * * *',
+    
+    // onTick(miolo, onComplete)
+    (onComplete) => {
+      try {
+        logger.silly(`[cron][Custom Job ${cyan(name)}] ${green_bold('ticks!')}`)
+        config.onTick(miolo, onComplete)
+      } catch(e) {
+        logger.error(`[cron][Custom Job ${cyan(name)}] Error at onTick()`)
+        logger.error(e)
+      }
+    },
+
+    // onComplete(miolo)
+    () => {
+      logger.silly(`[cron][Custom Job ${cyan(name)}] ${green_bold('completed!')}`)
+      if (config?.onComplete) {
+        try {
+          config.onComplete(miolo)
+        } catch(e) {
+          logger.error(`[cron][Custom Job ${cyan(name)}] Error at onComplete()`)
+          logger.error(e)
+        }
+      }
+    },
+
+    // Do not start yet
+    false,
+
+    config?.timezone || 'Europe/Madrid',
+    // config?.context || null,
+    // config?.runOnInit || null,
+    // config?.utcOffset || null,
+    // config?.unrefTimeout || null,
+  )    
+  return job
+}

package/src/server/engines/cron/ipsum.mjs

@@ -0,0 +1,151 @@
+import path from 'path'
+import fs from 'fs'
+import https from 'https'
+//import fetch from 'node-fetch'
+import { cyan, green, yellow } from 'tinguir'
+
+const _IPSUM_DEF_FOLDER = '/var/ipsum'
+const _IPSUM_FILE_NAME = 'ipsum.txt'
+const _IPSUM_REMOTE_FILE = 'https://raw.githubusercontent.com/stamparm/ipsum/master/ipsum.txt'
+const _IPSUM_NLISTS = 1
+
+/*function ipsum_download_file(callback, remote_file = _IPSUM_REMOTE_FILE) {
+  fetch(remote_file).then(response => {
+    response.text().then((content) => {
+      callback(content)
+    })
+  })
+}*/
+
+
+export function ipsum_download_file(callback, remote_file = _IPSUM_REMOTE_FILE, logger) {
+  const lerr = logger ? logger.error : console.error
+
+  https.get(remote_file, (res) => {
+    const data = []
+    res.on('data', (chunk) => {
+      data.push(chunk)
+    }).on('end', () => {
+      let content = Buffer.concat(data).toString()
+      callback(content)
+    })
+  }).on('error', (err) => {
+    lerr(`[cron][${cyan('IPsum')}] Error downloading remote file (${err.toString()})`)
+    callback('')
+  })
+
+}
+
+function _ipsum_ips_from_content(content, logger) {
+  const lerr = logger ? logger.error : console.error
+
+  if (! content) {
+    return []
+  }
+
+  try {
+    let ips = []
+    content
+      .split('\n')
+      .filter(line => line.indexOf('#') < 0)
+      .map(line => {
+        const [ip, nlists] = line.split('\t')
+        if (parseInt(nlists) >= _IPSUM_NLISTS) {
+          ips.push(ip)
+        }
+      })
+    return ips
+  } catch(error) {
+    lerr(`[cron][${cyan('IPsum')}] Error getting IPs from content`)
+    return []
+  }
+}
+
+function _ipsum_ips_from_file(folder= _IPSUM_DEF_FOLDER, logger) {
+  const lerr = logger ? logger.error : console.error
+
+  if (! fs.existsSync(folder)) {
+    if (logger) {
+      lerr(`[cron][${cyan('IPsum')}] Folder ${folder} does not exist`)
+    }
+    return []
+  }
+
+  const local_path = path.join(folder, _IPSUM_FILE_NAME)
+
+  if (! fs.existsSync(local_path)) {
+    if (logger) {
+      lerr(`[cron][${cyan('IPsum')}] File ${local_path} does not exist`)
+    }
+    return []
+  }
+
+  const content = fs.readFileSync(local_path, {encoding:'utf8'})  
+  return _ipsum_ips_from_content(content, logger)
+}
+
+
+function ipsum_update(folder= _IPSUM_DEF_FOLDER, callback, logger) {
+  const lerr = logger ? logger.error : console.error
+  const ldbg = logger ? logger.debug : console.log
+
+  if (! fs.existsSync(folder)) {
+    lerr(`[cron][${cyan('IPsum')}] Folder ${folder} does not exist`)
+    return
+  }
+
+  try {
+    ldbg(`[cron][${cyan('IPsum')}] Updating file...`)
+
+    ipsum_download_file((content) => {
+      
+      const local_path = path.join(folder, _IPSUM_FILE_NAME)
+      fs.writeFileSync(local_path, content, {encoding:'utf8', flag:'w'})
+
+      const ntot = content.split('\n').length
+      const ips = _ipsum_ips_from_content(content, logger)
+      const nfilt = ips.length
+      ldbg(`[cron][${cyan('IPsum')}] File downloaded. ${ntot} ips on the list (${nfilt} appearing in ${_IPSUM_NLISTS} or more lists)`)
+      
+      if (callback) {
+        callback(ips)
+      }
+    })
+  
+  } catch(error) {
+    lerr(`[cron][${cyan('IPsum')}] Error ${error} updating the file`)
+  }
+}
+
+
+export function ipsum_config() {
+  return {
+    name: 'IPsum',
+    cronTime: '0 0 * * *',
+    onTick: (miolo, _onCompleted) => {
+      const folder = miolo.config.http.ratelimit.ipsum_folder || _IPSUM_DEF_FOLDER
+      ipsum_update(folder, (ips) => {
+        miolo.logger.debug(`[cron][${cyan('IPsum')}] File downloaded. ${green(ips.length)} ips will be ${yellow('blacklisted')}!`)
+      }, miolo.logger)
+    },
+    start: true   
+  }
+}
+
+export function ipsum_read_ips(folder = _IPSUM_DEF_FOLDER, callback, logger) {
+  const ldbg = logger ? logger.debug : console.log
+  const lwarn = logger ? logger.warn : console.log
+
+  const ips = _ipsum_ips_from_file(folder, logger)
+  if (ips.length > 0) {
+    if (callback) {
+      callback(ips)
+    }
+    ldbg(`[cron][${cyan('IPsum')}] File contains ${ips.length} ips`)
+    return ips
+  } else {
+    lwarn(`[cron][${cyan('IPsum')}] File is empty. Launching update...`)
+    ipsum_update(folder, callback, logger)
+    return []
+  }
+}

package/src/server/engines/cron/syscheck.mjs

@@ -0,0 +1,56 @@
+import os from 'os'
+import diskspace from 'diskspace'
+import { cyan, green, yellow } from 'tinguir'
+
+function _toMB(bytes) {
+  if (!bytes) return 0
+  return bytes/1000000
+}
+
+
+function _toGB(kbytes) {
+  if (!kbytes) return 0
+  return kbytes/1000000
+}
+
+
+
+function _sys_check_and_log(logger) {
+  
+
+  const used= Math.round(_toMB(os.freemem()), 2)
+  const total= Math.round(_toMB(os.totalmem()), 2)
+  const perc= Math.round( (used*100)/total, 2)
+
+  if (perc>80) {
+    logger.error(`[cron][${cyan('SysCheck')}] RAM ${yellow(used)} MB used of ${green(total)} MB (${yellow(perc)} %)`)
+  } else {
+    logger.info(`[cron][${cyan('SysCheck')}] RAM ${yellow(used)} MB used of ${green(total)} MB (${yellow(perc)} %)`)
+  }
+  
+
+  diskspace.check('/', function (err, result)
+  {
+    const used = Math.round(_toGB(result.used), 2)
+    const total= Math.round(_toGB(result.total), 2)
+    const free = Math.round(_toGB(result.free), 2)
+
+    if (free<1) {
+      logger.error(`[cron][${cyan('SysCheck')}] DISK ${yellow(used)} GB used of ${green(total)} GB (${yellow(free)} GB free)`)
+    } else {
+      logger.info(`[cron][${cyan('SysCheck')}] DISK ${yellow(used)} GB used of ${green(total)} GB (${yellow(free)} GB free)`)
+    }
+  });  
+}
+
+
+export function sys_check_config() {
+  return {
+    name: 'SysCheck',
+    cronTime: '0,15,30,45 * * * *',
+    onTick: (miolo, _onCompleted) => {
+      _sys_check_and_log(miolo.logger)
+    },
+    start: true   
+  }
+}

package/src/server/engines/emailer/index.mjs

@@ -0,0 +1,2 @@
+
+export {init_emailer_transporter} from './transporter.mjs'

package/src/server/engines/emailer/queue.mjs

@@ -0,0 +1,54 @@
+import { v4 as uuidv4 } from 'uuid'
+
+export let EMAIL_QUEUE = {}
+
+export async function email_queue_an_email(email, logger= undefined) {
+  const _loge = logger?.info || console.error
+
+  try {
+    const id = uuidv4()
+    const emailData = { id, ...email, sent: false }
+    //await client.rPush('email_queue', JSON.stringify(emailData))
+    EMAIL_QUEUE[id] = emailData
+    return { ok: true, id }
+
+  } catch (error) {
+    _loge(`[emailer] Error al guardar en la cola: ${error}`)
+    return { ok: false, id: null, error }
+  }
+}
+
+export function email_queue_pop_pendings(logger= undefined) {
+
+  let grouped = {}
+
+  Object.values(EMAIL_QUEUE)
+    .filter(e => !e.sent)
+    .forEach(({ id: eid, from, to, subject }) => {
+      const key = `${from}_${to}_${subject}`
+      
+      if (!grouped[key]) {
+        grouped[key] = { to_subject_key: key, from, to, subject, count: 0, ids: [] }
+      }
+
+      grouped[key].count++
+      grouped[key].ids.push(eid)
+
+      // Mark as sent
+      try {
+        EMAIL_QUEUE[eid].sent= true
+      } catch(_) {}
+  })
+
+  
+  return Object.values(grouped)
+}
+
+export function email_queue_remove_ids(eids, logger= undefined) {
+  eids.forEach((eid) => {
+
+    delete EMAIL_QUEUE[eid]
+  })
+}
+
+

package/src/server/engines/emailer/transporter.mjs

@@ -0,0 +1,149 @@
+import nodemailer from 'nodemailer'
+import { email_queue_an_email, email_queue_pop_pendings, email_queue_remove_ids } from './queue.mjs'
+
+
+const _logi = (logger, msg) => logger?.info ? logger.info(msg) : console.info(msg)
+const _loge = (logger, msg) => logger?.error ? logger.info(msg) : console.error(msg)
+
+
+export function _init_emailer_transporter({options, defaults, silent}) {
+
+  const nmailer = nodemailer.createTransport(options, defaults)
+
+  function verify_emailer(logger= undefined) {
+    _logi(logger, '[emailer] Verifying...')
+    nmailer.verify(function(error, _success) {
+      if (error) {
+        _loge(logger, '[emailer] Verifying ERROR')
+        _loge(error)
+      } else {
+        _logi(logger, '[emailer] Verifyed OK: Server is ready to take our messages')
+      }
+    })
+  }
+
+  async function send_email(mail, logger= undefined) {
+    if (silent === true) {
+      _logi(logger, '[emailer] *********************************')
+      _logi(logger, '[emailer] This mail will not be send (emailing is disabled):')
+      _logi(mail)
+      _logi(logger, '[emailer] *********************************')
+
+      return {
+        ok: true,
+        silent: true,
+        error: undefined,
+        messageId: undefined
+      }
+    } else {
+      try {
+        let info = await nmailer.sendMail(mail)
+        info.ok = info?.messageId ? true : false
+
+        return info
+      } catch(error) {
+        _loge(logger, `[emailer] Error sending email: ${mail?.from || ''} => ${mail?.to || ''} (${mail?.subject || ''})`)
+
+        return {
+          ok: false,
+          silent: false,
+          error,
+          messageId: undefined
+        }
+      }
+    }
+  }
+
+  function queue_email(mail, logger= undefined) {
+    if (silent === true) {
+      _logi(logger, '[emailer] *********************************')
+      _logi(logger, '[emailer] This mail will not be send (emailing is disabled):')
+      _logi(mail)
+      _logi(logger, '[emailer] *********************************')
+
+      return {
+        ok: true,
+        silent: true,
+        error: undefined,
+        messageId: undefined
+      }
+    } else {
+      try {
+        const q = email_queue_an_email(mail, logger)
+        _logi(logger, `[emailer] Queued email: ${mail?.from || ''} => ${mail?.to || ''} (${mail?.subject || ''})`)
+
+        return {
+          ok: q.ok,
+          silent: false,
+          error: q.error,
+          messageId: q.id
+        }
+      } catch(error) {
+        _loge(logger, `[emailer] Error queueing email: ${mail?.from || ''} => ${mail?.to || ''} (${mail?.subject || ''})`)
+
+        return {
+          ok: false,
+          silent: false,
+          error,
+          messageId: undefined
+        }
+      }
+    }
+  }
+
+  async function queue_send_emails(logger= undefined) {
+    const pending = email_queue_pop_pendings(logger)
+    if (pending.length <= 0) {
+      return
+    }
+    _logi(logger, `[emailer] Sending emails queue (${pending.length} emails)`)
+  
+    // const emailString = await client.lPop('email_queue')
+    // if (!emailString) break
+    // const email = JSON.parse(emailString)
+  
+    for (const email of pending) {
+      if (email.count > 1) {
+        _logi(logger, `[emailer] Sending queued and stacked email [${email.subject}](x${email.count})...`)
+        email.subject = `${email.subject} (x${email.count})`
+      } else {
+        _logi(logger, `[emailer] Sending queued email [${email.subject}]...`)
+      }
+
+      send_email(email).then((res) => {
+        _logi(logger, `[emailer] Queued email [${email.subject}]sent ${res.ok ? 'OK' : 'NOT OK'}`)
+        if (res.ok) {
+          delete email_queue_remove_ids(email.ids, logger)
+        }
+      })
+    }
+    _logi(logger, `[emailer] Sent emails ${pending.length} from queue`)
+  }
+  
+  const emailer= {
+    send: send_email, 
+    verify: verify_emailer,
+    queue_email,
+    queue_send_emails,
+    options, 
+    defaults, 
+    silent
+  }
+  return emailer
+}
+
+
+
+let _cache = {}
+
+export function init_emailer_transporter({options, defaults, silent}, logger= undefined) {
+
+  const ckey = JSON.stringify({options, defaults, silent})
+  if (ckey in _cache) {
+    return _cache[ckey]
+  }
+
+  _cache[ckey] = _init_emailer_transporter({options, defaults, silent}, logger) 
+  return _cache[ckey]
+
+}

package/src/server/engines/geoip/index.mjs

@@ -0,0 +1,66 @@
+import fs from 'fs'
+import {Reader} from '@maxmind/geoip2-node'
+
+let _geoip_reader = undefined
+
+const _geoip_def_local_ips= [
+  '127.0.0.1',
+  '::1:',
+  '172.19.0.1' // Docker inner - probably healthcheck
+]
+
+function _geoip_is_local(ip, local_ips = []) {
+  // if (process.env.NODE_ENV == 'development') {
+  //   return true
+  // }
+  
+  const all_local_ips = [
+      ..._geoip_def_local_ips,
+      ...local_ips || []
+    ]
+  return all_local_ips.indexOf(ip) >= 0
+}
+
+
+
+function _geoip_init(db = '/var/lib/GeoIP/GeoLite2-City.mmdb', logger= console) {
+  try {
+    if (_geoip_reader != undefined) {
+      return _geoip_reader
+    }
+    const dbBuffer = fs.readFileSync(db)
+    _geoip_reader = Reader.openBuffer(dbBuffer)
+    return _geoip_reader
+  } catch(error) {
+    logger.error(`[geoip] Error initing:`)
+    logger.error(error)
+    return undefined
+  }
+}
+
+export const geoip_localize_ip= (ip, config, logger= console) => {
+  if (_geoip_is_local(ip, config?.local_ips)) {
+    return {
+      local: true,
+      country: '',
+      city: '',
+    }      
+  }
+
+  try {
+    const reader = _geoip_init(config?.db, logger)
+    const resp= reader.city(ip)
+    return {
+      country: resp.country.isoCode,
+      city   : resp.city?.names?.en
+    }
+  } catch(error) {
+    logger.error(`[geoip] Error localizing IP ${ip} (not in the database)`)
+    //logger.error(error)
+  }
+
+  return {
+    country: '',
+    city: '',
+  }  
+}