@opengis/fastify-table 1.4.88 → 1.5.1

package/server/plugins/md/funcs/formatMdoc.js

@@ -1,40 +0,0 @@
-/* eslint-disable no-param-reassign */
-function handleData(data, curr, acc, titles, plain, splittype = 1) {
-    const titleStr = {
-        1: `${(titles?.[curr] || curr)}\n`, // for lists
-        2: `${(titles?.[curr] || curr)}: `, // one-liner
-        3: '', // for nested lists w/ one property
-    }[splittype];
-
-    if (Array.isArray(data[curr])) {
-        const str = `${plain ? '' : '\n\n### '}${titleStr}${data[curr].map(el => {
-            if (typeof el === 'object' && !Array.isArray(el)) {
-                const str1 = Object.keys(el).filter(item => item !== 'id').reduce((acc1, curr1) => handleData(el, curr1, acc1, {}, 1, 3), '');
-                return `\n - ${str1}`;
-            }
-            return `\n - ${el}`;
-        })}`;
-        acc += `${str}`;
-        return acc;
-    } if (typeof data[curr] === 'object') {
-        const str = `${plain ? '' : '\n\n### '} ${titleStr}${JSON.stringify(data[curr])}`;
-        acc += str;
-        return acc;
-    }
-
-    acc += `${plain ? '' : '\n\n### '}${titleStr}${data[curr] || ''}`;
-    return acc;
-}
-
-export default function formatMdoc(data, titles = {}, headerColumns = [], descriptionColumns = []) {
-    if (!data) return 'empty data';
-    if (typeof data !== 'object' || Array.isArray(data)) return 'invalid data';
-    if (headerColumns && !Array.isArray(headerColumns)) return 'invalid headerColumns';
-    if (descriptionColumns && !Array.isArray(descriptionColumns)) return 'invalid descriptionColumns';
-
-    const header = Object.keys(data).filter(key => headerColumns.includes(key) && data[key]).reduce((acc, curr) => handleData(data, curr, acc, titles, 1, 2), '');
-
-    const description = Object.keys(data).filter(key => descriptionColumns.includes(key) && data[key]).reduce((acc, curr) => handleData(data, curr, acc, titles), '');
-
-    return `---\n${header}\n---${description}`.replace(/,,/g, ',');
-}

package/server/plugins/metric/index.js

@@ -1,8 +0,0 @@
-import addCron from '../cron/funcs/addCron.js';
-import systemMetricsFifthly from './systemMetricsFifthly.js';
-
-async function plugin() {
-    addCron(systemMetricsFifthly, 60 * 15);
-}
-
-export default plugin;

package/server/plugins/metric/loggerSystem.js

@@ -1,131 +0,0 @@
-import os from 'os';
-import process from 'process';
-import util from 'node:util';
-import { exec } from 'node:child_process';
-
-import {
-  config, getFolder, pgClients, getRedis,
-} from '../../../utils.js';
-
-const execAsync = util.promisify(exec);
-const platform = os.platform();
-const processFolder = os.homedir();
-const formatMemoryUsage = (data) => `${(data / 1024 / 1024).toFixed(2)} MB`;
-
-const rclient = getRedis();
-const rclient2 = getRedis({ db: 2 });
-// const rclient10 = getRedis({ db: 10 });
-const filesFolder = getFolder(config);
-
-const redisKey = 'logger-process-online';
-const sqlQuery = `select datname as dbname, application_name as app, client_addr as client_ip, 
-(now()-backend_start)::text as msec, state, query from pg_stat_activity where datname=$1`;
-
-export default async function loggerSystem(req) {
-  const { pg = pgClients.client } = req;
-  const dbName = pg?.options?.database;
-
-  const dbsize = config.redis ? await rclient.get(`${dbName}:content:dbsize:${redisKey}`) : null;
-
-  const dbVerion = await pg.query('select version();').then(el => el.rows?.[0]?.version);
-
-  const dbSize = dbsize
-    || (await pg.query(`select pg_size_pretty(pg_database_size('${dbName}'));`).then(el => el.rows?.[0]?.pg_size_pretty));
-
-  const { rows: query = [] } = await pg.query(sqlQuery, [dbName]);
-
-  const { stdout: topProcess } = platform === 'win32'
-    ? { stdout: 'Cant show top on this system type' }
-    : await execAsync('top -b -n 1');
-
-  const osInfo = `${os.version()} ${os.machine?.() || ''}`;
-  const cpuInfo = os.cpus();
-
-  const totalCpu = (Object.values(cpuInfo[0].times).reduce((acc, tv) => acc + tv, 0)) * cpuInfo.length;
-  const totalMemory = os.totalmem();
-
-  const memory = process.memoryUsage();
-  const resource = process.resourceUsage();
-
-  const currentCpuUsage = (((process.cpuUsage().user + process.cpuUsage().system) * 1000) / totalCpu) * 100;
-  const redisInfo = config.redis ? await rclient.info() : '';
-
-  const lines = redisInfo.split('\r\n').filter((el) => el && el.split);
-  const redis = {};
-
-  for (let i = 0; i < lines.length; i += 1) {
-    const [key, val] = lines[i].split(':');
-    if (val) {
-      redis[key] = val;
-    }
-  }
-
-  if (config.redis) { await rclient.set(`${dbName}:content:dbsize:${redisKey}`, dbSize, 'EX', 30 * 60); }
-
-  const latency = config.redis ? await rclient.latency('latest') : null;
-
-  const uptime = await pg.query('select extract(\'epoch\' from current_timestamp - pg_postmaster_start_time())::int as uptime')
-    .then(el => el.rows?.[0]?.uptime);
-
-  const metric5 = config.redis ? await rclient2.hgetall(`${dbName}:system_metrics`) : {};
-
-  Object.keys(metric5 || {}).filter((el) => el.startsWith('time')).forEach((m) => {
-    metric5[m] /= 1000.0;
-  });
-
-  const metricSort = metric5 ? JSON.parse(JSON.stringify(metric5, Object.keys(metric5).sort())) : undefined;
-  const userOnline = config.redis ? await rclient2.scan(['0', 'match', `${dbName}:user:*`, 'count', '10000']) : [];
-  // const userOnline = await rclient10.scan(['0', 'match', 'sess:*', 'count', '10000']);
-
-  return {
-    process: {
-      root: process.cwd(),
-      processFolder,
-      saveDirectory: filesFolder,
-      dbhost: pg?.options?.host,
-      dbport: pg?.options?.port,
-      dbname: dbName,
-    },
-
-    uptime: {
-      node: Math.round(process.uptime()),
-      system: os.uptime(),
-      pg: uptime,
-      redis: redis.uptime_in_seconds - 0,
-    },
-    metric: {
-      cpu: `${Math.round(currentCpuUsage)} %`,
-      memory: `${Math.round(((memory.rss / totalMemory) * 100) * 100) / 100} %`,
-      'user.online': userOnline?.[1]?.length,
-      'node.total': formatMemoryUsage(memory.heapTotal),
-      'node.used': formatMemoryUsage(memory.heapUsed),
-      'node.rss': formatMemoryUsage(memory.rss),
-      'node.cpu.time': resource.systemCPUTime,
-      'node.cpu.load': os.loadavg()[0],
-      'redis.ram': redis.used_memory_human,
-      'redis.latency': latency?.join?.(','),
-      'redis.ram.peak': redis.used_memory_peak_human,
-      'redis.clients': redis.connected_clients,
-      'redis.db0': redis.db0?.split(',')?.[0]?.substring(5),
-      'redis.db1': redis.db1?.split(',')?.[0]?.substring(5),
-      'redis.db2': redis.db2?.split(',')?.[0]?.substring(5),
-      'redis.db10': redis.db10?.split(',')?.[0]?.substring(5),
-      'pg.connection': query?.length,
-      dbsize: dbSize,
-      filesize: 0,
-    },
-    metricSort,
-    system: {
-      os: osInfo,
-      release: os.release(),
-      core: cpuInfo.length,
-      cpu: cpuInfo[0].model,
-      ram: formatMemoryUsage(totalMemory),
-      db: dbVerion,
-      redis: rclient?.server_info?.redis_version,
-      node: process.version,
-    },
-    top: topProcess,
-    query,
-  };
-}

package/server/plugins/metric/systemMetricsFifthly.js

@@ -1,24 +0,0 @@
-import pgClients from '../pg/pgClients.js';
-import logger from '../logger/getLogger.js';
-import getRedis from '../redis/funcs/getRedis.js';
-import config from '../../../config.js';
-
-import loggerSystem from './loggerSystem.js';
-
-const rclient2 = getRedis({ db: 2 });
-
-export default async function systemMetricsFifthly({ pg = pgClients.client }) {
-  const system = await loggerSystem({ pg });
-  const dbName = pg?.options?.database;
-
-  if (config.redis) { await rclient2.del(`${dbName}:system_metrics`); }
-
-  logger.file('metric', {
-    dbname: dbName,
-    ...system.metric,
-    uptime: system.uptime,
-    ...system.metricSort,
-  });
-
-  return { message: 'Saved Fifthly' };
-}

package/server/plugins/migration/exec.migrations.js

@@ -1,63 +0,0 @@
-/* eslint-disable no-console */
-import path from 'node:path';
-import { readdirSync, existsSync } from 'node:fs';
-
-import config from '../../../config.js';
-import execSql from './exec.sql.js';
-import pgClients from '../pg/pgClients.js';
-
-const time = Date.now();
-const debug = config.debug || config.local;
-
-export default async function execMigrations(dirPath, pg = pgClients.client, iscore = false) {
-  if (!dirPath) {
-    const txt = 'migrations skip: path not specified';
-    if (debug) console.warn(txt);
-    return txt;
-  }
-
-  if (config.migrationsCore === false && iscore) {
-    const txt = `migrations skip: core - ${dirPath}`;
-    if (debug) console.log(txt);
-    return txt;
-  }
-
-  if (config.migrations === false && !iscore) {
-    const txt = `migrations skip: path - ${dirPath}`;
-    if (debug) console.log(txt);
-    return txt;
-  }
-
-  if (process.env.NODE_ENV !== 'production' && !(iscore ? config.migrationsCore : config.migrations)) {
-    const txt = `migrations skip: not a production environment - ${iscore ? 'core' : 'path'} : ${dirPath}`;
-    if (debug) console.log(txt);
-    return txt;
-  }
-
-  if (debug) console.log('migrations start', dirPath, Date.now() - time);
-
-  const exists = existsSync(dirPath);
-
-  if (!exists) {
-    const txt = `migrations skip: directory not found - ${dirPath}`;
-    if (debug) console.warn(txt);
-    return txt;
-  }
-
-  // get directory sql file list
-  const content = readdirSync(dirPath, { withFileTypes: true })
-    ?.filter((el) => el.isFile() && path.extname(el.name) === '.sql')
-    ?.map((el) => el.name) || [];
-
-  // execute sql files
-  if (!content?.length) {
-    const txt = `migrations skip: no sql in specified directory - ${dirPath}`;
-    if (debug) console.warn(txt);
-    return txt;
-  }
-
-  await content.reduce((promise, filename) => promise.then(() => execSql(path.join(dirPath, filename), pg)), Promise.resolve());
-  if (debug) console.log('migrations success', dirPath, exists, Date.now() - time);
-
-  return 'success';
-}

package/server/plugins/migration/exec.sql.js

@@ -1,67 +0,0 @@
-/* eslint-disable no-console */
-import path from 'node:path';
-import { createHash } from 'node:crypto';
-import { existsSync, readFileSync } from 'node:fs';
-
-import config from '../../../config.js';
-
-import pgClients from '../pg/pgClients.js';
-
-import logger from '../logger/getLogger.js';
-
-import getRedis from '../redis/funcs/getRedis.js';
-
-const rclient = getRedis();
-
-const debug = config.debug || config.local;
-
-export default async function execSql(filepath, pg = pgClients.client) {
-    const start = Date.now();
-
-    if (!pg || !filepath) return null;
-
-    const filename = path.basename(filepath);
-
-    if (!existsSync(filepath)) {
-        logger.file('migration/notfound', {
-            filepath,
-            result: 'not found',
-        });
-        console.log(filepath, 'warn', 'not found', Date.now() - start);
-        return null;
-    }
-
-    const sql = readFileSync(filepath, 'utf-8');
-
-    const hash = createHash('md5').update(sql).digest('hex');
-    const hashes = config.redis ? await rclient.hgetall(`${pg?.options?.database}:migration-hashes`).then(obj => Object.keys(obj)) : [];
-
-    if (hashes.includes(hash) && !config.disableCache && debug) {
-        console.log(filename, 'skip equal hash', Date.now() - start);
-        return null;
-    }
-
-    try {
-        if (debug) console.log(filename, 'start', Date.now() - start);
-        await pg.query(sql);
-        if (!config.disableCache && config.redis) await rclient.hset(`${pg?.options?.database}:migration-hashes`, hash, 1);
-        if (debug) console.log(filename, 'finish', Date.now() - start);
-        logger.file('migration/success', {
-            filepath,
-            result: 'success',
-            time: Date.now() - start,
-        });
-        return 'ok';
-    }
-    catch (err) {
-        console.error(filename, 'error', err.toString(), Date.now() - start);
-        logger.file('migration/error', {
-            filepath,
-            result: 'error',
-            error: err.toString(),
-            stack: err.stack,
-            time: Date.now() - start,
-        });
-        return err.toString();
-    }
-}

package/server/plugins/pg/funcs/autoIndex.js

@@ -1,103 +0,0 @@
-/* eslint-disable no-console */
-import config from '../../../../config.js';
-import pgClient from '../pgClients.js';
-
-// subfunc
-const getTable = ({ table }) => table?.toLowerCase?.()?.replace?.(/[\n\r]+/g, ' ')?.split?.(' from ')
-  ?.filter((el) => /^[a-z0-9_]+\.[a-z0-9_]+/.test(el))
-  ?.map((el) => el?.split?.(/[ )]/)?.[0]);
-
-const loadedIndex = {};
-
-// main func
-async function autoIndex({
-  table, columns: filter1, pg = pgClient.client, gin,
-}) {
-  const filter = filter1.filter(el => el);
-
-  if (!filter?.length || !table || process.env.NODE_ENV === 'unit test') return null;
-
-  const attrs = filter.map((el) => (el.name || el).replace(/'/g, '')).filter(el => el?.indexOf('(') === -1).sort();
-  const types = filter.filter(el => (el?.name || el)?.indexOf?.('(') === -1).map((el) => (el.name ? el : { name: el }))
-    .reduce((p, el) => ({ ...p, [el.name]: el.type || '-' }), {});
-
-  const redisKey = `autoindex1:${table}:${attrs.join(';')}`;
-  const tableList = getTable({ table });
-  const existsCache = loadedIndex[redisKey];
-  if (existsCache && !config.disableCache) { return 'ok'; }
-
-  const tbs = tableList[0];
-  const [ns, tbl] = tableList[0]?.split?.('.') || [];
-  const { rows: index } = tbl ? await pg.query('select * from pg_indexes where tablename = $1 and schemaname = $2', [tbl, ns]) : {};
-
-  if (existsCache && !config.disableCache && index?.length > 0) { return null; }
-  // console.log('autoindex', table, filter?.map((el) => el.name || el));
-  const { rows: cols } = await pg.query(`SELECT attrelid::regclass AS tbl, attname AS aname, atttypid::regtype  AS datatype, atttypid  
-    FROM  pg_attribute WHERE  attrelid = '${tbs}'::regclass and attname = any('{ ${attrs} }')`);
-
-  const qIndex = [];
-  const indexAr = {};
-
-  // console.log(cols);
-  for (let i = 0; i < cols.length; i += 1) {
-    const el = cols[i];
-    el.relname = tbl;
-    el.nspname = ns;
-
-    const isGin = types[el.aname] === 'Text' || gin;
-    const indexUsing = (el.datatype === 'geometry' ? 'gist' : null)
-      || (isGin ? 'gin' : null)
-      || 'btree';
-
-    const name = `${el.nspname}_${el.relname}_${el.aname}_${indexUsing}_idx`;
-
-    const exists = index.filter((ind) => ind.tablename === el.relname && ind.schemaname === el.nspname && ind.indexdef.includes(
-      isGin ? `gin (${el.aname} gin_trgm_ops)` : `btree (${el.aname})`,
-    ));
-
-    indexAr[el.aname] = {
-      name, type: el.datatype, exists, // exists1,
-    };
-
-    // drop
-    /* if (exists?.length > 1) {
-      exists.filter((ind, i1) => i1).forEach((ind) => qIndex.push(`DROP INDEX if exists ${ind.schemaname}.${ind.indexname} `));
-    } */
-
-    // add
-    if (exists?.length <= (isGin ? 2 : 1)) {
-      // console.log(`${name} - ${el.datatype}`);
-
-      const suffix = isGin ? 'gin_trgm_ops' : '';
-
-      qIndex.push(`CREATE INDEX if not exists ${name} ON ${el.nspname}.${el.relname} USING ${indexUsing} (${el.aname} ${suffix})`);
-      if (isGin) {
-        qIndex.push(`CREATE INDEX if not exists ${name}_lower ON ${el.nspname}.${el.relname} USING ${indexUsing} (lower(${el.aname}) ${suffix})`);
-      }
-
-      if (indexUsing === 'gist') {
-        qIndex.push(`CREATE INDEX if not exists ${name.replace('gist', 'gist1')}  
-          ON ${el.nspname}.${el.relname} USING gist (${el.aname})`);
-
-        qIndex.push(`CREATE INDEX if not exists ${name.replace('gist', 'area')}  
-          ON ${el.nspname}.${el.relname} USING btree (st_area(st_transform(${el.aname},3857)))`);
-        qIndex.push(`CREATE INDEX if not exists ${name.replace('gist', 'area_true')}  
-          ON ${el.nspname}.${el.relname} USING btree (st_area(${el.aname}, true))`);
-        qIndex.push(`CREATE INDEX if not exists ${name.replace('gist', '4326')}  
-          ON ${el.nspname}.${el.relname} USING gist (st_transform(${el.aname},4326))`);
-      }
-    }
-  }
-
-  loadedIndex[redisKey] = 1;
-  // throw qIndex;
-  if (!qIndex.length) return null;
-  if (config.local) {
-    // console.log(qIndex.filter((v, i, a) => a.indexOf(v) === i));
-  }
-  // logger.file('index', { table, filter, sql: qIndex.filter((v, i, a) => a.indexOf(v) === i) });
-  qIndex.filter((v, i, a) => a.indexOf(v) === i).map((el) => pg.one(el).catch((err) => console.log(err.toString())));
-  return 'ok';
-}
-
-export default autoIndex;

package/server/plugins/pg/funcs/getMeta.js

@@ -1,49 +0,0 @@
-import getPG from './getPG.js';
-
-const data = {};
-
-// decorator
-export default async function getMeta(opt, nocache) {
-  const pg = opt?.pg || getPG({ name: 'client' });
-  if (!pg) return { error: 'pg connection not established', status: 400 };
-  const table1 = opt?.table || opt;
-  const table = pg.pk[table1] ? table1 : table1?.replace?.(/"/g, '');
-
-  if (pg?.options?.database && data[pg.options.database]?.[table] && !nocache) return data[pg.options.database][table];
-
-  if (!pg?.tlist?.includes(table) && !pg?.tlist?.includes(table?.replace?.(/"/g, ''))) {
-    return { error: `${table} - not found`, status: 400 };
-  }
-
-  const { fields = [] } = await pg.query(`select * from ${table} limit 0`);
-  const pks1 = await pg.query(`SELECT json_object_agg(c.conrelid::regclass, a.attname) as pks1 
-  FROM pg_constraint c 
-    left join pg_attribute a on c.conrelid=a.attrelid and a.attnum = c.conkey[1] 
-  WHERE c.contype='p'::"char" and c.conrelid::regclass = $1::regclass`, [table]).then(el => el.rows[0].pks1 || {});
-
-  const pk = table.startsWith('public.')
-    ? (pks1[table.replace('public.', '')] || pks1[table.replace('public.', '')])
-    : (pks1[table] || pks1[table]);
-
-  const geomAttr = fields.find((el) => pg.pgType?.[el.dataTypeID] === 'geometry')?.name; // change geometry text to geometry code
-
-  const dbColumns = await pg.query(`select json_object_agg(
-    attname, 
-    json_build_object(
-      'title', pg_catalog.col_description(attrelid,attnum)
-      /*,'type', atttypid::regtype */
-    )
-  ) from pg_catalog.pg_attribute a 
-  where attrelid=$1::regclass and attnum>0`, [table]).then(el => el.rows?.[0]?.json_object_agg || {});
-
-  fields.forEach(el => Object.assign(el, { ...dbColumns[el.name] || {} }));
-
-  const res = {
-    pk, columns: fields, geom: geomAttr, view: pg.relkinds?.[table] === 'v',
-  };
-  if (!data[pg.options.database]) {
-    data[pg.options.database] = {};
-  }
-  data[pg.options.database][table] = res;
-  return res;
-}

package/server/plugins/pg/funcs/getPG.js

@@ -1,42 +0,0 @@
-import pg from 'pg';
-
-const { types } = pg;
-types.setTypeParser(1082, (stringValue) => stringValue);
-types.setTypeParser(1114, (stringValue) => stringValue);
-
-import config from '../../../../config.js';
-import dblist from '../../../../dblist.js';
-import pgClients from '../pgClients.js';
-import init from './init.js';
-import getDBParams from './getDBParams.js';
-
-function getPG(param) {
-  if (!config.pg) return null;
-  const dbListParams = dblist.find(el => el.key === param?.key)
-    || dblist.find(el => el.database === (param?.db || param?.database || param) && el.port === param?.port);
-  const {
-    user, password, host, port, db, database, name: origin,
-  } = dbListParams ?? (typeof param === 'string' ? getDBParams(param) : param || {});
-  const name = origin || db || database || param || 'client';
-  if (pgClients[name]) return pgClients[name];
-
-  const dbConfig = {
-    user: user || config.pg?.user || 'postgres',
-    password: password || config.pg?.password || 'postgres',
-    host: host || config.pg?.host,
-    port: port || config.pg?.port,
-    database: db || database || config.pg?.db || config.pg?.database,
-    statement_timeout: config.pg?.statement_timeout || 10000,
-  };
-
-  if (!dbConfig.database) { return null; }
-
-  pgClients[name] = new pg.Pool(dbConfig);
-  pgClients[name].init = async () => {
-    await init(pgClients[name]);
-  };
-  init(pgClients[name]);
-  return pgClients[name];
-}
-
-export default getPG;

package/server/plugins/pg/funcs/getPGAsync.js

@@ -1,62 +0,0 @@
-import pg from 'pg';
-
-const { types } = pg;
-types.setTypeParser(1082, (stringValue) => stringValue);
-types.setTypeParser(1114, (stringValue) => stringValue);
-
-import config from '../../../../config.js';
-import dblist from '../../../../dblist.js';
-import pgClients from '../pgClients.js';
-import init from './init.js';
-import getDBParams from './getDBParams.js';
-
-async function getPGAsync(param) {
-  if (!config.pg) return null;
-  const dbListParams = dblist.find(el => el.key === param?.key)
-    || dblist.find(el => el.database === (param?.db || param?.database || param));
-  const {
-    user, password, host, port, db, database, name: origin,
-  } = dbListParams ?? (typeof param === 'string' ? getDBParams(param) : param || {});
-  const name = origin || db || database || param || 'client';
-
-  if (pgClients[name]?.tlist) return pgClients[name];
-
-  const dbConfig = {
-    user: user || config.pg?.user || 'postgres',
-    password: password || config.pg?.password || 'postgres',
-    host: host || config.pg?.host,
-    port: port || config.pg?.port,
-    database: db || database || config.pg?.db || config.pg?.database,
-    statement_timeout: config.pg?.statement_timeout || 10000,
-  };
-
-  if (!dbConfig.database) { return null; }
-
-  pgClients[name] = new pg.Pool(dbConfig);
-
-  try {
-    await init(pgClients[name]);
-  }
-  catch (err) {
-    console.error(
-      'pg client init error',
-      host,
-      port,
-      database,
-    );
-    // debug only
-    if (config.local && name !== 'client') {
-      console.warn(
-        'fall back to default pgClient',
-        pgClients.client?.host,
-        pgClients.client?.port,
-        pgClients.client?.database,
-      );
-      return pgClients.client;
-    }
-  }
-
-  return pgClients[name];
-}
-
-export default getPGAsync;