@opengis/fastify-table 1.2.19 → 1.2.21

package/server/plugins/crud/funcs/utils/logChanges.js

@@ -1,118 +1,118 @@
-import getTemplate from '../../..//table/funcs/getTemplate.js';
-import { metaFormat } from '@opengis/fastify-table/utils.js';
-
-const defaultTitles = {
-  editor_date: 'Дата оновлення',
-  editor_id: 'Редактор',
-  сdate: 'Дата створення',
-  uid: 'Автор',
-  body: 'Зміст',
-  entity_id: 'ID Сутності',
-  entity_type: 'Таблиця сутності',
-  file_path: 'Шлях до файлу',
-  uploaded_name: 'Назва файлу',
-  size: 'Розмір файлу',
-  ext: 'Розширення файлу',
-};
-
-function getValue(val, tableName) {
-  if (!val) return null;
-  if (['crm.files'].includes(tableName)) {
-    return typeof val === 'object'
-      ? JSON.stringify(val)
-      : val;
-  }
-  return typeof val === 'object'
-    ? JSON.stringify(val)?.substring?.(0, 30)
-    : val?.toString?.()?.substring?.(0, 30);
-}
-
-// extract titles and cls from form schema
-// alt: extract table template from referer -> form
-export default async function logChanges({
-  pg, table: table1, tokenData, referer, id, data, uid = 1, type,
-}) {
-  const table = table1.replace(/"/g, '');
-  if (!id) {
-    console.error('param id is required');
-    return null;
-  }
-  if (!table || !pg.pk?.[table]) {
-    console.error('table not found');
-    return null;
-  }
-  if (!pg.pk?.['log.table_changes'] || !pg.pk?.['log.table_changes_data']) {
-    console.error('log table not found');
-    return null;
-  }
-  if (!type) {
-    console.error('invalid type');
-    return null;
-  }
-
-  try {
-    const { change_id: changeId } = await pg.query(`insert into log.table_changes(change_date,change_type,change_user_id,entity_type,entity_id) 
-    values(CURRENT_DATE, $1, $2, $3, $4) returning change_id`, [type, uid, table, id]).then((res) => res.rows?.[0] || {});
-
-    const q = `select json_object_agg(entity_key, value_new) from (
-      select
-        entity_key,
-        value_new,
-        ( rank() over (partition by entity_key order by cdate desc) = 1 ) as is_latest
-      from log.table_changes_data
-
-      where change_id in (
-        select
-          change_id
-        from log.table_changes
-        where entity_id=$1
-        and entity_type=$2
-      )
-
-    )q where is_latest`;
-    // console.log(q, type, id);
-
-    const old = type !== 'INSERT' ? await pg.query(q, [id, table]).then(el => el.rows?.[0]?.json_object_agg || {}) : {};
-
-    const body = await getTemplate('form', tokenData?.form);
-    const schema = body?.schema || body || {};
-    const titles = Object.keys(schema)
-      .reduce((acc, curr) => Object.assign(acc, { [curr]: schema[curr].title || schema[curr].ua }), {});
-    const cls = Object.keys(schema)
-      .filter(el => schema[el]?.data)
-      .reduce((acc, curr) => Object.assign(acc, { [curr]: schema[curr].data }), {});
-
-    if (data) {
-      await metaFormat({ rows: [data], cls, sufix: false });
-    }
-
-    const newObj = Object.fromEntries(Object.entries(data || {}).map(el => ([[titles[el[0]] || defaultTitles[el[0]] || el[0]], el[1]])));
-    const changesData = Object.keys(newObj || {}).map(el => ({
-      change_id: changeId,
-      entity_key: el,
-      value_old: getValue(old?.[el], table),
-      value_new: type === 'DELETE' ? null : getValue(newObj?.[el], table),
-      uid,
-    })).filter(el => el?.value_new !== el?.value_old);
-
-    const res = await Promise.all(changesData.map(async (el) => {
-      const insertQuery = `insert into log.table_changes_data (${Object.entries(el)?.map((key) => `"${key[0]}"`).join(',')}) 
-      values (${Object.entries(el)?.map((key, i) => `$${i + 1}`).join(',')}) returning *`;
-
-      const { rows = [] } = await pg.query(insertQuery, [...Object.entries(el).map((el1) => (el1[1] && typeof el1[1] === 'object' && (!Array.isArray(el1[1]) || typeof el1[1]?.[0] === 'object') ? JSON.stringify(el1[1]) : el1[1]))]) || {};
-      return rows[0];
-    }));
-
-    const newData = type === 'DELETE' ? {} : (Array.isArray(res) ? res : [res]).reduce((acc, curr) => Object.assign(acc, { [curr.entity_key]: curr.value_new }), {});
-    // console.log('logChanges OK', type);
-    return {
-      change_id: changeId, entity_type: table, entity_id: id, uid, change_type: type, old, new: newData,
-    };
-  }
-  catch (err) {
-    console.error('logChanges error', type, table, id, data, err.toString());
-    return {
-      error: err.toString(), entity_type: table, entity_id: id, uid, change_type: type,
-    };
-  }
-}
+import getTemplate from '../../..//table/funcs/getTemplate.js';
+import { metaFormat } from '@opengis/fastify-table/utils.js';
+
+const defaultTitles = {
+  editor_date: 'Дата оновлення',
+  editor_id: 'Редактор',
+  сdate: 'Дата створення',
+  uid: 'Автор',
+  body: 'Зміст',
+  entity_id: 'ID Сутності',
+  entity_type: 'Таблиця сутності',
+  file_path: 'Шлях до файлу',
+  uploaded_name: 'Назва файлу',
+  size: 'Розмір файлу',
+  ext: 'Розширення файлу',
+};
+
+function getValue(val, tableName) {
+  if (!val) return null;
+  if (['crm.files'].includes(tableName)) {
+    return typeof val === 'object'
+      ? JSON.stringify(val)
+      : val;
+  }
+  return typeof val === 'object'
+    ? JSON.stringify(val)?.substring?.(0, 30)
+    : val?.toString?.()?.substring?.(0, 30);
+}
+
+// extract titles and cls from form schema
+// alt: extract table template from referer -> form
+export default async function logChanges({
+  pg, table: table1, tokenData, referer, id, data, uid = 1, type,
+}) {
+  const table = table1.replace(/"/g, '');
+  if (!id) {
+    console.error('param id is required');
+    return null;
+  }
+  if (!table || !pg.pk?.[table]) {
+    console.error('table not found');
+    return null;
+  }
+  if (!pg.pk?.['log.table_changes'] || !pg.pk?.['log.table_changes_data']) {
+    console.error('log table not found');
+    return null;
+  }
+  if (!type) {
+    console.error('invalid type');
+    return null;
+  }
+
+  try {
+    const { change_id: changeId } = await pg.query(`insert into log.table_changes(change_date,change_type,change_user_id,entity_type,entity_id) 
+    values(CURRENT_DATE, $1, $2, $3, $4) returning change_id`, [type, uid, table, id]).then((res) => res.rows?.[0] || {});
+
+    const q = `select json_object_agg(entity_key, value_new) from (
+      select
+        entity_key,
+        value_new,
+        ( rank() over (partition by entity_key order by cdate desc) = 1 ) as is_latest
+      from log.table_changes_data
+
+      where change_id in (
+        select
+          change_id
+        from log.table_changes
+        where entity_id=$1
+        and entity_type=$2
+      )
+
+    )q where is_latest`;
+    // console.log(q, type, id);
+
+    const old = type !== 'INSERT' ? await pg.query(q, [id, table]).then(el => el.rows?.[0]?.json_object_agg || {}) : {};
+
+    const body = await getTemplate('form', tokenData?.form);
+    const schema = body?.schema || body || {};
+    const titles = Object.keys(schema)
+      .reduce((acc, curr) => Object.assign(acc, { [curr]: schema[curr].title || schema[curr].ua }), {});
+    const cls = Object.keys(schema)
+      .filter(el => schema[el]?.data)
+      .reduce((acc, curr) => Object.assign(acc, { [curr]: schema[curr].data }), {});
+
+    if (data) {
+      await metaFormat({ rows: [data], cls, sufix: false });
+    }
+
+    const newObj = Object.fromEntries(Object.entries(data || {}).map(el => ([[titles[el[0]] || defaultTitles[el[0]] || el[0]], el[1]])));
+    const changesData = Object.keys(newObj || {}).map(el => ({
+      change_id: changeId,
+      entity_key: el,
+      value_old: getValue(old?.[el], table),
+      value_new: type === 'DELETE' ? null : getValue(newObj?.[el], table),
+      uid,
+    })).filter(el => el?.value_new !== el?.value_old);
+
+    const res = await Promise.all(changesData.map(async (el) => {
+      const insertQuery = `insert into log.table_changes_data (${Object.entries(el)?.map((key) => `"${key[0]}"`).join(',')}) 
+      values (${Object.entries(el)?.map((key, i) => `$${i + 1}`).join(',')}) returning *`;
+
+      const { rows = [] } = await pg.query(insertQuery, [...Object.entries(el).map((el1) => (el1[1] && typeof el1[1] === 'object' && (!Array.isArray(el1[1]) || typeof el1[1]?.[0] === 'object') ? JSON.stringify(el1[1]) : el1[1]))]) || {};
+      return rows[0];
+    }));
+
+    const newData = type === 'DELETE' ? {} : (Array.isArray(res) ? res : [res]).reduce((acc, curr) => Object.assign(acc, { [curr.entity_key]: curr.value_new }), {});
+    // console.log('logChanges OK', type);
+    return {
+      change_id: changeId, entity_type: table, entity_id: id, uid, change_type: type, old, new: newData,
+    };
+  }
+  catch (err) {
+    console.error('logChanges error', type, table, id, data, err.toString());
+    return {
+      error: err.toString(), entity_type: table, entity_id: id, uid, change_type: type,
+    };
+  }
+}

package/server/plugins/logger/getLogger.js

@@ -1,14 +1,18 @@
 import pino from 'pino';
 // import path from 'node:path';
 
-import config from '../../../config.js';
-
-if (!config.log) config.log = {};
+import { config, getRedis } from '../../../utils.js';
 
 // utils
 import getHooks from './getHooks.js';
 import serializers from './serializers.js';
 
+const isServer = process.argv[2];
+
+const rclient2 = getRedis({ db: 2 });
+
+if (!config.log) config.log = {};
+
 const level = config.log?.level || process.env.PINO_LOG_LEVEL || 'info';
 console.log(`log level: ${level}`);
 const options = {
@@ -32,4 +36,10 @@ const logger = pino(options);
 logger.file = function userFile(logfolder, msg, req) {
   logger.info({ logfolder, ...(typeof msg === 'string' ? { msg } : msg) }, req);
 };
+logger.metrics = function metrics(key, val, dbName) {
+  const dbname = dbName || config.pg?.database;
+  if (!dbname && !isServer) return;
+  rclient2.hincrby(`${dbname}:system_metrics`, key, val || 1);
+};
+
 export default logger;

package/server/plugins/metric/index.js

@@ -0,0 +1,8 @@
+import addCron from '../cron/funcs/addCron.js';
+import systemMetricsFifthly from './systemMetricsFifthly.js';
+
+async function plugin(fastify) {
+    addCron(systemMetricsFifthly, 60 * 15);
+}
+
+export default plugin;

package/server/plugins/metric/loggerSystem.js

@@ -0,0 +1,129 @@
+import os from 'os';
+import process from 'process';
+import util from 'node:util';
+import { exec } from 'node:child_process';
+
+import { config, getFolder, pgClients, getRedis } from '../../../utils.js';
+
+const execAsync = util.promisify(exec);
+const platform = os.platform();
+const processFolder = os.homedir();
+const formatMemoryUsage = (data) => `${(data / 1024 / 1024).toFixed(2)} MB`;
+
+const rclient = getRedis();
+const rclient2 = getRedis({ db: 2 });
+// const rclient10 = getRedis({ db: 10 });
+const filesFolder = getFolder(config);
+
+const redisKey = 'logger-process-online';
+const sqlQuery = `select datname as dbname, application_name as app, client_addr as client_ip, 
+(now()-backend_start)::text as msec, state, query from pg_stat_activity where datname=$1`;
+
+export default async function loggerSystem(req) {
+  const { pg = pgClients.client } = req;
+  const dbName = pg.options?.database;
+
+  const dbsize = await rclient.get(`${dbName}:content:dbsize:${redisKey}`);
+
+  const dbVerion = await pg.query('select version();').then(el => el.rows?.[0]?.version);
+
+  const dbSize = dbsize
+    || (await pg.query(`select pg_size_pretty(pg_database_size('${dbName}'));`).then(el => el.rows?.[0]?.pg_size_pretty));
+
+  const { rows: query = [] } = await pg.query(sqlQuery, [dbName]);
+
+  const { stdout: topProcess } = platform === 'win32'
+    ? { stdout: 'Cant show top on this system type' }
+    : await execAsync('top -b -n 1');
+
+  const osInfo = `${os.version()} ${os.machine?.() || ''}`;
+  const cpuInfo = os.cpus();
+
+  const totalCpu = (Object.values(cpuInfo[0].times).reduce((acc, tv) => acc + tv, 0)) * cpuInfo.length;
+  const totalMemory = os.totalmem();
+
+  const memory = process.memoryUsage();
+  const resource = process.resourceUsage();
+
+  const currentCpuUsage = (((process.cpuUsage().user + process.cpuUsage().system) * 1000) / totalCpu) * 100;
+  const redisInfo = await rclient.info();
+
+  const lines = redisInfo.split('\r\n').filter((el) => el && el.split);
+  const redis = {};
+
+  for (let i = 0; i < lines.length; i += 1) {
+    const [key, val] = lines[i].split(':');
+    if (val) {
+      redis[key] = val;
+    }
+  }
+
+  await rclient.set(`${dbName}:content:dbsize:${redisKey}`, dbSize, 'EX', 30 * 60);
+
+  const latency = await rclient.latency('latest');
+
+  const uptime = await pg.query(`select extract('epoch' from current_timestamp - pg_postmaster_start_time())::int as uptime`)
+    .then(el => el.rows?.[0]?.uptime);
+
+  const metric5 = await rclient2.hgetall(`${dbName}:system_metrics`);
+
+  Object.keys(metric5 || {}).filter((el) => el.startsWith('time')).forEach((m) => {
+    metric5[m] /= 1000.0;
+  });
+
+  const metricSort = metric5 ? JSON.parse(JSON.stringify(metric5, Object.keys(metric5).sort())) : undefined;
+  const userOnline = await rclient2.scan(['0', 'match', `${dbName}:user:*`, 'count', '10000']);
+  // const userOnline = await rclient10.scan(['0', 'match', 'sess:*', 'count', '10000']);
+
+  return {
+    process: {
+      root: process.cwd(),
+      processFolder,
+      saveDirectory: filesFolder,
+      dbhost: pg.options?.host,
+      dbport: pg.options?.port,
+      dbname: dbName,
+    },
+
+    uptime: {
+      node: Math.round(process.uptime()),
+      system: os.uptime(),
+      pg: uptime,
+      redis: redis.uptime_in_seconds - 0,
+    },
+    metric: {
+      cpu: `${Math.round(currentCpuUsage)} %`,
+      memory: `${Math.round(((memory.rss / totalMemory) * 100) * 100) / 100} %`,
+      'user.online': userOnline?.[1]?.length,
+      'node.total': formatMemoryUsage(memory.heapTotal),
+      'node.used': formatMemoryUsage(memory.heapUsed),
+      'node.rss': formatMemoryUsage(memory.rss),
+      'node.cpu.time': resource.systemCPUTime,
+      'node.cpu.load': os.loadavg()[0],
+      'redis.ram': redis.used_memory_human,
+      'redis.latency': latency?.join(','),
+      'redis.ram.peak': redis.used_memory_peak_human,
+      'redis.clients': redis.connected_clients,
+      'redis.db0': redis.db0?.split(',')?.[0]?.substring(5),
+      'redis.db1': redis.db1?.split(',')?.[0]?.substring(5),
+      'redis.db2': redis.db2?.split(',')?.[0]?.substring(5),
+      'redis.db10': redis.db10?.split(',')?.[0]?.substring(5),
+      'pg.connection': query?.length,
+      dbsize: dbSize,
+      filesize: 0,
+    },
+    metricSort,
+    system: {
+      os: osInfo,
+      release: os.release(),
+      core: cpuInfo.length,
+      cpu: cpuInfo[0].model,
+      ram: formatMemoryUsage(totalMemory),
+      db: dbVerion,
+      redis: rclient.server_info?.redis_version,
+      node: process.version,
+    },
+    top: topProcess,
+    query,
+  };
+};

package/server/plugins/metric/systemMetricsFifthly.js

@@ -0,0 +1,23 @@
+import pgClients from '../pg/pgClients.js';
+import logger from '../logger/getLogger.js';
+import getRedis from '../redis/funcs/getRedis.js';
+
+import loggerSystem from './loggerSystem.js';
+
+const rclient2 = getRedis({ db: 2 });
+
+export default async function systemMetricsFifthly({ pg = pgClients.client }) {
+  const system = await loggerSystem({ pg });
+  const dbName = pg.options?.database;
+
+  await rclient2.del(`${dbName}:system_metrics`);
+
+  logger.file('metric', {
+    dbname: dbName,
+    ...system.metric,
+    uptime: system.uptime,
+    ...system.metricSort,
+  });
+
+  return { message: 'Saved Fifthly' };
+}

package/server/plugins/migration/exec.migrations.js

@@ -1,37 +1,37 @@
-import path from 'node:path';
-import { readdirSync, existsSync } from 'node:fs';
-
-import config from '../../../config.js';
-import execSql from './exec.sql.js';
-// import getCallerDir from './get.caller.dir.js';
-
-const time = Date.now();
-
-export default async function execMigrations(dirPath, iscore) {
-  if (
-    !dirPath
-    || (config.migrationsCore === false && iscore)
-    || config.migrations === false
-  ) {
-    console.log('migrations skip', 'core: ', !!iscore, 'dir: ', dirPath || 'not specified');
-    return;
-  }
-
-  console.log('migrations start', dirPath, Date.now() - time);
-
-  const exists = existsSync(dirPath);
-
-  if (exists) {
-    // get directory sql file list
-    const content = readdirSync(dirPath, { withFileTypes: true })
-      ?.filter((el) => el.isFile() && path.extname(el.name) === '.sql')
-      ?.map((el) => el.name) || [];
-
-    // execute sql files
-    if (content?.length) {
-      await content.reduce((promise, filename) => promise.then(() => execSql(path.join(dirPath, filename))), Promise.resolve());
-    }
-  }
-
-  console.log('migrations finish', dirPath, exists, Date.now() - time);
-}
+import path from 'node:path';
+import { readdirSync, existsSync } from 'node:fs';
+
+import config from '../../../config.js';
+import execSql from './exec.sql.js';
+// import getCallerDir from './get.caller.dir.js';
+
+const time = Date.now();
+
+export default async function execMigrations(dirPath, iscore) {
+  if (
+    !dirPath
+    || (config.migrationsCore === false && iscore)
+    || config.migrations === false
+  ) {
+    console.log('migrations skip', 'core: ', !!iscore, 'dir: ', dirPath || 'not specified');
+    return;
+  }
+
+  console.log('migrations start', dirPath, Date.now() - time);
+
+  const exists = existsSync(dirPath);
+
+  if (exists) {
+    // get directory sql file list
+    const content = readdirSync(dirPath, { withFileTypes: true })
+      ?.filter((el) => el.isFile() && path.extname(el.name) === '.sql')
+      ?.map((el) => el.name) || [];
+
+    // execute sql files
+    if (content?.length) {
+      await content.reduce((promise, filename) => promise.then(() => execSql(path.join(dirPath, filename))), Promise.resolve());
+    }
+  }
+
+  console.log('migrations finish', dirPath, exists, Date.now() - time);
+}

package/server/plugins/policy/index.js

@@ -1,12 +1,12 @@
-import checkPolicy from './funcs/checkPolicy.js';
-
-async function plugin(fastify) {
-  fastify.addHook('preParsing', async (request, reply) => {
-    const resp = checkPolicy(request, reply);
-    if (resp) {
-      return resp;
-    }
-  });
-}
-
-export default plugin;
+import checkPolicy from './funcs/checkPolicy.js';
+
+async function plugin(fastify) {
+  fastify.addHook('preParsing', async (request, reply) => {
+    const resp = checkPolicy(request, reply);
+    if (resp) {
+      return resp;
+    }
+  });
+}
+
+export default plugin;