@opengis/fastify-table 1.0.68 → 1.0.70

package/Changelog.md

@@ -1,6 +1,14 @@
 # fastify-table
 
-## 1.0.68- 26.07.2024
+## 1.0.70 - 29.07.2024
+
+- add cron funcs and API
+
+## 1.0.69 - 26.07.2024
+
+- add column classifier mode for data API
+
+## 1.0.68 - 26.07.2024
 
 - code optimization
 

package/cron/controllers/cronApi.js

@@ -0,0 +1,22 @@
+import cronList from './utils/cronList.js';
+
+export default async function cronApi(req) {
+  const {
+    params = {}, user = {}, hostname,
+  } = req;
+
+  if ((!user.uid || !user.user_type?.includes('admin')) && !hostname?.includes('localhost')) {
+    return { message: 'access restricted', status: 403 };
+  }
+
+  if (params.name === 'list') {
+    return { data: Object.keys(cronList || {}) };
+  }
+
+  if (!cronList[params.name]) {
+    return { message: `cron not found: ${params.name}`, status: 404 };
+  }
+
+  const result = await cronList[params.name](req);
+  return result;
+}

package/cron/controllers/utils/cronList.js

@@ -0,0 +1 @@
+export default {};

package/cron/funcs/addCron.js

@@ -0,0 +1,131 @@
+import { createHash } from 'crypto';
+
+import cronList from '../controllers/utils/cronList.js';
+import getRedis from '../../redis/funcs/getRedis.js';
+import getPG from '../../pg/funcs/getPG.js';
+
+const md5 = (string) => createHash('md5').update(string).digest('hex');
+
+async function verifyUnique(name, config, rclient) {
+  const cronId = config.port || 3000 + md5(name);
+  // one per node check
+  const key = `cron:unique:${cronId}`;
+  const unique = await rclient.setnx(key, 1);
+  const ttl = await rclient.ttl(key);
+  if (!unique && ttl !== -1) {
+    return false;
+  }
+  await rclient.expire(key, 20);
+  return true;
+}
+
+const intervalStringMs = {
+  everyMin: 1000 * 60,
+  tenMin: 1000 * 60 * 10,
+  everyHour: 1000 * 60 * 60,
+  isHalfday: 1000 * 60 * 60 * 12,
+  dailyHour: 1000 * 60 * 60 * 24,
+};
+
+const interval2ms = {
+  string: (interval) => {
+    const date = new Date();
+    const intervarSplit = interval.match(/^(\*{2}|(\*)?(\d{1,2})):(\*(\d)|(\d{2}))/);
+    if (!intervarSplit) {
+      throw new Error(`interval ${interval} not suported`);
+    }
+    const [, , isHalfday, dailyHour, , tenMin, HourlyMin] = intervarSplit;
+    const intervalMs = (isHalfday && intervalStringMs.isHalfday)
+      || (dailyHour && intervalStringMs.dailyHour)
+      || (tenMin && intervalStringMs.tenMin)
+      || intervalStringMs.everyHour;
+    const offsetDay = ((+dailyHour || 0) * 60 + (+tenMin || +HourlyMin)) * 60 * 1000;
+    const offsetCur = (date - date.getTimezoneOffset() * 1000 * 60) % intervalMs;
+    const waitMs = (offsetDay - offsetCur + intervalMs) % intervalMs;
+    return [waitMs, intervalMs];
+  },
+  number: (interval) => {
+    const date = new Date();
+    const intervalMs = interval * 1000;
+    const dateWithTZ = date - date.getTimezoneOffset() * 1000 * 60;
+    const offsetCur = dateWithTZ % intervalMs;
+    // start every cron within 1 hour
+    const sixtyMinutesStartMs = 3600000;
+    const waitMs = (intervalMs - offsetCur) % sixtyMinutesStartMs;
+    return [waitMs, intervalMs];
+  },
+};
+
+async function runCron({
+  pg, funcs, func, name, rclient, log,
+}) {
+  const unique = await verifyUnique(name, funcs.config, rclient);
+
+  if (!unique) return;
+  const db = pg.options.database;
+  log.debug(`cron.${name}`, 1, db);
+  try {
+    const data = await func({ pg, funcs, log });
+    log.debug('cron', { db, name, result: data });
+    log.info('cron', { db, name, result: data });
+  }
+  catch (err) {
+    log.debug('cron', { db, name, error: err.toString() });
+    log.error('cron', { db, name, error: err.toString() });
+  }
+}
+
+/**
+ * interval:
+ * -   02:54 - every day
+ * -    2:03 - every day
+ * -   *1:43 - 2 times a day
+ * -  *12:03 - 2 times a day
+ * -   **:54 - every hour
+ * -   **:*3 - every 10 minutes
+ * -      60 - every minute
+ * - 10 * 60 - every 10 minutes
+ */
+
+export default async function addCron(func, interval, fastify) {
+  if (!fastify) {
+    throw new Error('not enough params: fastify');
+  }
+
+  const { config = {}, log } = fastify;
+  const { time = {}, disabled = [] } = config.cron || {};
+  const pg = getPG();
+  const rclient = getRedis();
+
+  const name = func.name || func.toString().split('/').at(-1).split('\'')[0];
+
+  // if (!config.isServer) return;
+
+  if (disabled.includes(name)) {
+    log.debug('cron', { name, message: 'cron disabled' });
+    return;
+  }
+
+  cronList[name] = func;
+
+  const userInterval = time[name] || interval;
+  const [waitMs, intervalMs] = interval2ms[typeof interval](userInterval);
+
+  if (intervalMs < 1000) {
+    log.warn('cron', { name, error: `interval ${interval} to small` });
+    return;
+  }
+
+  // setTimeout to w8 for the time to start
+  setTimeout(() => {
+    runCron({
+      pg, funcs: fastify, func, name, rclient, log,
+    });
+    // interval
+    setInterval(() => {
+      runCron({
+        pg, funcs: fastify, func, name, rclient, log,
+      });
+    }, intervalMs);
+  }, waitMs);
+}

package/cron/index.js

@@ -0,0 +1,10 @@
+import cronApi from './controllers/cronApi.js';
+import addCron from './funcs/addCron.js';
+
+async function plugin(fastify, config = {}) {
+  const prefix = config.prefix || '/api';
+  fastify.decorate('addCron', addCron);
+  fastify.get(`${prefix}/cron/:name`, {}, cronApi);
+}
+
+export default plugin;

package/index.js

@@ -1,87 +1,89 @@
-import path from 'path';
-import { existsSync, readdirSync, readFileSync } from 'fs';
-
-import fp from 'fastify-plugin';
-import config from './config.js';
-// import rclient from './redis/client.js';
-
-import redisPlugin from './redis/index.js';
-import pgPlugin from './pg/index.js';
-import tablePlugin from './table/index.js';
-import notificationPlugin from './notification/index.js';
-import widgetPlugin from './widget/index.js';
-import crudPlugin from './crud/index.js';
-import policyPlugin from './policy/index.js';
-import utilPlugin from './util/index.js';
-
-import pgClients from './pg/pgClients.js';
-
-import execMigrations from './migration/exec.migrations.js';
-
-async function plugin(fastify, opt) {
-  // console.log(opt);
-  config.pg = opt.pg;
-  config.redis = opt.redis;
-  config.root = opt.root;
-  config.mapServerRoot = opt.mapServerRoot;
-
-  // independent npm start / unit test
-  if (!fastify.config) {
-    fastify.decorate('config', config);
-  }
-
-  fastify.register(import('@opengis/fastify-hb'));
-  fastify.decorate('getFolder', (req, type = 'server') => {
-    if (!['server', 'local'].includes(type)) throw new Error('params type is invalid');
-    const types = { local: req.root, server: req.mapServerRoot };
-    const filepath = path.posix.join(types[type] || '/data/local', req.folder || config.folder || '');
-    return filepath;
-  });
-
-  fastify.addHook('onListen', async () => {
-    const { client } = pgClients;
-    if (client?.pk?.['crm.cls']) {
-      const clsDir = path.join(process.cwd(), 'server/templates/cls');
-      const files = existsSync(clsDir) ? readdirSync(clsDir) : [];
-      if (files.length) {
-        const res = await Promise.all(files.map(async (filename) => {
-          const filepath = path.join(clsDir, filename);
-          const data = JSON.parse(readFileSync(filepath));
-          return { name: path.parse(filename).name, data };
-        }));
-        await client.query('truncate table crm.cls');
-        const { rows } = await client.query(`insert into crm.cls(name, type) 
-        select value->>'name', 'json' from json_array_elements($1) returning cls_id as id, name`, [JSON.stringify(res).replace(/'/g, "''")]);
-        rows.forEach((row) => Object.assign(row, { data: res.find((cls) => row.name === cls.name)?.data }));
-        const sql = `insert into crm.cls(code, name, parent)
-        select json_array_elements(value->'data')->>'id', json_array_elements(value->'data')->>'text', value->>'name' from json_array_elements($1)`;
-        await client.query(sql, [JSON.stringify(rows).replace(/'/g, "''")]);
-      }
-    }
-    // call from another repo / project
-    fastify.execMigrations = execMigrations;
-    // execute core migrations
-    await fastify.execMigrations();
-  });
-  if (!fastify.funcs) {
-    fastify.addHook('onRequest', async (req) => {
-      req.funcs = fastify;
-      if (!req.user && req.session?.passport?.user) {
-        const { user } = req.session?.passport || {};
-        req.user = user;
-      }
-    });
-    // fastify.decorateRequest('funcs', fastify);
-  }
-
-  policyPlugin(fastify);
-  redisPlugin(fastify);
-  await pgPlugin(fastify, opt);
-  tablePlugin(fastify, opt);
-  crudPlugin(fastify, opt);
-  notificationPlugin(fastify, opt);
-  widgetPlugin(fastify, opt);
-  utilPlugin(fastify, opt);
-}
-export default fp(plugin);
-// export { rclient };
+import path from 'path';
+import { existsSync, readdirSync, readFileSync } from 'fs';
+
+import fp from 'fastify-plugin';
+import config from './config.js';
+// import rclient from './redis/client.js';
+
+import redisPlugin from './redis/index.js';
+import pgPlugin from './pg/index.js';
+import tablePlugin from './table/index.js';
+import notificationPlugin from './notification/index.js';
+import widgetPlugin from './widget/index.js';
+import crudPlugin from './crud/index.js';
+import policyPlugin from './policy/index.js';
+import utilPlugin from './util/index.js';
+import cronPlugin from './cron/index.js';
+
+import pgClients from './pg/pgClients.js';
+
+import execMigrations from './migration/exec.migrations.js';
+
+async function plugin(fastify, opt) {
+  // console.log(opt);
+  config.pg = opt.pg;
+  config.redis = opt.redis;
+  config.root = opt.root;
+  config.mapServerRoot = opt.mapServerRoot;
+
+  // independent npm start / unit test
+  if (!fastify.config) {
+    fastify.decorate('config', config);
+  }
+
+  fastify.register(import('@opengis/fastify-hb'));
+  fastify.decorate('getFolder', (req, type = 'server') => {
+    if (!['server', 'local'].includes(type)) throw new Error('params type is invalid');
+    const types = { local: req.root, server: req.mapServerRoot };
+    const filepath = path.posix.join(types[type] || '/data/local', req.folder || config.folder || '');
+    return filepath;
+  });
+
+  fastify.addHook('onListen', async () => {
+    const { client } = pgClients;
+    if (client?.pk?.['crm.cls']) {
+      const clsDir = path.join(process.cwd(), 'server/templates/cls');
+      const files = existsSync(clsDir) ? readdirSync(clsDir) : [];
+      if (files.length) {
+        const res = await Promise.all(files.map(async (filename) => {
+          const filepath = path.join(clsDir, filename);
+          const data = JSON.parse(readFileSync(filepath));
+          return { name: path.parse(filename).name, data };
+        }));
+        await client.query('truncate table crm.cls');
+        const { rows } = await client.query(`insert into crm.cls(name, type) 
+        select value->>'name', 'json' from json_array_elements($1) returning cls_id as id, name`, [JSON.stringify(res).replace(/'/g, "''")]);
+        rows.forEach((row) => Object.assign(row, { data: res.find((cls) => row.name === cls.name)?.data }));
+        const sql = `insert into crm.cls(code, name, parent)
+        select json_array_elements(value->'data')->>'id', json_array_elements(value->'data')->>'text', value->>'name' from json_array_elements($1)`;
+        await client.query(sql, [JSON.stringify(rows).replace(/'/g, "''")]);
+      }
+    }
+    // call from another repo / project
+    fastify.execMigrations = execMigrations;
+    // execute core migrations
+    await fastify.execMigrations();
+  });
+  if (!fastify.funcs) {
+    fastify.addHook('onRequest', async (req) => {
+      req.funcs = fastify;
+      if (!req.user && req.session?.passport?.user) {
+        const { user } = req.session?.passport || {};
+        req.user = user;
+      }
+    });
+    // fastify.decorateRequest('funcs', fastify);
+  }
+
+  policyPlugin(fastify);
+  redisPlugin(fastify);
+  await pgPlugin(fastify, opt);
+  tablePlugin(fastify, opt);
+  crudPlugin(fastify, opt);
+  notificationPlugin(fastify, opt);
+  widgetPlugin(fastify, opt);
+  utilPlugin(fastify, opt);
+  cronPlugin(fastify, opt);
+}
+export default fp(plugin);
+// export { rclient };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengis/fastify-table",
-  "version": "1.0.68",
+  "version": "1.0.70",
   "type": "module",
   "description": "core-plugins",
   "main": "index.js",

package/table/controllers/data.js

@@ -3,6 +3,7 @@ import getFilterSQL from '../funcs/getFilterSQL/index.js';
 import getMeta from '../../pg/funcs/getMeta.js';
 import metaFormat from '../funcs/metaFormat/index.js';
 import setToken from '../../crud/funcs/setToken.js';
+import gisIRColumn from './utils/gisIRColumn.js';
 
 const maxLimit = 100;
 export default async function dataAPI(req) {
@@ -28,6 +29,12 @@ export default async function dataAPI(req) {
   const cardSqlFiltered = opt?.id || params.id ? (cardSql?.filter?.((el) => !el?.disabled && el?.name && el?.sql?.replace) || []) : [];
   const cardSqlTable = cardSqlFiltered.length ? cardSqlFiltered.map((el, i) => ` left join lateral (select json_agg(row_to_json(q)) as ${el.name} from (${el.sql})q) ct${i}  on 1=1 `).join('') || '' : '';
 
+  if (params.id && columnList.includes(params.id)) {
+    return gisIRColumn({
+      pg, funcs, layer: params.table, column: params.id, sql: query.sql,
+    });
+  }
+
   const fData = query.filter ? await getFilterSQL({
     filter: query.filter,
     table: params.table,

package/table/controllers/utils/gisIRColumn.js

@@ -0,0 +1,68 @@
+import pgClients from '../../../pg/pgClients.js';
+
+import getTemplate from './getTemplate.js';
+import getSelect from './getSelect.js';
+import getFilterSQL from '../../funcs/getFilterSQL/index.js';
+
+export default async function gisIRColumn({
+  pg = pgClients.client, funcs = {}, layer, column, sql, query = '1=1',
+}) {
+  const time = Date.now();
+
+  const { config = {} } = funcs;
+
+  const sel = await getSelect(query.cls || column);
+
+  const body = await getTemplate('table', layer);
+  const fData = await getFilterSQL({
+    table: body?.table || layer, query: body?.query,
+  });
+
+  const { tlist } = await pg.one(`select array_agg((select nspname from pg_namespace where oid=relnamespace)||'.'||relname) tlist from pg_class 
+  where relkind in ('r','v','m')`);
+
+  const tableName = body?.table || layer;
+  if (!tlist.includes(body?.table || layer)) return { error: `table not found: ${tableName}`, status: 400 };
+
+  // eslint-disable-next-line max-len
+  const { fields } = await pg.query(`select * from (${fData?.optimizedSQL || `select * from ${body?.table || layer}`})q limit 0`);
+
+  const col = fields.find((el) => el.name === column);
+
+  if (!col) return { status: 404, message: 'not found' };
+  const colField = pg.pgType[col.dataTypeID]?.includes('[]') ? `unnest(${column})` : column;
+
+  const q = `select ${colField} as id, count(*)::int from ${tableName} t where ${body?.query || 'true'} 
+  group by ${colField} order by count desc limit 15`;
+
+  if (sql) return q;
+
+  if (!body?.columns?.length) {
+    const { rows } = await pg.query(q);
+    if (sel?.arr?.length) {
+      rows.forEach((el) => {
+        const data = sel?.find((item) => item.id?.toString() === el.id?.toString());
+        Object.assign(el, data || {});
+      });
+    }
+    return {
+      count: rows?.reduce((acc, el) => acc + el.count, 0),
+      sql: config.local ? q : undefined,
+      rows,
+    };
+  }
+
+  const { rows } = await pg.query(q);
+  const cls = query.cls || body?.columns?.find((el) => el.name === column)?.data || col.data || col.option;
+  const select = await getSelect(cls, { val: rows.map((el) => el.id), ar: 1 });
+  rows.forEach((el) => {
+    const data = select?.arr ? select.arr?.find((item) => item.id?.toString() === el.id?.toString()) : undefined;
+    Object.assign(el, data || {});
+  });
+  return {
+    time: Date.now() - time,
+    count: rows.reduce((acc, el) => acc + el.count, 0),
+    sql: config.local ? q : undefined,
+    rows,
+  };
+}