@opengis/fastify-table 1.2.60 → 1.2.62

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengis/fastify-table",
-  "version": "1.2.60",
+  "version": "1.2.62",
   "type": "module",
   "description": "core-plugins",
   "keywords": [

package/server/plugins/pg/funcs/getPG.js

@@ -24,6 +24,7 @@ function getPG(param) {
     host: host || config.pg?.host,
     port: port || config.pg?.port,
     database: db || database || config.pg?.db || config.pg?.database,
+    statement_timeout: config.pg?.statement_timeout || 10000,
   };
 
   pgClients[name] = new pg.Pool(dbConfig);

package/server/plugins/pg/funcs/getPGAsync.js

@@ -25,6 +25,7 @@ async function getPGAsync(param) {
     host: host || config.pg?.host,
     port: port || config.pg?.port,
     database: db || database || config.pg?.db || config.pg?.database,
+    statement_timeout: config.pg?.statement_timeout || 10000,
   };
 
   pgClients[name] = new pg.Pool(dbConfig);

package/server/plugins/pg/funcs/init.js

@@ -1,6 +1,8 @@
 import { createHash } from 'node:crypto';
 
+import config from '../../../../config.js';
 import getRedis from '../../redis/funcs/getRedis.js';
+import logger from '../../logger/getLogger.js';
 
 const rclient = getRedis({ db: 0 });
 
@@ -18,20 +20,35 @@ async function init(client) {
       from pg_class where relkind in ('r','v')`);
   const relkinds = rows.reduce((acc, curr) => Object.assign(acc, { [curr.tname]: curr.relkind }), {});
 
-  async function one(query, param = {}) {
-    const data = await client.query(query, Array.isArray(param) ? param : param.args || []);
+  // const { queryOriginal = client.query } = client;
+
+  async function query(q, args = []) {
+    try {
+      const data = await client.query(q, args);
+      return data;
+    } catch (err) {
+      if (err.message === 'canceling statement due to statement timeout') {
+        logger.file('timeout/query', { q, stack: err.stack });
+        return { rows: [], timeout: true };
+      }
+      throw new Error(err);
+    }
+  }
+
+  async function one(q, param = {}) {
+    const data = await query(q, Array.isArray(param) ? param : param.args || []);
     const result = ((Array.isArray(data) ? data.pop() : data)?.rows || [])[0] || {};
     return result;
   }
 
-  async function queryNotice(query, args = [], cb = () => { }) {
+  async function queryNotice(q, args = [], cb = () => { }) {
     const clientCb = await client.connect();
     clientCb.on('notice', (e) => {
       cb(e.message);
     });
     let result;
     try {
-      result = await clientCb.query(query, args);
+      result = await clientCb.query(q, args);
       clientCb.end();
     }
     catch (err) {
@@ -42,16 +59,16 @@ async function init(client) {
     return result;
   }
 
-  async function queryCache(query, param = {}) {
+  async function queryCache(q, param = {}) {
     const { table, args = [], time = 15 } = param;
-    const seconds = typeof time !== 'number' || time < 0 ? 15 : time * 60;
+    const seconds = typeof time !== 'number' || time < 0 ? 15 : (config.local ? 0 : time * 60);
 
     // CRUD table state
     const keyCacheTable = `pg:${table}:crud`;
     const crudInc = table ? await rclient.get(keyCacheTable) || 0 : 0;
 
     //
-    const hash = createHash('sha1').update([query, JSON.stringify(args)].join()).digest('base64');
+    const hash = createHash('sha1').update([q, JSON.stringify(args)].join()).digest('base64');
     const keyCache = `pg:${hash}:${crudInc}`;
 
     const cacheData = await rclient.get(keyCache);
@@ -61,7 +78,7 @@ async function init(client) {
       return JSON.parse(cacheData);
     }
 
-    const data = await client.query(query, args || []);
+    const data = await query(q, args || []);
 
     if (seconds > 0) {
       rclient.set(keyCache, JSON.stringify(data), 'EX', seconds);
@@ -71,7 +88,7 @@ async function init(client) {
   }
 
   Object.assign(client, {
-    one, pgType, pk, tlist, relkinds, queryCache, queryNotice,
+    one, pgType, pk, tlist, relkinds, queryCache, queryNotice, /* queryOriginal, */
   });
 }
 

package/server/plugins/pg/pgClients.js

@@ -1,21 +1,22 @@
-import pg from 'pg';
-
-import config from '../../../config.js';
-import init from './funcs/init.js';
-
-const pgClients = {};
-if (config.pg) {
-  const client = new pg.Pool({
-    host: config.pg?.host || '127.0.0.1',
-    port: config.pg?.port || 5432,
-    database: config.pg?.database || 'postgres',
-    user: config.pg?.user || 'postgres',
-    password: config.pg?.password || 'postgres',
-  });
-  client.init = async () => {
-    await init(client);
-  };
-  client.init();
-  pgClients.client = client;
-}
-export default pgClients;
+import pg from 'pg';
+
+import config from '../../../config.js';
+import init from './funcs/init.js';
+
+const pgClients = {};
+if (config.pg) {
+  const client = new pg.Pool({
+    host: config.pg?.host || '127.0.0.1',
+    port: config.pg?.port || 5432,
+    database: config.pg?.database || 'postgres',
+    user: config.pg?.user || 'postgres',
+    password: config.pg?.password || 'postgres',
+    statement_timeout: config.pg?.statement_timeout || 10000,
+  });
+  client.init = async () => {
+    await init(client);
+  };
+  client.init();
+  pgClients.client = client;
+}
+export default pgClients;

package/server/routes/table/controllers/filter.js

@@ -32,14 +32,14 @@ export default async function filterAPI(req) {
   const filters = (loadTable?.filter_list || loadTable?.filters || loadTable?.filterList || []).concat(loadTable?.filterSql || []);
 
   // admin.custom_column - user filter NA-165
-  const { rows: properties = [] } = await pg.query(`select column_id, name, title, format, data from admin.custom_column where entity=$1 and uid=$2 and filter`, [params.name, user?.uid]);
+  const { rows: properties = [] } = await pg.query(`select column_id, name, title, format, data from admin.custom_column where entity=$1 and uid=$2 and filter`, [params.table, user?.uid]);
   properties.forEach((row) => filters.push({ id: row.name, name: row.name, ua: row.title, type: row.format, data: row.data }));
 
   // KRYVYIRIH-231
   autoIndex({ table: loadTable.table, columns: filters.filter((el) => columns?.find?.((item) => item?.name === el.name)) })
     .catch(err => {
       console.error(err.toString());
-      logger.file('autoindex/error', { name: params?.name, error: err.toString(), stack: err.stack });
+      logger.file('autoindex/error', { name: params?.table, error: err.toString(), stack: err.stack });
     });
 
   filters?.forEach?.(el => Object.assign(el, { id: el.id || el.name }));
@@ -59,9 +59,17 @@ export default async function filterAPI(req) {
       Object.assign(el, { options });
     }
 
-    const countArr = pg.pgType[dataTypeID]?.includes('[]')
-      ? await pg.queryCache(`select unnest(${el.id})::text as id,count(*) from (${optimizedSQL})q group by unnest(${el.id})`, { table: loadTable.table, time: 5 })
-      : await pg.queryCache(`select ${el.id}::text as id,count(*) from (${optimizedSQL})q group by ${el.id}`, { table: loadTable.table, time: 5 });
+    const q = pg.pgType[dataTypeID]?.includes('[]')
+      ? `select unnest(${el.id})::text as id,count(*) from (${optimizedSQL})q group by unnest(${el.id}) limit 100`
+      : `select ${el.id}::text as id,count(*) from (${optimizedSQL})q group by ${el.id} limit 100`;
+
+    const countArr = await pg.queryCache(q, { table: loadTable.table });
+    if (countArr.timeout) {
+      Object.assign(el, { timeout: countArr.timeout });
+      console.log('timeout filter', params.table, el.id);
+      logger.file('timeout/filter', { table: params.table, type: 'cls', filter: el.id });
+    }
+
     const ids = countArr.rows.map(el1 => el1.id);
 
     const clsData = await getSelectVal({ pg, values: ids, name: el.data });
@@ -71,10 +79,15 @@ export default async function filterAPI(req) {
       return { ...cel, ...data };
     });
     Object.assign(el, { options });
+
   }));
 
   const q = ((loadTable?.filterState || []).concat(loadTable?.filterCustom || [])).filter((el) => el.name && el.sql).map((el) => `select count(*), '${el.name}' as name from (${optimizedSQL})q where ${el.sql}`).join(' union all ');
-  const { rows = [] } = q ? await pg.query(q) : {};
+
+
+  const { rows = [], timeout: timeout1 } = q ? await pg.queryCache(q) : {};
+  if (timeout1) logger.file('timeout/filter', { table: params.table, type: 'state/custom' });
+
   if (rows?.length) {
     ((loadTable?.filterState || []).concat(loadTable?.filterCustom || [])).filter((el) => el.name && el.sql).forEach((el) => {
       const { count } = rows.find((row) => row.name === el.name) || {};
@@ -82,6 +95,7 @@ export default async function filterAPI(req) {
     });
   }
 
+
   const sqlList = loadTable?.sql
     ?.filter((el) => !el.disabled && el?.sql?.replace)
     ?.map((el, i) => ` left join lateral (${el.filter ? el.sql.replace(/limit 1/ig, '') : el.sql}) as ${el.name || `t${i + 1}`} on 1=1 `)
@@ -89,19 +103,31 @@ export default async function filterAPI(req) {
 
   // percentile_cont - alternative
   await Promise.all(filters.filter((el) => el.name && el.type === 'Range' && fields?.find?.((item) => item?.name === el.name)).map(async (el) => {
+
     const data = await pg.queryCache(`select array[
         min(${el.name}),
         percentile_disc(0.25) within group (order by ${el.name}),
         percentile_disc(0.5) within group (order by ${el.name}),
         percentile_disc(0.75) within group (order by ${el.name}),
         max(${el.name})
-    ] as range from ${loadTable.table} ${sqlList && false ? ` t ${sqlList}` : ''} where ${loadTable.query || '1=1'}`, { table: loadTable.table }).then(el => el.rows?.[0]?.range);
+    ] as range from ${loadTable.table} ${sqlList && false ? ` t ${sqlList}` : ''} where ${loadTable.query || '1=1'}`,
+      { table: loadTable.table }).then(el => {
+        if (el.timeout) {
+          logger.file('timeout/filter', { table: params.table, type: 'Range', filter: el.name });
+          return el;
+        }
+        return el.rows?.[0]?.range;
+      });
     Object.assign(el, { data });
+
   }));
 
   const sqlFilters = (loadTable?.filterCustom || []).filter((el) => el.name && el.sql);
   const q1 = sqlFilters.map((el) => `select count(*), '${el.name}' as name from ${loadTable.table} where ${loadTable.query || '1=1'} and ${el.sql}`).join(' union all ');
-  const { rows: sqlRows = [] } = q1 ? await pg.queryCache(q1, { table: loadTable.table }) : {};
+
+  const { rows: sqlRows = [], timeout } = q1 ? await pg.queryCache(q1, { table: loadTable.table }) : {};
+  if (timeout) logger.file('timeout/filter', { table: params.table, type: 'sqlFilters' });
+
   if (sqlRows?.length) {
     sqlFilters.forEach((el) => {
       const { count } = sqlRows.find((row) => row.name === el.name) || {};