@opengis/fastify-table 1.2.61 → 1.2.62

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengis/fastify-table",
-  "version": "1.2.61",
+  "version": "1.2.62",
   "type": "module",
   "description": "core-plugins",
   "keywords": [

package/server/plugins/pg/funcs/init.js

@@ -1,6 +1,8 @@
 import { createHash } from 'node:crypto';
 
+import config from '../../../../config.js';
 import getRedis from '../../redis/funcs/getRedis.js';
+import logger from '../../logger/getLogger.js';
 
 const rclient = getRedis({ db: 0 });
 
@@ -18,20 +20,35 @@ async function init(client) {
       from pg_class where relkind in ('r','v')`);
   const relkinds = rows.reduce((acc, curr) => Object.assign(acc, { [curr.tname]: curr.relkind }), {});
 
-  async function one(query, param = {}) {
-    const data = await client.query(query, Array.isArray(param) ? param : param.args || []);
+  // const { queryOriginal = client.query } = client;
+
+  async function query(q, args = []) {
+    try {
+      const data = await client.query(q, args);
+      return data;
+    } catch (err) {
+      if (err.message === 'canceling statement due to statement timeout') {
+        logger.file('timeout/query', { q, stack: err.stack });
+        return { rows: [], timeout: true };
+      }
+      throw new Error(err);
+    }
+  }
+
+  async function one(q, param = {}) {
+    const data = await query(q, Array.isArray(param) ? param : param.args || []);
     const result = ((Array.isArray(data) ? data.pop() : data)?.rows || [])[0] || {};
     return result;
   }
 
-  async function queryNotice(query, args = [], cb = () => { }) {
+  async function queryNotice(q, args = [], cb = () => { }) {
     const clientCb = await client.connect();
     clientCb.on('notice', (e) => {
       cb(e.message);
     });
     let result;
     try {
-      result = await clientCb.query(query, args);
+      result = await clientCb.query(q, args);
       clientCb.end();
     }
     catch (err) {
@@ -42,16 +59,16 @@ async function init(client) {
     return result;
   }
 
-  async function queryCache(query, param = {}) {
+  async function queryCache(q, param = {}) {
     const { table, args = [], time = 15 } = param;
-    const seconds = typeof time !== 'number' || time < 0 ? 15 : time * 60;
+    const seconds = typeof time !== 'number' || time < 0 ? 15 : (config.local ? 0 : time * 60);
 
     // CRUD table state
     const keyCacheTable = `pg:${table}:crud`;
     const crudInc = table ? await rclient.get(keyCacheTable) || 0 : 0;
 
     //
-    const hash = createHash('sha1').update([query, JSON.stringify(args)].join()).digest('base64');
+    const hash = createHash('sha1').update([q, JSON.stringify(args)].join()).digest('base64');
     const keyCache = `pg:${hash}:${crudInc}`;
 
     const cacheData = await rclient.get(keyCache);
@@ -61,7 +78,7 @@ async function init(client) {
       return JSON.parse(cacheData);
     }
 
-    const data = await client.query(query, args || []);
+    const data = await query(q, args || []);
 
     if (seconds > 0) {
       rclient.set(keyCache, JSON.stringify(data), 'EX', seconds);
@@ -71,7 +88,7 @@ async function init(client) {
   }
 
   Object.assign(client, {
-    one, pgType, pk, tlist, relkinds, queryCache, queryNotice,
+    one, pgType, pk, tlist, relkinds, queryCache, queryNotice, /* queryOriginal, */
   });
 }
 

package/server/plugins/pg/pgClients.js

@@ -1,21 +1,22 @@
-import pg from 'pg';
-
-import config from '../../../config.js';
-import init from './funcs/init.js';
-
-const pgClients = {};
-if (config.pg) {
-  const client = new pg.Pool({
-    host: config.pg?.host || '127.0.0.1',
-    port: config.pg?.port || 5432,
-    database: config.pg?.database || 'postgres',
-    user: config.pg?.user || 'postgres',
-    password: config.pg?.password || 'postgres',
-  });
-  client.init = async () => {
-    await init(client);
-  };
-  client.init();
-  pgClients.client = client;
-}
-export default pgClients;
+import pg from 'pg';
+
+import config from '../../../config.js';
+import init from './funcs/init.js';
+
+const pgClients = {};
+if (config.pg) {
+  const client = new pg.Pool({
+    host: config.pg?.host || '127.0.0.1',
+    port: config.pg?.port || 5432,
+    database: config.pg?.database || 'postgres',
+    user: config.pg?.user || 'postgres',
+    password: config.pg?.password || 'postgres',
+    statement_timeout: config.pg?.statement_timeout || 10000,
+  });
+  client.init = async () => {
+    await init(client);
+  };
+  client.init();
+  pgClients.client = client;
+}
+export default pgClients;

package/server/routes/table/controllers/filter.js

@@ -1,4 +1,4 @@
-import { config, logger, autoIndex, getSelect, getFilterSQL, getTemplate, getSelectVal, pgClients } from '../../../../utils.js';
+import { logger, autoIndex, getSelect, getFilterSQL, getTemplate, getSelectVal, pgClients } from '../../../../utils.js';
 
 export default async function filterAPI(req) {
   const time = Date.now();
@@ -60,44 +60,42 @@ export default async function filterAPI(req) {
     }
 
     const q = pg.pgType[dataTypeID]?.includes('[]')
-      ? `select unnest(${el.id})::text as id,count(*) from (${optimizedSQL})q group by unnest(${el.id}) limit 500`
-      : `select ${el.id}::text as id,count(*) from (${optimizedSQL})q group by ${el.id} limit 500`;
+      ? `select unnest(${el.id})::text as id,count(*) from (${optimizedSQL})q group by unnest(${el.id}) limit 100`
+      : `select ${el.id}::text as id,count(*) from (${optimizedSQL})q group by ${el.id} limit 100`;
+
+    const countArr = await pg.queryCache(q, { table: loadTable.table });
+    if (countArr.timeout) {
+      Object.assign(el, { timeout: countArr.timeout });
+      console.log('timeout filter', params.table, el.id);
+      logger.file('timeout/filter', { table: params.table, type: 'cls', filter: el.id });
+    }
 
-    try {
-      if (config.debug) console.log('filter options start', el.id, 'array', pg.pgType[dataTypeID]?.includes('[]'));
-      const countArr = await pg.queryCache(q, { table: loadTable.table, time: config.local ? 0 : undefined });
-      if (config.debug) console.log('filter options finish', el.id, 'array', pg.pgType[dataTypeID]?.includes('[]'));
+    const ids = countArr.rows.map(el1 => el1.id);
 
-      const ids = countArr.rows.map(el1 => el1.id);
+    const clsData = await getSelectVal({ pg, values: ids, name: el.data });
 
-      const clsData = await getSelectVal({ pg, values: ids, name: el.data });
+    const options = countArr.rows.map(cel => {
+      const data = cls?.arr?.find(c => c.id === cel.id) || { text: clsData[cel.id] };
+      return { ...cel, ...data };
+    });
+    Object.assign(el, { options });
 
-      const options = countArr.rows.map(cel => {
-        const data = cls?.arr?.find(c => c.id === cel.id) || { text: clsData[cel.id] };
-        return { ...cel, ...data };
-      });
-      Object.assign(el, { options });
-    } catch (err) {
-      Object.assign(el, { timeout: true });
-      logger.file('timeout', { table: params?.table, filter: el.id, stack: err.stack });
-      console.log('filter query timeout', params.table, el.id);
-    }
   }));
 
   const q = ((loadTable?.filterState || []).concat(loadTable?.filterCustom || [])).filter((el) => el.name && el.sql).map((el) => `select count(*), '${el.name}' as name from (${optimizedSQL})q where ${el.sql}`).join(' union all ');
 
-  try {
-    const { rows = [] } = q ? await pg.query(q) : {};
-    if (rows?.length) {
-      ((loadTable?.filterState || []).concat(loadTable?.filterCustom || [])).filter((el) => el.name && el.sql).forEach((el) => {
-        const { count } = rows.find((row) => row.name === el.name) || {};
-        Object.assign(el, { count, sql: undefined });
-      });
-    }
-  } catch (err) {
-    logger.file('timeout', { table: params?.table, filter: 'sql', stack: err.stack });
+
+  const { rows = [], timeout: timeout1 } = q ? await pg.queryCache(q) : {};
+  if (timeout1) logger.file('timeout/filter', { table: params.table, type: 'state/custom' });
+
+  if (rows?.length) {
+    ((loadTable?.filterState || []).concat(loadTable?.filterCustom || [])).filter((el) => el.name && el.sql).forEach((el) => {
+      const { count } = rows.find((row) => row.name === el.name) || {};
+      Object.assign(el, { count, sql: undefined });
+    });
   }
 
+
   const sqlList = loadTable?.sql
     ?.filter((el) => !el.disabled && el?.sql?.replace)
     ?.map((el, i) => ` left join lateral (${el.filter ? el.sql.replace(/limit 1/ig, '') : el.sql}) as ${el.name || `t${i + 1}`} on 1=1 `)
@@ -105,35 +103,36 @@ export default async function filterAPI(req) {
 
   // percentile_cont - alternative
   await Promise.all(filters.filter((el) => el.name && el.type === 'Range' && fields?.find?.((item) => item?.name === el.name)).map(async (el) => {
-    try {
-      const data = await pg.queryCache(`select array[
+
+    const data = await pg.queryCache(`select array[
         min(${el.name}),
         percentile_disc(0.25) within group (order by ${el.name}),
         percentile_disc(0.5) within group (order by ${el.name}),
         percentile_disc(0.75) within group (order by ${el.name}),
         max(${el.name})
     ] as range from ${loadTable.table} ${sqlList && false ? ` t ${sqlList}` : ''} where ${loadTable.query || '1=1'}`,
-        { table: loadTable.table, time: config.local ? 0 : undefined }).then(el => el.rows?.[0]?.range);
-      Object.assign(el, { data });
-    } catch (err) {
-      Object.assign(el, { timeout: 1 });
-      logger.file('timeout', { table: params?.table, filter: 'sql', stack: err.stack });
-    }
+      { table: loadTable.table }).then(el => {
+        if (el.timeout) {
+          logger.file('timeout/filter', { table: params.table, type: 'Range', filter: el.name });
+          return el;
+        }
+        return el.rows?.[0]?.range;
+      });
+    Object.assign(el, { data });
+
   }));
 
   const sqlFilters = (loadTable?.filterCustom || []).filter((el) => el.name && el.sql);
   const q1 = sqlFilters.map((el) => `select count(*), '${el.name}' as name from ${loadTable.table} where ${loadTable.query || '1=1'} and ${el.sql}`).join(' union all ');
 
-  try {
-    const { rows: sqlRows = [] } = q1 ? await pg.queryCache(q1, { table: loadTable.table, time: config.local ? 0 : undefined }) : {};
-    if (sqlRows?.length) {
-      sqlFilters.forEach((el) => {
-        const { count } = sqlRows.find((row) => row.name === el.name) || {};
-        Object.assign(el, { count, sql: undefined });
-      });
-    }
-  } catch (err) {
-    logger.file('timeout', { table: params?.table, filter: 'custom', stack: err.stack });
+  const { rows: sqlRows = [], timeout } = q1 ? await pg.queryCache(q1, { table: loadTable.table }) : {};
+  if (timeout) logger.file('timeout/filter', { table: params.table, type: 'sqlFilters' });
+
+  if (sqlRows?.length) {
+    sqlFilters.forEach((el) => {
+      const { count } = sqlRows.find((row) => row.name === el.name) || {};
+      Object.assign(el, { count, sql: undefined });
+    });
   }
 
   return {