@opengis/fastify-table 1.0.0

package/.eslintrc.cjs

@@ -0,0 +1,42 @@
+/* eslint-env node */
+
+module.exports = {
+  env: {
+    node: true,
+  },
+  root: true,
+  extends: [
+    'eslint:recommended',
+    'airbnb-base',
+
+  ],
+  rules: {
+    'brace-style': [2, 'stroustrup', { allowSingleLine: true }],
+    'vue/max-attributes-per-line': 0,
+    'vue/valid-v-for': 0,
+
+    // allow async-await
+    'generator-star-spacing': 'off',
+
+    // allow paren-less arrow functions
+    'arrow-parens': 0,
+    'one-var': 0,
+    'max-len': 0,
+    'import/first': 0,
+    'import/named': 2,
+    'import/namespace': 2,
+    'import/default': 2,
+    'import/export': 2,
+    'import/extensions': 0,
+    'no-console': ['warn', { allow: ['warn', 'error'] }],
+    'import/no-unresolved': 0,
+    'import/no-extraneous-dependencies': 0,
+    'linebreak-style': ['error', 'unix'],
+    // allow debugger during development
+    'no-debugger': process.env.NODE_ENV === 'production' ? 2 : 0,
+  },
+
+  parserOptions: {
+    ecmaVersion: 'latest',
+  },
+};

package/README.md

@@ -0,0 +1,26 @@
+# fastify-table
+
+[![NPM version](https://img.shields.io/npm/v/@opengis/fastify-table)](https://www.npmjs.com/package/@opengis/fastify-table)
+[![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat)](http://standardjs.com/)
+
+It standardizes the entire form building process, while taking care of everything from rendering to validation and processing:
+
+- pg
+- redis
+- crud
+
+## Install
+
+```bash
+npm i @opengis/fastify-table
+```
+
+## Usage
+
+```js
+fastify.register(import('@opengis/fastify-table'), config);
+```
+
+## Documenation
+
+For a detailed understanding fastify-table, its features, and how to use them, refer to our [Documentation](https://apidocs.softpro.ua/gis.storage/).

package/config.js

@@ -0,0 +1,3 @@
+// import pg from './pg'
+const config = { allTemplates: {} };
+export default config;

package/crud/controllers/deleteCrud.js

@@ -0,0 +1,10 @@
+export default async function deleteCrud(req) {
+  const { funcs } = req;
+  const { table, id } = req.params || {};
+  if (!table) return { status: 404, message: 'table is required' };
+
+  const { pk } = await funcs.getMeta(table);
+  const data = await funcs.pg.query(`delete from ${table} where ${pk}=$1`, [id]);
+
+  return { rowCount: data.rowCount };
+}

package/crud/controllers/insert.js

@@ -0,0 +1,9 @@
+import dataInsert from '../funcs/dataInsert.js';
+
+export default async function insert(req) {
+  const { table } = req.params || {};
+  if (!table) return { status: 404, message: 'table is required' };
+
+  const res = await dataInsert({ table, data: req.body });
+  return res;
+}

package/crud/controllers/update.js

@@ -0,0 +1,10 @@
+import dataUpdate from '../funcs/dataUpdate.js';
+
+export default async function update(req) {
+  const { table, id } = req.params || {};
+  if (!table) return { status: 404, message: 'table is required' };
+  if (!id) return { status: 404, message: 'id is required' };
+
+  const res = await dataUpdate({ table, id, data: req.body });
+  return res;
+}

package/crud/funcs/dataInsert.js

@@ -0,0 +1,23 @@
+import pgClients from '../../pg/pgClients.js';
+import getMeta from '../../pg/funcs/getMeta.js';
+
+export default async function dataInsert({ table, data, pg = pgClients.client }) {
+  if (!data) return null;
+  const { columns } = await getMeta(table);
+  if (!columns) return null;
+
+  const names = columns.map((el) => el.name);
+  const filterData = Object.keys(data)
+    .filter((el) => data[el] && names.includes(el)).map((el) => [el, data[el]]);
+
+  const insertQuery = `insert into ${table}
+
+    ( ${filterData?.map((key) => `"${key[0]}"`).join(',')}) 
+
+    values  (${filterData?.map((key, i) => `$${i + 1}`).join(',')}) 
+    
+    returning *`;
+  //  console.log(updateDataset);
+  const res = await pg.one(insertQuery, [...filterData.map((el) => (typeof el[1] === 'object' ? JSON.stringify(el[1]) : el[1]))]) || {};
+  return res;
+}

package/crud/funcs/dataUpdate.js

@@ -0,0 +1,18 @@
+import pgClients from '../../pg/pgClients.js';
+import getMeta from '../../pg/funcs/getMeta.js';
+
+export default async function dataUpdate({
+  table, id, data, pg = pgClients.client,
+}) {
+  const { columns, pk } = await getMeta(table);
+
+  const names = columns.map((el) => el.name);
+  const filterData = Object.keys(data)
+    .filter((el) => data[el] && names.includes(el)).map((el) => [el, data[el]]);
+
+  const updateQuery = `UPDATE ${table} SET ${filterData?.map((key, i) => `"${key[0]}"=$${i + 2}`).join(',')} 
+      WHERE ${pk} = $1 returning *`;
+  //  console.log(updateDataset);
+  const res = await pg.one(updateQuery, [id, ...filterData.map((el) => (typeof el[1] === 'object' ? JSON.stringify(el[1]) : el[1]))]) || {};
+  return res;
+}

package/crud/funcs/getOpt.js

@@ -0,0 +1,8 @@
+import rclient from '../../redis/client.js';
+
+export default async function getOpt(token) {
+  const key = `opt:${token}`;
+  const data = await rclient.get(key);
+  if (!data) return null;
+  return JSON.parse(data);
+}

package/crud/funcs/isFileExists.js

@@ -0,0 +1,13 @@
+import { access } from 'fs/promises';
+
+const isFileExists = async (filepath) => {
+  try {
+    await access(filepath);
+    return true;
+  }
+  catch (err) {
+    return false;
+  }
+};
+
+export default isFileExists;

package/crud/funcs/setOpt.js

@@ -0,0 +1,14 @@
+import { createHash } from 'crypto';
+import rclient from '../../redis/client.js';
+
+function md5(string) {
+  return createHash('md5').update(string).digest('hex');
+}
+
+export default async function setOpt(params) {
+  const token = Buffer.from(md5(typeof params === 'object' ? JSON.stringify(params) : params), 'hex').toString('base64').replace(/[+-=]+/g, '');
+  // const token = md5(params);
+  const key = `opt:${token}`;
+  await rclient.set(key, JSON.stringify(params), 'EX', 60 * 60);
+  return token;
+}

package/crud/index.js

@@ -0,0 +1,29 @@
+import fp from 'fastify-plugin';
+
+import getOPt from './funcs/getOpt.js';
+import setOpt from './funcs/setOpt.js';
+import isFileExists from './funcs/isFileExists.js';
+import dataUpdate from './funcs/dataUpdate.js';
+
+import update from './controllers/update.js';
+import insert from './controllers/insert.js';
+import deleteCrud from './controllers/deleteCrud.js';
+
+import config from '../config.js';
+
+const prefix = config.prefix || '/api';
+
+async function plugin(fastify) {
+  // funcs
+  fastify.decorate('setOpt', setOpt);
+  fastify.decorate('getOpt', getOPt);
+  fastify.decorate('dataUpdate', dataUpdate);
+  fastify.decorate('isFileExists', isFileExists);
+
+  // api
+  fastify.put(`${prefix}/crud/:table/:id`, {}, update);
+  fastify.delete(`${prefix}/crud/:table/:id`, {}, deleteCrud);
+  fastify.post(`${prefix}/crud/:table`, {}, insert);
+}
+
+export default plugin;

package/helper.js

@@ -0,0 +1,28 @@
+// This file contains code that we reuse
+// between our tests.
+import Fastify from 'fastify';
+import config from './test/confg.js';
+import appService from './index.js';
+
+import rclient from './redis/client.js';
+import pgClients from './pg/pgClients.js';
+
+// automatically build and tear down our instance
+async function build(t) {
+  // you can set all the options supported by the fastify CLI command
+  // const argv = [AppPath]
+  process.env.NODE_ENV = 'production';
+  const app = Fastify({ logger: false });
+  app.register(appService, config);
+  // close the app after we are done
+  t.after(() => {
+    // console.log('close app');
+    pgClients.client.end();
+    rclient.quit();
+    app.close();
+  });
+
+  return app;
+}
+
+export default build;

package/index.js

@@ -0,0 +1,22 @@
+// import pg from 'pg';
+import fp from 'fastify-plugin';
+import config from './config.js';
+// import rclient from './redis/client.js';
+
+import redisPlugin from './redis/index.js';
+import pgPlugin from './pg/index.js';
+import tablePlugin from './table/index.js';
+import crudPlugin from './crud/index.js';
+
+async function plugin(fastify, opt) {
+  // console.log(opt);
+  config.pg = opt.pg;
+  config.redis = opt.redis;
+
+  redisPlugin(fastify);
+  await pgPlugin(fastify, opt);
+  tablePlugin(fastify, opt);
+  crudPlugin(fastify, opt);
+}
+export default fp(plugin);
+// export { rclient };

package/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@opengis/fastify-table",
+  "version": "1.0.0",
+  "type": "module",
+  "description": "core-plugins",
+  "main": "index.js",
+  "scripts": {
+    "lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs,.ts,.tsx,.cts,.mts --fix --ignore-path .gitignore",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "dependencies": {
+    "ioredis": "^5.3.2",
+    "fastify": "^4.26.1",
+    "fastify-plugin": "^4.0.0",
+    "pg": "^8.11.3"
+  },
+  "devDependencies": {
+    "eslint": "^8.49.0",
+    "eslint-config-airbnb": "^19.0.4"
+  },
+  "author": "Softpro",
+  "license": "ISC"
+}

package/pg/funcs/autoIndex.js

@@ -0,0 +1,89 @@
+// const rclient = require('../../redis/client')
+import pgClient from '../pgClients.js';
+
+// subfunc
+const getTable = ({ table }) => table.toLowerCase().replace(/[\n\r]+/g, ' ').split(' from ')
+  .filter((el) => /^[a-z0-9_]+\.[a-z0-9_]+/.test(el))
+  .map((el) => el.split(/[ )]/)[0]);
+
+const loadedIndex = {};
+
+// main func
+async function autoIndex({ table, columns: filter, pg = pgClient.client }) {
+  if (!filter?.length || !table) return null;
+
+  const attrs = filter.map((el) => (el.name || el).replace(/'/g, '')).sort();
+  const types = filter.map((el) => (el.name ? el : { name: el }))
+    .reduce((p, el) => ({ ...p, [el.name]: el.type || '-' }), {});
+
+  const redisKey = `autoindex1:${table}:${attrs.join(';')}`;
+  const tableList = getTable({ table });
+  const existsCache = loadedIndex[redisKey];
+  if (existsCache) { return 'ok'; }
+
+  const tbs = tableList[0];
+  const [ns, tbl] = tableList[0].split('.');
+  const { rows: index } = await pg.query(`select * from pg_indexes where tablename = '${tbl}' and schemaname = '${ns}'`);
+
+  if (existsCache && index?.length > 0) { return null; }
+  // console.log('autoindex', table, filter?.map((el) => el.name || el));
+  const { rows: cols } = await pg.query(`SELECT attrelid::regclass AS tbl, attname AS aname, atttypid::regtype  AS datatype, atttypid  
+    FROM   pg_attribute WHERE  attrelid = '${tbs}'::regclass and attname = any('{ ${attrs} }')`);
+
+  const qIndex = [];
+  const indexAr = {};
+
+  // console.log(cols);
+  for (let i = 0; i < cols.length; i += 1) {
+    const el = cols[i];
+    el.relname = tbl;
+    el.nspname = ns;
+
+    const indexUsing = (el.datatype === 'geometry' ? 'gist' : null)
+      || (types[el.aname] === 'Text' || el.datatype?.includes('[]') ? 'gin' : null)
+      || 'btree';
+
+    const name = `${el.nspname}_${el.relname}_${el.aname}_${indexUsing}_idx`;
+    const exists = index.filter((ind) => ind.tablename === el.relname && ind.schemaname === el.nspname && ind.indexdef.includes(types[el.aname] === 'Text'
+      ? `gin (${el.aname} gin_trgm_ops)` : `btree (${el.aname})`));
+
+    indexAr[el.aname] = {
+      name, type: el.datatype, exists, // exists1,
+    };
+
+    // drop
+    if (exists?.length > 1) {
+      exists.filter((ind, i1) => i1).forEach((ind) => qIndex.push(`DROP INDEX if exists ${ind.schemaname}.${ind.indexname} `));
+    }
+
+    // add
+    if (exists?.length === 0) {
+      // console.log(`${name} - ${el.datatype}`);
+
+      const suffix = types[el.aname] === 'Text' ? 'gin_trgm_ops' : '';
+      if (['text'].includes(el?.datatype)) {
+        qIndex.push(`CREATE INDEX if not exists ${name} ON ${el.nspname}.${el.relname} USING ${indexUsing} (${el.aname} ${suffix})`);
+      }
+      if (indexUsing === 'gist') {
+        qIndex.push(`CREATE INDEX if not exists ${name.replace('gist', 'gist1')}  
+          ON ${el.nspname}.${el.relname} USING gist (${el.aname})`);
+
+        qIndex.push(`CREATE INDEX if not exists ${name.replace('gist', 'area')}  
+          ON ${el.nspname}.${el.relname} USING btree (st_area(st_transform(${el.aname},3857)))`);
+        qIndex.push(`CREATE INDEX if not exists ${name.replace('gist', '4326')}  
+          ON ${el.nspname}.${el.relname} USING gist (st_transform(${el.aname},4326))`);
+      }
+    }
+  }
+
+  loadedIndex[redisKey] = 1;
+  // throw qIndex;
+  if (!qIndex.length) return null;
+
+  // console.log(qIndex.filter((v, i, a) => a.indexOf(v) === i));
+  // logger.file('index', { table, filter, sql: qIndex.filter((v, i, a) => a.indexOf(v) === i) });
+  qIndex.filter((v, i, a) => a.indexOf(v) === i).map((el) => pg.one(el).catch((err) => console.log(err.toString())));
+  return 'ok';
+}
+
+export default autoIndex;

package/pg/funcs/getMeta.js

@@ -0,0 +1,31 @@
+import pgClient from '../pgClients.js';
+
+const data = {};
+
+// decorator
+export default async function getMeta(opt) {
+  const pg = pgClient.client;
+  const table = opt.table || opt;
+
+  if (data[table]) return data[table];
+
+  if (!pg.tlist.includes(table)) {
+    return { error: `${table} - not found`, status: 400 };
+  }
+
+  const { fields } = await pg.query(`select * from ${table} where $1=$1 limit 0`, [1]);
+  const { pks1 } = await pg.one(`SELECT json_object_agg(c.conrelid::regclass, a.attname) as pks1 FROM pg_constraint c 
+    left join pg_attribute a on c.conrelid=a.attrelid and a.attnum = c.conkey[1] WHERE c.contype='p'::"char"`, { cache: 0 });
+  const pk = pks1[table];
+
+  const geomAttr = fields.find((el) => pg.pgType[el.dataTypeID] === 'geometry')?.name; // change geometry text to geometry code
+  const q = `select st_extent( ${geomAttr}) as bounds, count(*), 
+    (select st_srid(${geomAttr}) from ${table} where ${geomAttr} is not null limit 1) as srid,  
+    json_agg(distinct ST_GeometryType(${geomAttr}))::json  as geometry from ${table} 
+    where ${geomAttr} is not null `;
+  const geom = await pg.one(q, { cache: 0 });
+
+  const res = { pk, columns: fields, geom };
+  data[table] = res;
+  return res;
+}

package/pg/funcs/getPG.js

@@ -0,0 +1,24 @@
+import pg from 'pg';
+import config from '../../config.js';
+import pgClients from '../pgClients.js';
+import init from './init.js';
+
+async function getPG({
+  user, password, host, port, db,
+}) {
+  if (pgClients[db]) return pgClients[db];
+
+  const dbConfig = {
+    user: user || config.pg?.user,
+    password: password || config.pg?.password,
+    host: host || config.pg?.host,
+    port: port || config.pg?.port,
+    database: db || config.pg?.db,
+  };
+
+  pgClients[db] = new pg.Pool(dbConfig);
+  init(pgClients[db]);
+  return pgClients[db];
+}
+
+export default getPG;

package/pg/funcs/init.js

@@ -0,0 +1,41 @@
+import crypto from 'crypto';
+
+// import pg from 'pg';
+import rclient from '../../redis/client.js';
+// import config from '../config.js';
+
+async function init(client) {
+  const textQuery = `select 
+  (select json_object_agg(conrelid::regclass  ,(SELECT attname FROM pg_attribute WHERE attrelid = c.conrelid and attnum = c.conkey[1]))
+  from   pg_constraint c where contype='p' and connamespace::regnamespace::text not in ('sde')) as pk,
+  (SELECT json_object_agg(t.oid::text,pg_catalog.format_type(t.oid, NULL)) FROM pg_catalog.pg_type t) as "pgType"`;
+  const { pgType, pk } = await client.query(textQuery).then((d) => d.rows[0]);
+
+  const tlist = await client.query(`select array_agg((select nspname from pg_namespace where oid=relnamespace)||'.'||relname) tlist 
+    from pg_class where relkind in ('r','v')`).then((d) => d.rows[0].tlist);
+
+  async function one(query, param = {}) {
+    const data = await client.query(query, Array.isArray(param) ? param : param.args || []);
+    const result = ((Array.isArray(data) ? data.pop() : data)?.rows || [])[0] || {};
+    return result;
+  }
+
+  async function queryCache(query) {
+    const hash = crypto.createHash('sha1').update(query).digest('base64');
+    const keyCache = `pg:${hash}`;
+    const cache = await rclient.get(keyCache);
+    if (cache) {
+      return JSON.parse(cache);
+    }
+    const data = await client.query(query);
+    rclient.set(keyCache, JSON.stringify(data), 'EX', 60 * 60);
+    return data;
+  }
+
+  Object.assign(client, {
+    one, pgType, pk, tlist, queryCache,
+  });
+}
+
+// export default client;
+export default init;

package/pg/funcs/pgClients.js

@@ -0,0 +1,2 @@
+const pgClients = {};
+export default pgClients;

package/pg/index.js

@@ -0,0 +1,41 @@
+// import fp from 'fastify-plugin';
+import pg from 'pg';
+import pgClients from './funcs/pgClients.js';
+import init from './funcs/init.js';
+import autoIndex from './funcs/autoIndex.js';
+import getMeta from './funcs/getMeta.js';
+import getPG from './funcs/getPG.js';
+
+function close() {
+  // console.log('pg close');
+  Object.keys(pgClients).forEach((el) => {
+    // console.log(el);
+    pgClients[el].end();
+  });
+
+  // return pgClients.client.end();
+}
+
+async function plugin(fastify, config) {
+  const client = new pg.Pool({
+    host: config.pg?.host || '127.0.0.1',
+    port: config.pg?.port || 5432,
+    database: config.pg?.database || 'postgres',
+    user: config.pg?.user || 'postgres',
+    password: config.pg?.password || 'postgres',
+  });
+  await init(client);
+  fastify.addHook('onRequest', async (req) => {
+    // req.funcs = fastify;
+    req.pg = client;
+  });
+  pgClients.client = client;
+
+  // fastify.decorate('pg', client);
+  fastify.decorate('autoIndex', autoIndex);
+  fastify.decorate('getMeta', getMeta);
+  fastify.decorate('getPG', getPG);
+  fastify.addHook('onClose', close);
+}
+
+export default plugin;

package/pg/pgClients.js

@@ -0,0 +1,17 @@
+import pg from 'pg';
+import config from '../config.js';
+import init from './funcs/init.js';
+
+const pgClients = {};
+if (config.pg) {
+  const client = new pg.Pool({
+    host: config.pg?.host || '127.0.0.1',
+    port: config.pg?.port || 5432,
+    database: config.pg?.database || 'postgres',
+    user: config.pg?.user || 'postgres',
+    password: config.pg?.password || 'postgres',
+  });
+  await init(client);
+  pgClients.client = client;
+}
+export default pgClients;

package/redis/client.js

@@ -0,0 +1,14 @@
+import Redis from 'ioredis';
+
+// const config = {};
+import config from '../config.js';
+
+const rclient = new Redis({
+  keyPrefix: `${config?.db}:`,
+  host: config?.redis?.host || '127.0.0.1',
+  port: config?.redis?.port || 6379, // Redis port
+  family: 4, // 4 (IPv4) or 6 (IPv6)
+  closeClient: true,
+});
+
+export default rclient;

package/redis/index.js

@@ -0,0 +1,15 @@
+import client from './client.js';
+
+function close(fastify) {
+  fastify.rclient.quit();
+  // fastify.rclient2.quit();
+}
+
+async function plugin(fastify) {
+  client.getJSON = client.get;
+  fastify.decorate('rclient', client);
+  // fastify.decorate('rclient2', client2);
+  fastify.addHook('onClose', close);
+}
+
+export default plugin;

package/table/controllers/data.js

@@ -0,0 +1,47 @@
+import getTemplate from './utils/getTemplate.js';
+import getFilterSQL from '../funcs/getFilterSQL/index.js';
+import getMeta from '../../pg/funcs/getMeta.js';
+
+const maxLimit = 10;
+export default async function data(req) {
+  const time = Date.now();
+  const {
+    pg, params, query = {},
+  } = req;
+
+  const loadTable = await getTemplate('table', params.table);
+
+  if (!loadTable) { return { status: 404, message: 'not found' }; }
+
+  const {
+    table, columns, sql, filters, form, meta,
+  } = loadTable;
+  const { pk } = await getMeta(table);
+
+  const cols = columns.map((el) => el.name || el).join(',');
+  const sqlTable = sql?.filter?.((el) => !el?.disabled && el?.sql?.replace).map((el, i) => ` left join lateral (${el.sql}) ${el.name || `t${i}`} on 1=1 `)?.join('') || '';
+
+  const fData = query.filter ? await getFilterSQL(query.filter, {
+    table,
+    json: 1,
+  }) : {};
+
+  const keyQuery = query.key && loadTable.key && !params.id ? `${loadTable.key}=$1` : null;
+
+  const limit = Math.min(maxLimit, +(query.limit || 10));
+
+  const offset = query.page ? ` offset ${(query.page - 1) * limit}` : '';
+  // id, query, filter
+  const where = [(params.id ? ` "${pk}" = $1` : null), keyQuery, loadTable.query, fData.q].filter((el) => el);
+  const q = `select  ${pk ? `"${pk}" as id,` : ''} ${query.id || query.key ? '*' : cols || '*'} from ${table} t ${sqlTable} where ${where.join(' and ') || 'true'} ${offset} limit ${limit}`;
+
+  if (query.sql) { return q; }
+
+  const { rows } = await pg.query(q, (params.id ? [params.id] : null) || (query.key && loadTable.key ? [query.key] : []));
+
+  const total = keyQuery || params.id ? rows.length : await pg.queryCache(`select count(*) from ${table} where ${where.join(' and ') || 'true'}`).then((el) => el?.rows[0]?.count);
+
+  return {
+    time: Date.now() - time, total, pk, form, rows, meta, columns, filters,
+  };
+}

package/table/controllers/filter.js

@@ -0,0 +1,17 @@
+import getTemplate from './utils/getTemplate.js';
+import getSelect from './utils/getSelect.js';
+
+export default async function filter(req) {
+  const time = Date.now();
+  const {
+    params,
+  } = req;
+  const loadTable = await getTemplate('table', params.table);
+  if (!loadTable) { return { status: 404, message: 'not found' }; }
+
+  await Promise.all(loadTable?.filters.filter((el) => el.data).map(async (el) => {
+    const cls = await getSelect(el.data);
+    Object.assign(el, { options: cls?.arr });
+  }));
+  return { time: Date.now() - time, filters: loadTable?.filters || [] };
+}

package/table/controllers/form.js

@@ -0,0 +1,10 @@
+import getTemplate from './utils/getTemplate.js';
+
+export default async function formFunction(req) {
+  const time = Date.now();
+  const { params } = req;
+  const form = await getTemplate('form', params.form);
+  if (!form) { return { status: 404, message: 'not found' }; }
+
+  return { time: Date.now() - time, form };
+}