@opengis/bi 1.0.21 → 1.0.22

package/server/routes/dataset/controllers/data.js

@@ -0,0 +1,145 @@
+import {
+  config, pgClients, getMeta, getFilterSQL, setToken, getToken,
+} from '@opengis/fastify-table/utils.js';
+
+import metaFormat from '../utils/metaFormat.js';
+import datasetFormData from './table.js';
+
+const maxLimit = 100;
+
+export default async function datasetData(req, reply) {
+  const {
+    pg = pgClients.client, params = {}, query = {}, user = {},
+  } = req;
+  const time = Date.now();
+  const { uid, user_type: userType = 'regular' } = user;
+
+  if (!uid) {
+    return { message: 'access restricted', status: 403 };
+  }
+
+  if (!params?.id) {
+    return { message: 'not enough params: id', status: 404 };
+  }
+
+  const tokenData = await getToken({
+    uid: user?.uid,
+    token: params.id,
+    mode: 'w',
+    json: 1,
+  });
+
+  if (tokenData?.dataset && tokenData?.id) {
+    return datasetFormData(req, reply);
+  }
+
+  const dataset = await pg.query(`select dataset_id as id, table_name as table, column_list as columns, filter_list as filters,
+    sql_list as sql, style, form_setting, pk, access_level from bi.dataset where dataset_id=$1`, [params.id]).then(el => el.rows[0] || {});
+
+  if (!dataset?.id) {
+    return { message: 'dataset not found', status: 404 };
+  }
+
+  const tlist = await pg.query(`select array_agg((select nspname from pg_namespace where oid=relnamespace)||'.'||relname) tlist 
+    from pg_class where relkind in ('r','v')`).then(el => el.rows[0]?.tlist || []);
+
+  if (!dataset.table || !tlist.includes(dataset.table.replace(/"/g, '')) || (!pg.pk?.[dataset.table.replace(/"/g, '')] && !dataset.pk)) {
+    return { message: `table not found: ${dataset.table}`, status: 404 };
+  }
+
+  const { pk = dataset.pk, columns: dbColumns = [] } = await getMeta(dataset.table) || {};
+  const cols = dataset.columns?.filter((el) => el.name !== 'geom')?.map((el) => el.name || el)?.join(',');
+
+  const limit = Math.min(maxLimit, +(query.limit || 20));
+  const offset = query.page && query.page > 0 ? ` offset ${(query.page - 1) * limit}` : '';
+
+  const checkFilter = [query.filter, query.search].filter((el) => el).length;
+
+  const fData = checkFilter ? await getFilterSQL({
+    table: dataset.table,
+    filter: query.filter,
+    search: query.search,
+    json: 1,
+  }) : {};
+
+  const where = [dataset.query, fData.q].filter((el) => el);
+
+  const sqlTable = dataset.sql
+    ?.filter?.((el) => !el?.disabled && el?.sql?.replace)
+    ?.map((el, i) => ` left join lateral (${el.sql.replace('{{uid}}', uid)}) ${el.name || `t${i}`} on 1=1 `)
+    ?.join('') || '';
+
+  const columnList = dbColumns.map((el) => el.name || el).join(',');
+  const order = columnList.includes('cdate') ? (`order by cdate ${query.desc ? 'desc' : ''}`) : '';
+
+  const q = `select 
+  ${cols || '*'} 
+    ${pk ? `,"${pk}" as id` : ''}
+    ${dbColumns.find((el) => el.name === 'files' && pg.pgType[el.dataTypeID] === 'text') ? ',files' : ''} 
+    ${dbColumns.find((el) => el.name === 'geom' && pg.pgType[el.dataTypeID] === 'geometry') ? ',st_asgeojson(geom)::json as geom' : ''} 
+    from (select * from ${dataset.table} where ${sqlTable ? 'true' : (where.join(' and ') || 'true')}  ${order}) t 
+    ${sqlTable} where ${where.join(' and ') || 'true'} ${order} ${offset}  limit ${limit}`.replace(/{{uid}}/g, uid);
+
+  if (query.sql === '1' && (config.debug || userType.includes('admin'))) { return q; }
+
+  // console.log(pg.options.database, q);
+  const { rows = [] } = await pg.query(q);
+
+  const qCount = `select 
+    count(*)::int as total,  
+    count(*) FILTER(WHERE ${[fData.q].filter(el => el).join(' and ') || 'true'})::int as filtered 
+    from ${dataset.table} t ${sqlTable} 
+    where ${[dataset.query].filter(el => el).join(' and ') || 'true'} `.replace(/{{uid}}/g, uid);
+
+  const { total, filtered } = await pg.queryCache(qCount).then(el => el?.rows?.[0] || {});
+
+  await metaFormat({ rows, columns: dataset.columns });
+
+  if (rows?.length && uid) {
+    rows.filter((row) => row[pk] || row.id).forEach((row) => Object.assign(row, {
+      token: setToken({
+        ids: [JSON.stringify({
+          id: row[pk] || row.id,
+          table: dataset.table,
+          form: `${params.id}.form`,
+          dataset: params.id,
+        })],
+        uid,
+        array: 1,
+      })[0],
+    }));
+  }
+  const res = {
+    time: Date.now() - time,
+    access: dataset.access_level,
+    card: dataset.card,
+    actions: ['add', 'edit', 'del', 'get'],
+    total,
+    filtered,
+    count: rows.length,
+    pk,
+    table: dataset.table,
+    form: `${params.id}.form`,
+    rows,
+    columns: dataset.columns || dbColumns.map(col => ({ name: col.name, format: pg.pgType[col.dataTypeID], title: col.name })),
+    filters: dataset.filters,
+  };
+
+  if (uid) {
+    const addTokens = setToken({
+      ids: [JSON.stringify({
+        table: dataset.table,
+        form: `${params.id}.form`,
+        dataset: dataset.id,
+      })],
+      uid,
+      array: 1,
+    });
+
+    Object.assign(res, {
+      addToken: addTokens[0],
+    });
+  }
+
+  return res;
+}

package/server/routes/{db → dataset}/controllers/dbTablePreview.js

@@ -16,28 +16,21 @@ export default async function dbTablePreview({ pg, params = {}, query = {} }) {
   if (query.sql) return q;
   try {
     const { table, columns } = await pg
-      .query(q, [params.name])
-      .then((res) => res.rows?.[0] || {});
+      .query(q, [params.name.replace(/"/g, '')])
+      .then(el => el.rows?.[0] || {});
     if (!table) {
       return { message: 'table not found', status: 404 };
     }
 
-    const { count = 0 } = await pg
-      .query(
-        `select reltuples as count from pg_class where oid  = to_regclass($1)`,
-        [params.name]
-      )
-      .then((res) => res.rows?.[0] || {});
+    const { count = 0 } = await pg.query('select reltuples as count from pg_class where oid  = to_regclass($1)', [params.name])
+      .then(el => el.rows?.[0] || {});
     const geom = columns.find((el) => el.type === 'geometry')?.name;
     const { bounds, extentStr } = geom
-      ? await pg
-          .query(
-            `select count(*),
-      st_asgeojson(st_extent(${geom}))::json as bounds,
-      replace(regexp_replace(st_extent(${geom})::box2d::text,'BOX\\(|\\)','','g'),' ',',') as "extentStr"
-    from ${params.name}`
-          )
-          .then((res) => res.rows?.[0] || {})
+      ? await pg.query(`select 
+        count(*),
+        st_asgeojson(st_extent(${geom}))::json as bounds,
+        replace(regexp_replace(st_extent(${geom})::box2d::text,'BOX\\(|\\)','','g'),' ',',') as "extentStr"
+      from ${params.name}`).then(el => el.rows?.[0] || {})
       : {};
     const extent = extentStr ? extentStr.split(',') : undefined;
 
@@ -49,15 +42,15 @@ export default async function dbTablePreview({ pg, params = {}, query = {} }) {
       'editor_id',
       geom,
     ];
-    const columnList = columns
-      .map((el) => el?.name)
-      .filter((el) => !systemColumns.includes(el));
-    const { rows = [] } = await pg.query(
-      `select ${columnList.join(',')} ${geom ? `, st_asgeojson(geom)::json as geom` : ''} from ${table} limit 10`
-    );
 
-    return { count, geom: !!geom, bounds, extent, columns, rows };
-  } catch (err) {
+    const columnList = columns.map((el) => el?.name).filter((el) => !systemColumns.includes(el));
+    const { rows = [] } = await pg.query(`select ${columnList.map(el => `"${el.replace(/'/g, "''")}"`).join(',')} ${geom ? ', st_asgeojson(geom)::json as geom' : ''} from ${params.name.replace(/'/g, '')} limit 10`);
+
+    return {
+      count, geom: !!geom, bounds, extent, columns, rows,
+    };
+  }
+  catch (err) {
     return { error: err.toString(), status: 500 };
   }
 }

package/server/routes/{db → dataset}/controllers/dbTables.js

@@ -1,6 +1,6 @@
 export default async function dbTables({ pg, query = {} }) {
   const q = `select 
-  t.table_schema ||'.'|| t.table_name as table, 
+  t.table_schema ||'."'|| t.table_name ||'"' as table, 
   obj_description(to_regclass(t.table_schema ||'."'|| t.table_name||'"')) as description,
   t.table_schema as schema,
   (select reltuples from pg_class where oid  = to_regclass(t.table_schema ||'."'|| t.table_name||'"') ) as total,
@@ -22,15 +22,11 @@ export default async function dbTables({ pg, query = {} }) {
     INNER JOIN pg_catalog.pg_namespace nsp ON nsp.oid = connamespace 
     WHERE nsp.nspname = t.table_schema AND rel.relname = t.table_name and contype='p'
   )
-  and ${query.schema ? `t.table_schema=$1` : '1=1'} order by total desc`;
+  and isgeom
+  order by total desc`;
+
   if (query.sql) return q;
-  try {
-    const { rows = [] } = await pg.query(
-      q,
-      [query.schema].filter((el) => el)
-    );
-    return { rows };
-  } catch (err) {
-    return { error: err.toString(), status: 500 };
-  }
+
+  const { rows = [] } = await pg.queryCache(q, { time: 0 });
+  return { rows };
 }

package/server/routes/dataset/controllers/delete.js

@@ -0,0 +1,39 @@
+import {
+  config, getToken, dataDelete, pgClients,
+} from '@opengis/fastify-table/utils.js';
+
+export default async function datasetDataDelete({
+  pg = pgClients.client, params = {}, user = {},
+}) {
+  if (!user?.uid) {
+    return { message: 'access restricted', status: 403 };
+  }
+
+  const tokenData = await getToken({
+    uid: user?.uid,
+    token: params.id,
+    mode: 'a',
+    json: 1,
+  });
+
+  if (!tokenData && !config.local && !config.debug) {
+    return { message: 'token not allow', status: 403 };
+  }
+
+  const dataset = await pg.query('select dataset_id as id, table_name as table from bi.dataset where dataset_id=$1', [tokenData?.dataset || params.id])
+    .then(el => el.rows[0] || {});
+
+  const table = tokenData?.table || dataset.table;
+
+  if (!dataset.id) {
+    return { message: 'dataset not found', status: 404 };
+  }
+
+  const res = await dataDelete({
+    id: tokenData?.id || params?.object_id,
+    table,
+    uid: user?.uid,
+  });
+
+  return { rowCount: res.rowCount, msg: !res.rowCount ? res : null };
+}

package/server/routes/dataset/controllers/{bi.dataset.edit.js → editDataset.js}

@@ -1,4 +1,4 @@
-import { dataUpdate, pgClients, getPG, initPG } from "@opengis/fastify-table/utils.js";
+import { dataUpdate, pgClients } from '@opengis/fastify-table/utils.js';
 
 const columnTypeMatch = {
   text: 'text',
@@ -12,12 +12,12 @@ const columnTypeMatch = {
 };
 
 /**
- * Додавання нового набору даних BI
+ * Редагування структури набору даних
  *
- * @method POST
- * @summary Додавання нового набору даних BI
+ * @method PUT
+ * @summary Редагування структури набору даних
  * @priority 4
- * @alias biDatasetAdd
+ * @alias editDataset
  * @type api
  * @tag bi
  * @param {Object} params.id Dashboard ID
@@ -27,47 +27,53 @@ const columnTypeMatch = {
  * @returns {Object} rows Масив з колонками таблиці
  */
 
-export default async function biDatasetEdit(req) {
-  const { params = {}, body = {}, session = {} } = req;
-  const { uid } = session?.passport?.user || {};
-
-  if (!uid) {
+export default async function editDataset(req) {
+  const {
+    pg = pgClients.client, query = {}, body = {}, user = {},
+  } = req;
+  if (!user?.uid) {
     return { message: 'access restricted', status: 403 };
   }
 
-  if (!body.columns?.length) {
+  if (!query?.id) {
+    return { message: 'not enough params: id', status: 404 };
+  }
+
+  if (!body.column_list?.length) {
     return { message: 'not enough params: columns', status: 400 };
   }
 
-  const dataset = await pgClients.client.query('select db, dataset_id as id, table_name as table from bi.dataset where dataset_id=$1', [params.id])
-    .then(el => el.rows[0]);
+  const dataset = await pg.query('select dataset_id as id, table_name as table, column_list as columns from bi.dataset where dataset_id=$1', [query.id])
+    .then(el => el.rows[0] || {});
 
-  if (!dataset.id) {
+  if (!dataset?.id) {
     return { message: 'dataset not found', status: 404 };
   }
 
-  const pg = dataset.db ? getPG({ db: dataset.db }) : pgClients.client;
-  if (!pg.pk) await initPG(pg);
-
   if (!dataset.table || !pg.pk?.[dataset.table]) {
     return { message: `table not found: ${dataset.table}`, status: 404 };
   }
 
+  if (!dataset.table.startsWith('data_user.')) {
+    return { message: 'access restricted: source', status: 403 };
+  }
+
   const { fields = [] } = await pg.query(`select * from ${dataset.table} limit 0`);
   const columnList = fields.map((col) => col.name);
 
-  const columns = (dataset.columns || body.columns).map((col, idx) => ({
-    ...col,
-    name: col.name && columnList.includes(col.name) ? col.name : `col_${idx}`,
-    disabled: col.name && !body.columns.find((item) => item.name === col.name),
-  }));
+  const columns = (dataset.columns || [])
+    .concat(body.column_list.filter((col) => !col.name || !columnList.includes(col.name)))
+    .map((col, idx) => ({
+      ...col,
+      name: col.name && columnList.includes(col.name) ? col.name : `col_${idx}`,
+      disabled: col.name && !body.column_list.find((item) => item.name === col.name),
+    }));
 
   await dataUpdate({
-    pg: pgClients.client,
     table: 'bi.dataset',
     data: { column_list: columns },
     id: dataset.id,
-    uid,
+    uid: user.uid,
   });
 
   const sqlList = columns
@@ -79,5 +85,5 @@ export default async function biDatasetEdit(req) {
     await pg.query(sqlList.join(';'));
   }
 
-  return { message: { id: dataset?.id, action: 'edit', columns }, status: 200 };
-};
+  return { message: { id: dataset?.id, table: dataset.table, columns }, status: 200 };
+}

package/server/routes/dataset/controllers/export.js

@@ -0,0 +1,213 @@
+/* eslint-disable no-await-in-loop */
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+import { existsSync, rmSync } from 'node:fs';
+
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+
+import {
+  config, pgClients, getMeta, getFolder, getFilterSQL, eventStream, logger,
+} from '@opengis/fastify-table/utils.js';
+
+import convertJSONToXls from '../utils/convertJSONToXls.js';
+import convertJSONToCSV from '../utils/convertJSONToCSV.js';
+import metaFormat from '../utils/metaFormat.js';
+
+export default async function datasetDataExport(req, reply) {
+  const {
+    pg = pgClients.client, params = {}, query = {}, user = {}, unittest,
+  } = req;
+
+  if (!user?.uid) {
+    return { message: 'access restricted', status: 403 };
+  }
+
+  const { user_type: userType = 'regular' } = user;
+
+  if (!params?.id) {
+    return { message: 'not enough params: id', status: 404 };
+  }
+
+  const { format = 'json' } = query;
+
+  const dataset = await pg.query(`select dataset_id as id, table_name as table, column_list as columns,
+      export_columns as "exportColumns", sql_list as sql from bi.dataset where dataset_id=$1`, [params.id])
+    .then(el => el.rows[0] || {});
+
+  if (!dataset?.id || !dataset.table) {
+    return { message: 'dataset not found', status: 404 };
+  }
+
+  const { pk = dataset.pk, view, columns: dbColumns = [] } = await getMeta(dataset.table);
+
+  if (!dataset.table || (!pg.pk?.[dataset.table] && !pk && !view)) {
+    return { message: `table not found: ${dataset.table}`, status: 404 };
+  }
+
+  const rootDir = getFolder(req, 'local');
+
+  const date = new Date();
+  const sufixName = `${query.filter || '1=1'}-${query.col || 'all'}-${query.search || '1=1'}-${query.limit || 'unlimited'}`;
+  const sufixDate = [date.getFullYear(), date.getMonth(), date.getDate(), date.getHours()].join('-');
+
+  const objInfo = createHash('md5').update([sufixName, sufixDate].join('-')).digest('hex');
+  const fileName = dataset.table.concat('_').concat(objInfo).concat('.').concat(format);
+  const filepath = path.join(rootDir, '/files/temp', fileName);
+
+  if (existsSync(filepath) && query.nocache) {
+    rmSync(filepath);
+  }
+
+  if (existsSync(filepath) && !query?.sql) {
+    const encoding = path.extname(filepath) === '.xlsx' ? 'base64' : 'utf8';
+    const data = await readFile(filepath, encoding);
+    return reply.headers({ 'Content-Disposition': `attachment; filename=${path.basename(filepath)}` }).send(Buffer.from(data, encoding));
+  }
+
+  const fData = await getFilterSQL({
+    filter: query.filter,
+    search: query?.search,
+    table: dataset.table,
+    json: 1,
+  });
+
+  const where = [(params.object_id ? ` "${pk}" = $1` : null), dataset.query, fData.q].filter((el) => el);
+
+  const cols = dataset.exportColumns
+    || dataset.columns?.map((el) => el.name);
+
+  if (!cols?.length) {
+    return { message: `empty export columns: ${dataset.id}`, status: 400 };
+  }
+
+  const sqlTable = dataset.sql
+    ?.filter?.((el) => !el?.disabled && el?.sql?.replace)
+    ?.map((el, i) => ` left join lateral (${el.sql.replace('{{uid}}', user?.uid)}) ${el.name || `t${i}`} on 1=1 `)
+    ?.join('') || '';
+
+  const { total = '0' } = await pg.query(`select count(*) as total from ${dataset.table} 
+        where ${dataset.query || 'true'} and ${params.object_id ? `${pk}='${params.object_id}'` : '1=1'}`)
+    .then(el => el.rows?.[0] || {});
+
+  if (total === '0') {
+    return { message: 'Немає даних, які можна експортувати', status: 200 };
+  }
+
+  const ext = path.extname(filepath);
+  const filepathJSON = ['csv', 'xlsx'].includes(format) ? filepath.replace(ext, '.json') : filepath;
+
+  const cacheFileJSON = existsSync(filepathJSON);
+
+  const send = +total < 10000 || unittest
+    ? console.log
+    : eventStream(reply);
+
+  if (cacheFileJSON && !query.sql) {
+    if (query.nocache) rmSync(filepathJSON); // delete old file, prevent append
+    if (!query.nocache && ['xlsx', 'csv'].includes(format)) {
+      send(`Знайдено файл формату json. Пропуск обробки даних. Початок конвертації в ${format}...`);
+      if (format === 'xlsx') {
+        return convertJSONToXls({
+          filepathJSON,
+          colmodel: dataset.columns,
+          domain: req.hostname,
+          source: dataset.table,
+          send,
+        });
+      }
+      if (format === 'csv') {
+        return convertJSONToCSV({ send, filepathJSON });
+      }
+    }
+  }
+
+  const limit = Math.min(query.limit, 10000) || 10000;
+
+  const optimizedSQL = `select  ${pk ? `"${pk}" as id,` : ''} *
+    ${['geojson', 'shp'].includes(format) && dbColumns.find((el) => el.name === 'geom' && pg.pgType[el.dataTypeID] === 'geometry') ? ',st_asgeojson(geom)::json as geom' : ''} 
+  from (select * from ${dataset.table} where ${sqlTable ? 'true' : (where.join(' and ') || 'true')}) t 
+  ${sqlTable} where ${where.join(' and ') || 'true'} offset 0 limit ${limit}`.replace(/{{uid}}/g, user?.uid);
+
+  if (query.sql === '1' && (config.debug || userType.includes('admin'))) return optimizedSQL;
+
+  const res = {};
+  let offset = 0;
+  let seq = 0;
+
+  send(`Всього в реєстрі: ${total}`);
+
+  while ((+total - offset > 0) && !res?.error) {
+    try {
+      const q = optimizedSQL.replace(/offset 0/g, `offset ${offset}`);
+      const { rows = [] } = await pg.query(q, (params.object_id ? [params.object_id] : null));
+
+      send(`Оброблено: ${offset}/${total}`);
+      send(`seq: ${++seq}`);
+      send(`Обробка ${rows.length} об'єктів...`);
+
+      if (!rows.length) {
+        send('Обробка даних успішно завершена');
+        break;
+      }
+
+      await metaFormat({ rows, columns: dataset.columns });
+
+      // skip non present after metaFormat
+      const allowColumnList = []; // 'id'
+      if (['geojson', 'shp'].includes(format)) {
+        allowColumnList.push('geom');
+      }
+      rows.forEach((row) => {
+        Object.keys(row)
+          .filter((el) => !allowColumnList.includes(el.name) && !cols.includes(el))
+          .forEach((key) => delete row[key]);
+      });
+
+      // convert csv / xlsx from json
+      if (!existsSync(filepathJSON)) { // if json not exists
+        await mkdir(path.dirname(filepath), { recursive: true });
+        await writeFile(filepathJSON, JSON.stringify(rows));
+      }
+      else { // if json exists
+        const jsonData = JSON.parse(await readFile(filepathJSON) || '{}');
+        const moreData = jsonData.concat(rows); // rewrite to appendFile?
+        await writeFile(filepathJSON, JSON.stringify(moreData));
+      }
+
+      offset += rows.length;
+    }
+    catch (err) {
+      Object.assign(res, { error: err.toString(), status: 500 });
+      send(`error: ${err.toString()}`, 1);
+      logger.error({
+        name: 'export/table',
+        filepathJSON,
+        total,
+        offset,
+        result: res,
+        error: err.toString(),
+      });
+      return err.toString();
+    }
+  }
+
+  if (!res?.error && format === 'csv') {
+    send('Сформовано файл формату json. Початок конвертації в csv...');
+    return convertJSONToCSV({ send, filepathJSON });
+  }
+
+  if (!res?.error && format === 'xlsx') {
+    send('Сформовано файл формату json. Початок конвертації в xlsx...');
+    return convertJSONToXls({
+      filepathJSON,
+      colmodel: dataset.columns,
+      domain: req.hostname,
+      source: dataset.table,
+      send,
+    });
+  }
+
+  const message = res.error || 'Файл успішно сформовано. Натистіть кнопку ще раз для завантаження даних';
+  send(message, 1);
+  return { message, status: 200 };
+}