@opengis/bi 1.0.21 → 1.0.23

package/server/routes/dataset/controllers/export.js

@@ -0,0 +1,213 @@
+/* eslint-disable no-await-in-loop */
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+import { existsSync, rmSync } from 'node:fs';
+
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+
+import {
+  config, pgClients, getMeta, getFolder, getFilterSQL, eventStream, logger,
+} from '@opengis/fastify-table/utils.js';
+
+import convertJSONToXls from '../utils/convertJSONToXls.js';
+import convertJSONToCSV from '../utils/convertJSONToCSV.js';
+import metaFormat from '../utils/metaFormat.js';
+
+export default async function datasetDataExport(req, reply) {
+  const {
+    pg = pgClients.client, params = {}, query = {}, user = {}, unittest,
+  } = req;
+
+  if (!user?.uid) {
+    return { message: 'access restricted', status: 403 };
+  }
+
+  const { user_type: userType = 'regular' } = user;
+
+  if (!params?.id) {
+    return { message: 'not enough params: id', status: 404 };
+  }
+
+  const { format = 'json' } = query;
+
+  const dataset = await pg.query(`select dataset_id as id, table_name as table, column_list as columns,
+      export_columns as "exportColumns", sql_list as sql from bi.dataset where dataset_id=$1`, [params.id])
+    .then(el => el.rows[0] || {});
+
+  if (!dataset?.id || !dataset.table) {
+    return { message: 'dataset not found', status: 404 };
+  }
+
+  const { pk = dataset.pk, view, columns: dbColumns = [] } = await getMeta(dataset.table);
+
+  if (!dataset.table || (!pg.pk?.[dataset.table] && !pk && !view)) {
+    return { message: `table not found: ${dataset.table}`, status: 404 };
+  }
+
+  const rootDir = getFolder(req, 'local');
+
+  const date = new Date();
+  const sufixName = `${query.filter || '1=1'}-${query.col || 'all'}-${query.search || '1=1'}-${query.limit || 'unlimited'}`;
+  const sufixDate = [date.getFullYear(), date.getMonth(), date.getDate(), date.getHours()].join('-');
+
+  const objInfo = createHash('md5').update([sufixName, sufixDate].join('-')).digest('hex');
+  const fileName = dataset.table.concat('_').concat(objInfo).concat('.').concat(format);
+  const filepath = path.join(rootDir, '/files/temp', fileName);
+
+  if (existsSync(filepath) && query.nocache) {
+    rmSync(filepath);
+  }
+
+  if (existsSync(filepath) && !query?.sql) {
+    const encoding = path.extname(filepath) === '.xlsx' ? 'base64' : 'utf8';
+    const data = await readFile(filepath, encoding);
+    return reply.headers({ 'Content-Disposition': `attachment; filename=${path.basename(filepath)}` }).send(Buffer.from(data, encoding));
+  }
+
+  const fData = await getFilterSQL({
+    filter: query.filter,
+    search: query?.search,
+    table: dataset.table,
+    json: 1,
+  });
+
+  const where = [(params.object_id ? ` "${pk}" = $1` : null), dataset.query, fData.q].filter((el) => el);
+
+  const cols = dataset.exportColumns
+    || dataset.columns?.map((el) => el.name);
+
+  if (!cols?.length) {
+    return { message: `empty export columns: ${dataset.id}`, status: 400 };
+  }
+
+  const sqlTable = dataset.sql
+    ?.filter?.((el) => !el?.disabled && el?.sql?.replace)
+    ?.map((el, i) => ` left join lateral (${el.sql.replace('{{uid}}', user?.uid)}) ${el.name || `t${i}`} on 1=1 `)
+    ?.join('') || '';
+
+  const { total = '0' } = await pg.query(`select count(*) as total from ${dataset.table} 
+        where ${dataset.query || 'true'} and ${params.object_id ? `${pk}='${params.object_id}'` : '1=1'}`)
+    .then(el => el.rows?.[0] || {});
+
+  if (total === '0') {
+    return { message: 'Немає даних, які можна експортувати', status: 200 };
+  }
+
+  const ext = path.extname(filepath);
+  const filepathJSON = ['csv', 'xlsx'].includes(format) ? filepath.replace(ext, '.json') : filepath;
+
+  const cacheFileJSON = existsSync(filepathJSON);
+
+  const send = +total < 10000 || unittest
+    ? console.log
+    : eventStream(reply);
+
+  if (cacheFileJSON && !query.sql) {
+    if (query.nocache) rmSync(filepathJSON); // delete old file, prevent append
+    if (!query.nocache && ['xlsx', 'csv'].includes(format)) {
+      send(`Знайдено файл формату json. Пропуск обробки даних. Початок конвертації в ${format}...`);
+      if (format === 'xlsx') {
+        return convertJSONToXls({
+          filepathJSON,
+          colmodel: dataset.columns,
+          domain: req.hostname,
+          source: dataset.table,
+          send,
+        });
+      }
+      if (format === 'csv') {
+        return convertJSONToCSV({ send, filepathJSON });
+      }
+    }
+  }
+
+  const limit = Math.min(query.limit, 10000) || 10000;
+
+  const optimizedSQL = `select  ${pk ? `"${pk}" as id,` : ''} *
+    ${['geojson', 'shp'].includes(format) && dbColumns.find((el) => el.name === 'geom' && pg.pgType[el.dataTypeID] === 'geometry') ? ',st_asgeojson(geom)::json as geom' : ''} 
+  from (select * from ${dataset.table} where ${sqlTable ? 'true' : (where.join(' and ') || 'true')}) t 
+  ${sqlTable} where ${where.join(' and ') || 'true'} offset 0 limit ${limit}`.replace(/{{uid}}/g, user?.uid);
+
+  if (query.sql === '1' && (config.debug || userType.includes('admin'))) return optimizedSQL;
+
+  const res = {};
+  let offset = 0;
+  let seq = 0;
+
+  send(`Всього в реєстрі: ${total}`);
+
+  while ((+total - offset > 0) && !res?.error) {
+    try {
+      const q = optimizedSQL.replace(/offset 0/g, `offset ${offset}`);
+      const { rows = [] } = await pg.query(q, (params.object_id ? [params.object_id] : null));
+
+      send(`Оброблено: ${offset}/${total}`);
+      send(`seq: ${++seq}`);
+      send(`Обробка ${rows.length} об'єктів...`);
+
+      if (!rows.length) {
+        send('Обробка даних успішно завершена');
+        break;
+      }
+
+      await metaFormat({ rows, columns: dataset.columns });
+
+      // skip non present after metaFormat
+      const allowColumnList = []; // 'id'
+      if (['geojson', 'shp'].includes(format)) {
+        allowColumnList.push('geom');
+      }
+      rows.forEach((row) => {
+        Object.keys(row)
+          .filter((el) => !allowColumnList.includes(el.name) && !cols.includes(el))
+          .forEach((key) => delete row[key]);
+      });
+
+      // convert csv / xlsx from json
+      if (!existsSync(filepathJSON)) { // if json not exists
+        await mkdir(path.dirname(filepath), { recursive: true });
+        await writeFile(filepathJSON, JSON.stringify(rows));
+      }
+      else { // if json exists
+        const jsonData = JSON.parse(await readFile(filepathJSON) || '{}');
+        const moreData = jsonData.concat(rows); // rewrite to appendFile?
+        await writeFile(filepathJSON, JSON.stringify(moreData));
+      }
+
+      offset += rows.length;
+    }
+    catch (err) {
+      Object.assign(res, { error: err.toString(), status: 500 });
+      send(`error: ${err.toString()}`, 1);
+      logger.error({
+        name: 'export/table',
+        filepathJSON,
+        total,
+        offset,
+        result: res,
+        error: err.toString(),
+      });
+      return err.toString();
+    }
+  }
+
+  if (!res?.error && format === 'csv') {
+    send('Сформовано файл формату json. Початок конвертації в csv...');
+    return convertJSONToCSV({ send, filepathJSON });
+  }
+
+  if (!res?.error && format === 'xlsx') {
+    send('Сформовано файл формату json. Початок конвертації в xlsx...');
+    return convertJSONToXls({
+      filepathJSON,
+      colmodel: dataset.columns,
+      domain: req.hostname,
+      source: dataset.table,
+      send,
+    });
+  }
+
+  const message = res.error || 'Файл успішно сформовано. Натистіть кнопку ще раз для завантаження даних';
+  send(message, 1);
+  return { message, status: 200 };
+}

package/server/routes/dataset/controllers/form.js

@@ -0,0 +1,99 @@
+import {
+  config, pgClients, getMeta, getToken,
+} from '@opengis/fastify-table/utils.js';
+
+import datasetForms from '../utils/datasetForms.js';
+
+const inputType = {
+  text: 'Text',
+  autocomplete: 'Autocomplete',
+  select: 'Autocomplete',
+  date: 'DatePicker',
+  'yes/no': 'Switcher',
+  badge: 'Select',
+  number: 'Number',
+  tags: 'Tags',
+  file: 'File',
+};
+
+const systemColumns = [
+  'uid',
+  'cdate',
+  'editor_id',
+  'editor_date',
+  'files',
+];
+
+export default async function datasetForm({
+  pg = pgClients.client, params = {}, query = {}, user = {},
+}) {
+  if (!user?.uid) {
+    return { message: 'access restricted', status: 403 };
+  }
+
+  if (!params?.id) {
+    return { message: 'not enough params: id', status: 404 };
+  }
+
+  const tokenData = await getToken({
+    uid: user?.uid,
+    token: params.id,
+    mode: 'a',
+    json: 1,
+  });
+
+  if (!tokenData && !config.local && !config.debug) {
+    return { message: 'token not allow', status: 403 };
+  }
+
+  const {
+    id, columns, formSetting, table, style,
+  } = await pg.query(`select dataset_id as id, table_name as table, column_list as columns, style, 
+      form_setting as "formSetting" from bi.dataset where dataset_id=$1`, [tokenData?.dataset || params.id])
+    .then(el => el.rows?.[0] || {});
+
+  if (!id) {
+    return { message: 'dataset not found', status: 404 };
+  }
+
+  if (datasetForms[id] && !query.nocache) {
+    return datasetForms[id];
+  }
+
+  const { columns: dbColumns = [] } = await getMeta(table) || {};
+  const isFilesColumn = dbColumns.find((el) => el.name === 'files' && pg.pgType[el.dataTypeID] === 'text');
+
+  const formSchema = formSetting
+    || ((columns || dbColumns).filter((col) => !systemColumns.includes(col.name))).reduce((acc, curr) => Object.assign(acc, {
+      [curr.name]: {
+        type: inputType[curr?.format || curr?.type || 'text'] || 'Text',
+        ua: curr?.description || curr?.title || curr?.name,
+        validators: curr?.is_required ? ['required'] : null,
+        options: curr?.values?.length ? curr?.values : null,
+        data: curr?.data,
+        min: curr?.type || curr?.format === 'number' ? 0 : null,
+      },
+    }), {});
+
+  Object.assign(formSchema, {
+    geom: {
+      type: 'Geom',
+      ua: 'Геометрія',
+      height: '500',
+      geom_edit_controol: [style?.type ? [style.type] : null],
+    },
+  });
+
+  if (isFilesColumn) {
+    Object.assign(formSchema, {
+      files: {
+        type: 'FileList',
+        ua: 'Файли',
+      },
+    });
+  }
+
+  datasetForms[id] = formSchema;
+
+  return formSchema;
+}

package/server/routes/dataset/controllers/format.js

@@ -0,0 +1,44 @@
+import getTableData from "../utils/getTableData.js";
+import getLayersData from "../utils/getLayersData.js";
+
+export default async function gisFormat({ query = {}, pg }) {
+  const time = [Date.now()];
+  const { layer, table, id, lat, lng } = query;
+
+  if (!table && !layer) {
+    return { message: 'not enough params', status: 400 };
+  }
+
+  if (layer && !(lat && lng)) {
+    return { message: 'not enough params: lat/lng', status: 400 };
+  }
+
+  if (table && !(id || layer)) {
+    return { message: 'not enough params: id', status: 400 };
+  }
+
+  const { message, template, rows, layers, html, body } = layer
+    ? await getLayersData({ pg, layer, table, id, lat, lng, time })
+    : await getTableData({ pg, table, id, time });
+
+  if (message) return { message, status: 400 };
+
+  return {
+    time: {
+      layer: time[1] - time[0],
+      data: time[2] - time[1],
+      html: time[3] - time[2],
+      total: time[3] - time[0],
+    },
+
+    id: rows[0]?.id,
+    table,
+    layers,
+
+    data: { geom: rows[0]?.geom },
+    rows,
+    html,
+    template,
+    body,
+  };
+}

package/server/routes/dataset/controllers/insert.js

@@ -0,0 +1,46 @@
+import {
+  config, getToken, dataInsert, pgClients,
+} from '@opengis/fastify-table/utils.js';
+
+import datasetForms from '../utils/datasetForms.js';
+
+export default async function datasetDataInsert({
+  pg = pgClients.client, params = {}, body = {}, user = {},
+}) {
+  if (!user?.uid) {
+    return { message: 'access restricted', status: 403 };
+  }
+
+  const tokenData = await getToken({
+    uid: user?.uid,
+    token: params.id,
+    mode: 'a',
+    json: 1,
+  });
+
+  if (!tokenData && !config.local && !config.debug) {
+    return { message: 'token not allow', status: 403 };
+  }
+
+  const dataset = await pg.query('select dataset_id as id, table_name as table from bi.dataset where dataset_id=$1', [tokenData?.dataset || params.id])
+    .then(el => el.rows[0] || {});
+
+  const { form, table: add } = tokenData
+    || { form: `${dataset.id}.form`, table: dataset.table };
+
+  if (!form || !dataset.id || !datasetForms[dataset.id]) {
+    return { message: 'dataset not found', status: 404 };
+  }
+
+  const res = await dataInsert({
+    table: add,
+    data: body,
+    uid: user?.uid,
+  });
+
+  if (!res) {
+    return { message: 'nothing added' };
+  }
+
+  return { rows: res.rows };
+}

package/server/routes/dataset/controllers/table.js

@@ -0,0 +1,69 @@
+import {
+    config, pgClients, getMeta, setToken, getToken,
+} from '@opengis/fastify-table/utils.js';
+  
+export default async function datasetFormData({
+    pg = pgClients.client, params = {}, query = {}, user = {},
+}) {  
+    if (!params?.id) {
+      return { message: 'not enough params: id', status: 400 };
+    }
+      
+    if (!user?.uid) {
+        return { message: 'access restricted', status: 403 };
+    }
+    
+    const { user_type: userType = 'regular' } = user;
+      
+    const tokenData = await getToken({
+        uid: user?.uid,
+        token: params.id,
+        mode: 'w',
+        json: 1,
+    });
+    
+    if (!tokenData && !params?.object_id) {
+        return { message: 'not enough params: object_id', status: 400 };
+    }
+  
+    const dataset = await pg.query(`select dataset_id as id, table_name as table, column_list as columns, filter_list as filters,
+      sql_list as sql, style, form_setting, pk, access_level from bi.dataset where dataset_id=$1`, [tokenData?.dataset || params.id]).then(el => el.rows[0] || {});
+  
+    if (!dataset?.id) {
+      return { message: 'dataset not found', status: 404 };
+    }
+
+    const tlist = await pg.query(`select array_agg((select nspname from pg_namespace where oid=relnamespace)||'.'||relname) tlist 
+      from pg_class where relkind in ('r','v')`).then(el => el.rows[0]?.tlist || []);
+  
+    if (!dataset.table || !tlist.includes(dataset.table.replace(/"/g, '')) || (!pg.pk?.[dataset.table.replace(/"/g, '')] && !dataset.pk)) {
+      return { message: `table not found: ${dataset.table}`, status: 404 };
+    }
+  
+    const { pk = dataset.pk, columns: dbColumns = [] } = await getMeta(dataset.table) || {};
+    const cols = dataset.columns?.filter((el) => el.name !== 'geom')?.map((el) => el.name || el)?.join(',');
+    
+    const where = [`"${pk}" = $1`, dataset.query].filter((el) => el);
+
+  const geom = dbColumns.find((el) => el.name === 'geom' && pg.pgType[el.dataTypeID] === 'geometry') ? ',st_asgeojson(geom)::json as geom' : '';
+  const files = dbColumns.find((el) => el.name === 'files' && pg.pgType[el.dataTypeID] === 'text') ? ',files' : '';
+    const q = `select "${pk}" as id, ${cols || '*'} ${files} ${geom} from ${dataset.table} t where ${where.join(' and ') || 'true'} limit 1`;
+  
+    if (query.sql === '1' && (config.debug || userType.includes('admin'))) { return q; }
+  
+    const data = await pg.query(q, [tokenData?.id || params.object_id]).then(el => el.rows?.[0] || {});
+  
+    data.token = setToken({
+        ids: [JSON.stringify({
+            id: data.id,
+            table: dataset.table,
+            form: `${dataset.id}.form`,
+            dataset: dataset.id,
+        })],
+        uid: user?.uid,
+        array: 1,
+    })[0];
+    
+    return data;
+}
+  

package/server/routes/dataset/controllers/update.js

@@ -0,0 +1,42 @@
+import {
+  config, getToken, dataUpdate, pgClients,
+} from '@opengis/fastify-table/utils.js';
+
+import datasetForms from '../utils/datasetForms.js';
+
+export default async function datasetDataUpdate({
+  pg = pgClients.client, params = {}, body = {}, user = {},
+}) {
+  if (!user?.uid) {
+    return { message: 'access restricted', status: 403 };
+  }
+
+  const tokenData = await getToken({
+    uid: user?.uid,
+    token: params.id,
+    mode: 'a',
+    json: 1,
+  });
+
+  if (!tokenData && !config.local && !config.debug) {
+    return { message: 'token not allow', status: 403 };
+  }
+
+  const dataset = await pg.query('select dataset_id as id, table_name as table from bi.dataset where dataset_id=$1', [tokenData?.dataset || params.id])
+    .then(el => el.rows[0] || {});
+
+  const table = tokenData?.table || dataset.table;
+
+  if (!dataset.id || !datasetForms[dataset.id]) {
+    return { message: 'dataset not found', status: 404 };
+  }
+
+  const res = await dataUpdate({
+    id: tokenData?.id || params?.object_id,
+    table,
+    data: body,
+    uid: user?.uid,
+  });
+
+  return res;
+}

package/server/routes/dataset/index.mjs

@@ -1,75 +1,120 @@
-import biDatasetDataList from './controllers/bi.dataset.data.list.js'; // get dataset structure + data
-import biDatasetDemoAdd from './controllers/bi.dataset.demo.add.js'; // import demo datasets
-import biDatasetList from './controllers/bi.dataset.list.js'; // get dataset list
-import biDatasetAdd from './controllers/bi.dataset.add.js'; // assign table / import data from body (vue sheet) or file (relative path)
-import biDatasetDel from './controllers/bi.dataset.del.js'; // delete dataset + data
-import biDatasetEdit from './controllers/bi.dataset.edit.js'; // edit dataset structure
-import biDbList from './controllers/bi.db.list.js'; // list db
-import biDatasetDataAdd from './controllers/bi.dataset.data.add.js';
-import biDatasetDataEdit from './controllers/bi.dataset.data.edit.js';
-import biDatasetDataDel from './controllers/bi.dataset.data.del.js';
+import dbTables from './controllers/dbTables.js';
+import dbTablePreview from './controllers/dbTablePreview.js';
 
-const biSchema = {};
+import createDatasetPost from './controllers/createDatasetPost.js';
+import editDataset from './controllers/editDataset.js';
+import datasetData from './controllers/data.js';
+import datasetFormData from './controllers/table.js';
+import datasetDataInsert from './controllers/insert.js';
+import datasetDataUpdate from './controllers/update.js';
+import datasetDataDelete from './controllers/delete.js';
+import datasetDataExport from './controllers/export.js';
+import datasetForm from './controllers/form.js';
+import format from './controllers/format.js';
+import datasetEditComment from './controllers/comment.js';
+import biDbList from './controllers/bi.db.list.js'; // list db
+import biDatasetList from './controllers/bi.dataset.list.js'; // list datasets
 
 export default async function route(fastify, opts) {
   fastify.route({
-    method: 'POST',
-    url: '/bi-datasets',
-    schema: biSchema,
-    handler: biDatasetAdd,
+    method: 'GET',
+    url: '/bi-db-list',
+    schema: {},
+    handler: biDbList,
   });
   fastify.route({
     method: 'GET',
-    url: '/bi-dataset-demo',
-    schema: biSchema,
-    handler: biDatasetDemoAdd,
+    url: '/bi-datasets',
+    schema: {},
+    handler: biDatasetList,
   });
   fastify.route({
     method: 'GET',
-    url: '/bi-db-list',
-    schema: biSchema,
-    handler: biDbList,
+    url: '/gis-format',
+    config: { policy: ['site'] },
+    handler: format,
   });
   fastify.route({
     method: 'GET',
-    url: '/bi-datasets',
-    schema: biSchema,
-    handler: biDatasetList,
+    url: '/db-tables',
+    schema: {},
+    config: { policy: ['site'] },
+    handler: dbTables,
   });
   fastify.route({
     method: 'GET',
-    url: '/bi-datasets/:id',
-    schema: biSchema,
-    handler: biDatasetDataList,
+    url: '/db-tables/:name',
+    schema: {},
+    config: { policy: ['site'] },
+    handler: dbTablePreview,
   });
   fastify.route({
-    method: 'DELETE',
-    url: '/bi-datasets/:id',
-    schema: biSchema,
-    handler: biDatasetDel,
+    method: 'POST',
+    url: '/dataset',
+    schema: {},
+    config: { policy: ['site'] },
+    handler: createDatasetPost,
   });
   fastify.route({
     method: 'PUT',
-    url: '/bi-datasets/:id',
-    schema: biSchema,
-    handler: biDatasetEdit,
+    url: '/dataset',
+    schema: {},
+    config: { policy: ['site'] },
+    handler: editDataset,
+  });
+  fastify.route({
+    method: 'GET',
+    url: '/dataset-data/:id',
+    schema: {},
+    config: { policy: ['site'] },
+    handler: datasetData,
+  });
+  fastify.route({
+    method: 'GET',
+    url: '/dataset-data/:id/:object_id',
+    schema: {},
+    config: { policy: ['site'] },
+    handler: datasetFormData,
   });
   fastify.route({
     method: 'POST',
-    url: '/bi-datasets/:id',
-    schema: biSchema,
-    handler: biDatasetDataAdd,
+    url: '/dataset-data/:id',
+    schema: {},
+    config: { policy: ['site'] },
+    handler: datasetDataInsert,
   });
   fastify.route({
     method: 'PUT',
-    url: '/bi-datasets/:id/:objectId',
-    schema: biSchema,
-    handler: biDatasetDataEdit,
+    url: '/dataset-data/:id/:object_id?',
+    schema: {},
+    config: { policy: ['site'] },
+    handler: datasetDataUpdate,
   });
   fastify.route({
     method: 'DELETE',
-    url: '/bi-datasets/:id/:objectId',
-    schema: biSchema,
-    handler: biDatasetDataDel,
+    url: '/dataset-data/:id/:object_id?',
+    schema: {},
+    config: { policy: ['site'] },
+    handler: datasetDataDelete,
+  });
+  fastify.route({
+    method: 'GET',
+    url: '/dataset-form/:id',
+    schema: {},
+    config: { policy: ['site'] },
+    handler: datasetForm,
+  });
+  fastify.route({
+    method: 'GET',
+    url: '/dataset-export/:id',
+    schema: {},
+    config: { policy: ['site'] },
+    handler: datasetDataExport,
+  });
+  fastify.route({
+    method: 'POST',
+    url: '/dataset-comment/:id',
+    config: { policy: ['site'] },
+    handler: datasetEditComment,
   });
 }