@opengis/bi 1.2.31 → 1.2.32

package/server/routes/dataset/controllers/export.js

@@ -1,214 +1,214 @@
-/* eslint-disable no-await-in-loop */
-import path from 'node:path';
-import { createHash } from 'node:crypto';
-import { existsSync, rmSync } from 'node:fs';
-
-import { readFile, writeFile, mkdir } from 'node:fs/promises';
-
-import {
-  config, pgClients, getMeta, getFolder, getFilterSQL, eventStream, logger,
-} from '@opengis/fastify-table/utils.js';
-
-// import convertJSONToXls from '../utils/convertJSONToXls.js';
-import convertJSONToCSV from '../utils/convertJSONToCSV.js';
-import metaFormat from '../utils/metaFormat.js';
-
-export default async function datasetDataExport(req, reply) {
-  const {
-    pg = pgClients.client, params = {}, query = {}, user = {}, unittest,
-  } = req;
-
-  if (!user?.uid) {
-    return { message: 'access restricted', status: 403 };
-  }
-
-  const { user_type: userType = 'regular' } = user;
-
-  if (!params?.id) {
-    return { message: 'not enough params: id', status: 404 };
-  }
-
-  const { format = 'json' } = query;
-
-  const dataset = await pg.query(`select dataset_id as id, table_name as table, column_list as columns,
-      export_columns as "exportColumns", sql_list as sql from bi.dataset where dataset_id=$1`, [params.id])
-    .then(el => el.rows[0] || {});
-
-  if (!dataset?.id || !dataset.table) {
-    return { message: 'dataset not found', status: 404 };
-  }
-
-  const { pk = dataset.pk, view, columns: dbColumns = [] } = await getMeta(dataset.table);
-
-  if (!dataset.table || (!pg.pk?.[dataset.table] && !pk && !view)) {
-    return { message: `table not found: ${dataset.table}`, status: 404 };
-  }
-
-  const rootDir = getFolder(req, 'local');
-
-  const date = new Date();
-  const sufixName = `${query.filter || '1=1'}-${query.col || 'all'}-${query.search || '1=1'}-${query.limit || 'unlimited'}`;
-  const sufixDate = [date.getFullYear(), date.getMonth(), date.getDate(), date.getHours()].join('-');
-
-  const objInfo = createHash('md5').update([sufixName, sufixDate].join('-')).digest('hex');
-  const fileName = dataset.table.concat('_').concat(objInfo).concat('.').concat(format);
-  const filepath = path.join(rootDir, '/files/temp', fileName);
-
-  if (existsSync(filepath) && query.nocache) {
-    rmSync(filepath);
-  }
-
-  if (existsSync(filepath) && !query?.sql) {
-    const encoding = path.extname(filepath) === '.xlsx' ? 'base64' : 'utf8';
-    const data = await readFile(filepath, encoding);
-    return reply.headers({ 'Content-Disposition': `attachment; filename=${path.basename(filepath)}` }).send(Buffer.from(data, encoding));
-  }
-
-  const fData = await getFilterSQL({
-    filter: query.filter,
-    search: query?.search,
-    table: dataset.table,
-    json: 1,
-  });
-
-  const where = [(params.object_id ? ` "${pk}" = $1` : null), dataset.query, fData.q].filter((el) => el);
-
-  const cols = dataset.exportColumns
-    || dataset.columns?.map((el) => el.name);
-
-  if (!cols?.length) {
-    return { message: `empty export columns: ${dataset.id}`, status: 400 };
-  }
-
-  const sqlTable = dataset.sql
-    ?.filter?.((el) => !el?.disabled && el?.sql?.replace)
-    ?.map((el, i) => ` left join lateral (${el.sql.replace('{{uid}}', user?.uid)}) ${el.name || `t${i}`} on 1=1 `)
-    ?.join('') || '';
-
-  const { total = '0' } = await pg.query(`select count(*) as total from ${dataset.table} 
-        where ${dataset.query || 'true'} and ${params.object_id ? `${pk}='${params.object_id}'` : '1=1'}`)
-    .then(el => el.rows?.[0] || {});
-
-  if (total === '0') {
-    return { message: 'Немає даних, які можна експортувати', status: 200 };
-  }
-
-  const ext = path.extname(filepath);
-  const filepathJSON = ['csv', 'xlsx'].includes(format) ? filepath.replace(ext, '.json') : filepath;
-
-  const cacheFileJSON = existsSync(filepathJSON);
-
-  const send = +total < 10000 || unittest
-    ? console.log
-    : eventStream(reply);
-
-  if (cacheFileJSON && !query.sql) {
-    if (query.nocache) rmSync(filepathJSON); // delete old file, prevent append
-    if (!query.nocache && ['xlsx', 'csv'].includes(format)) {
-      send(`Знайдено файл формату json. Пропуск обробки даних. Початок конвертації в ${format}...`);
-      if (format === 'xlsx') {
-        return 'temporary not work';
-        return convertJSONToXls({
-          filepathJSON,
-          colmodel: dataset.columns,
-          domain: req.hostname,
-          source: dataset.table,
-          send,
-        });
-      }
-      if (format === 'csv') {
-        return convertJSONToCSV({ send, filepathJSON });
-      }
-    }
-  }
-
-  const limit = Math.min(query.limit, 10000) || 10000;
-
-  const optimizedSQL = `select  ${pk ? `"${pk}" as id,` : ''} *
-    ${['geojson', 'shp'].includes(format) && dbColumns.find((el) => el.name === 'geom' && pg.pgType[el.dataTypeID] === 'geometry') ? ',st_asgeojson(geom)::json as geom' : ''} 
-  from (select * from ${dataset.table} where ${sqlTable ? 'true' : (where.join(' and ') || 'true')}) t 
-  ${sqlTable} where ${where.join(' and ') || 'true'} offset 0 limit ${limit}`.replace(/{{uid}}/g, user?.uid);
-
-  if (query.sql === '1' && (config.debug || userType.includes('admin'))) return optimizedSQL;
-
-  const res = {};
-  let offset = 0;
-  let seq = 0;
-
-  send(`Всього в реєстрі: ${total}`);
-
-  while ((+total - offset > 0) && !res?.error) {
-    try {
-      const q = optimizedSQL.replace(/offset 0/g, `offset ${offset}`);
-      const { rows = [] } = await pg.query(q, (params.object_id ? [params.object_id] : null));
-
-      send(`Оброблено: ${offset}/${total}`);
-      send(`seq: ${++seq}`);
-      send(`Обробка ${rows.length} об'єктів...`);
-
-      if (!rows.length) {
-        send('Обробка даних успішно завершена');
-        break;
-      }
-
-      await metaFormat({ rows, columns: dataset.columns });
-
-      // skip non present after metaFormat
-      const allowColumnList = []; // 'id'
-      if (['geojson', 'shp'].includes(format)) {
-        allowColumnList.push('geom');
-      }
-      rows.forEach((row) => {
-        Object.keys(row)
-          .filter((el) => !allowColumnList.includes(el.name) && !cols.includes(el))
-          .forEach((key) => delete row[key]);
-      });
-
-      // convert csv / xlsx from json
-      if (!existsSync(filepathJSON)) { // if json not exists
-        await mkdir(path.dirname(filepath), { recursive: true });
-        await writeFile(filepathJSON, JSON.stringify(rows));
-      }
-      else { // if json exists
-        const jsonData = JSON.parse(await readFile(filepathJSON) || '{}');
-        const moreData = jsonData.concat(rows); // rewrite to appendFile?
-        await writeFile(filepathJSON, JSON.stringify(moreData));
-      }
-
-      offset += rows.length;
-    }
-    catch (err) {
-      Object.assign(res, { error: err.toString(), status: 500 });
-      send(`error: ${err.toString()}`, 1);
-      logger.error({
-        name: 'export/table',
-        filepathJSON,
-        total,
-        offset,
-        result: res,
-        error: err.toString(),
-      });
-      return err.toString();
-    }
-  }
-
-  if (!res?.error && format === 'csv') {
-    send('Сформовано файл формату json. Початок конвертації в csv...');
-    return convertJSONToCSV({ send, filepathJSON });
-  }
-
-  if (!res?.error && format === 'xlsx') {
-    send('Сформовано файл формату json. Початок конвертації в xlsx...');
-    return convertJSONToXls({
-      filepathJSON,
-      colmodel: dataset.columns,
-      domain: req.hostname,
-      source: dataset.table,
-      send,
-    });
-  }
-
-  const message = res.error || 'Файл успішно сформовано. Натистіть кнопку ще раз для завантаження даних';
-  send(message, 1);
-  return { message, status: 200 };
-}
+/* eslint-disable no-await-in-loop */
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+import { existsSync, rmSync } from 'node:fs';
+
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+
+import {
+  config, pgClients, getMeta, getFolder, getFilterSQL, eventStream, logger,
+} from '@opengis/fastify-table/utils.js';
+
+// import convertJSONToXls from '../utils/convertJSONToXls.js';
+import convertJSONToCSV from '../utils/convertJSONToCSV.js';
+import metaFormat from '../utils/metaFormat.js';
+
+export default async function datasetDataExport(req, reply) {
+  const {
+    pg = pgClients.client, params = {}, query = {}, user = {}, unittest,
+  } = req;
+
+  if (!user?.uid) {
+    return { message: 'access restricted', status: 403 };
+  }
+
+  const { user_type: userType = 'regular' } = user;
+
+  if (!params?.id) {
+    return { message: 'not enough params: id', status: 404 };
+  }
+
+  const { format = 'json' } = query;
+
+  const dataset = await pg.query(`select dataset_id as id, table_name as table, column_list as columns,
+      export_columns as "exportColumns", sql_list as sql from bi.dataset where dataset_id=$1`, [params.id])
+    .then(el => el.rows[0] || {});
+
+  if (!dataset?.id || !dataset.table) {
+    return { message: 'dataset not found', status: 404 };
+  }
+
+  const { pk = dataset.pk, view, columns: dbColumns = [] } = await getMeta(dataset.table);
+
+  if (!dataset.table || (!pg.pk?.[dataset.table] && !pk && !view)) {
+    return { message: `table not found: ${dataset.table}`, status: 404 };
+  }
+
+  const rootDir = getFolder(req, 'local');
+
+  const date = new Date();
+  const sufixName = `${query.filter || '1=1'}-${query.col || 'all'}-${query.search || '1=1'}-${query.limit || 'unlimited'}`;
+  const sufixDate = [date.getFullYear(), date.getMonth(), date.getDate(), date.getHours()].join('-');
+
+  const objInfo = createHash('md5').update([sufixName, sufixDate].join('-')).digest('hex');
+  const fileName = dataset.table.concat('_').concat(objInfo).concat('.').concat(format);
+  const filepath = path.join(rootDir, '/files/temp', fileName);
+
+  if (existsSync(filepath) && query.nocache) {
+    rmSync(filepath);
+  }
+
+  if (existsSync(filepath) && !query?.sql) {
+    const encoding = path.extname(filepath) === '.xlsx' ? 'base64' : 'utf8';
+    const data = await readFile(filepath, encoding);
+    return reply.headers({ 'Content-Disposition': `attachment; filename=${path.basename(filepath)}` }).send(Buffer.from(data, encoding));
+  }
+
+  const fData = await getFilterSQL({
+    filter: query.filter,
+    search: query?.search,
+    table: dataset.table,
+    json: 1,
+  });
+
+  const where = [(params.object_id ? ` "${pk}" = $1` : null), dataset.query, fData.q].filter((el) => el);
+
+  const cols = dataset.exportColumns
+    || dataset.columns?.map((el) => el.name);
+
+  if (!cols?.length) {
+    return { message: `empty export columns: ${dataset.id}`, status: 400 };
+  }
+
+  const sqlTable = dataset.sql
+    ?.filter?.((el) => !el?.disabled && el?.sql?.replace)
+    ?.map((el, i) => ` left join lateral (${el.sql.replace('{{uid}}', user?.uid)}) ${el.name || `t${i}`} on 1=1 `)
+    ?.join('') || '';
+
+  const { total = '0' } = await pg.query(`select count(*) as total from ${dataset.table} 
+        where ${dataset.query || 'true'} and ${params.object_id ? `${pk}='${params.object_id}'` : '1=1'}`)
+    .then(el => el.rows?.[0] || {});
+
+  if (total === '0') {
+    return { message: 'Немає даних, які можна експортувати', status: 200 };
+  }
+
+  const ext = path.extname(filepath);
+  const filepathJSON = ['csv', 'xlsx'].includes(format) ? filepath.replace(ext, '.json') : filepath;
+
+  const cacheFileJSON = existsSync(filepathJSON);
+
+  const send = +total < 10000 || unittest
+    ? console.log
+    : eventStream(reply);
+
+  if (cacheFileJSON && !query.sql) {
+    if (query.nocache) rmSync(filepathJSON); // delete old file, prevent append
+    if (!query.nocache && ['xlsx', 'csv'].includes(format)) {
+      send(`Знайдено файл формату json. Пропуск обробки даних. Початок конвертації в ${format}...`);
+      if (format === 'xlsx') {
+        return 'temporary not work';
+        return convertJSONToXls({
+          filepathJSON,
+          colmodel: dataset.columns,
+          domain: req.hostname,
+          source: dataset.table,
+          send,
+        });
+      }
+      if (format === 'csv') {
+        return convertJSONToCSV({ send, filepathJSON });
+      }
+    }
+  }
+
+  const limit = Math.min(query.limit, 10000) || 10000;
+
+  const optimizedSQL = `select  ${pk ? `"${pk}" as id,` : ''} *
+    ${['geojson', 'shp'].includes(format) && dbColumns.find((el) => el.name === 'geom' && pg.pgType[el.dataTypeID] === 'geometry') ? ',st_asgeojson(geom)::json as geom' : ''} 
+  from (select * from ${dataset.table} where ${sqlTable ? 'true' : (where.join(' and ') || 'true')}) t 
+  ${sqlTable} where ${where.join(' and ') || 'true'} offset 0 limit ${limit}`.replace(/{{uid}}/g, user?.uid);
+
+  if (query.sql === '1' && (config.debug || userType.includes('admin'))) return optimizedSQL;
+
+  const res = {};
+  let offset = 0;
+  let seq = 0;
+
+  send(`Всього в реєстрі: ${total}`);
+
+  while ((+total - offset > 0) && !res?.error) {
+    try {
+      const q = optimizedSQL.replace(/offset 0/g, `offset ${offset}`);
+      const { rows = [] } = await pg.query(q, (params.object_id ? [params.object_id] : null));
+
+      send(`Оброблено: ${offset}/${total}`);
+      send(`seq: ${++seq}`);
+      send(`Обробка ${rows.length} об'єктів...`);
+
+      if (!rows.length) {
+        send('Обробка даних успішно завершена');
+        break;
+      }
+
+      await metaFormat({ rows, columns: dataset.columns });
+
+      // skip non present after metaFormat
+      const allowColumnList = []; // 'id'
+      if (['geojson', 'shp'].includes(format)) {
+        allowColumnList.push('geom');
+      }
+      rows.forEach((row) => {
+        Object.keys(row)
+          .filter((el) => !allowColumnList.includes(el.name) && !cols.includes(el))
+          .forEach((key) => delete row[key]);
+      });
+
+      // convert csv / xlsx from json
+      if (!existsSync(filepathJSON)) { // if json not exists
+        await mkdir(path.dirname(filepath), { recursive: true });
+        await writeFile(filepathJSON, JSON.stringify(rows));
+      }
+      else { // if json exists
+        const jsonData = JSON.parse(await readFile(filepathJSON) || '{}');
+        const moreData = jsonData.concat(rows); // rewrite to appendFile?
+        await writeFile(filepathJSON, JSON.stringify(moreData));
+      }
+
+      offset += rows.length;
+    }
+    catch (err) {
+      Object.assign(res, { error: err.toString(), status: 500 });
+      send(`error: ${err.toString()}`, 1);
+      logger.error({
+        name: 'export/table',
+        filepathJSON,
+        total,
+        offset,
+        result: res,
+        error: err.toString(),
+      });
+      return err.toString();
+    }
+  }
+
+  if (!res?.error && format === 'csv') {
+    send('Сформовано файл формату json. Початок конвертації в csv...');
+    return convertJSONToCSV({ send, filepathJSON });
+  }
+
+  if (!res?.error && format === 'xlsx') {
+    send('Сформовано файл формату json. Початок конвертації в xlsx...');
+    return convertJSONToXls({
+      filepathJSON,
+      colmodel: dataset.columns,
+      domain: req.hostname,
+      source: dataset.table,
+      send,
+    });
+  }
+
+  const message = res.error || 'Файл успішно сформовано. Натистіть кнопку ще раз для завантаження даних';
+  send(message, 1);
+  return { message, status: 200 };
+}

package/server/routes/dataset/controllers/form.js

@@ -1,99 +1,99 @@
-import {
-  config, pgClients, getMeta, getToken,
-} from '@opengis/fastify-table/utils.js';
-
-import datasetForms from '../utils/datasetForms.js';
-
-const inputType = {
-  text: 'Text',
-  autocomplete: 'Autocomplete',
-  select: 'Autocomplete',
-  date: 'DatePicker',
-  'yes/no': 'Switcher',
-  badge: 'Select',
-  number: 'Number',
-  tags: 'Tags',
-  file: 'File',
-};
-
-const systemColumns = [
-  'uid',
-  'cdate',
-  'editor_id',
-  'editor_date',
-  'files',
-];
-
-export default async function datasetForm({
-  pg = pgClients.client, params = {}, query = {}, user = {},
-}) {
-  if (!user?.uid) {
-    return { message: 'access restricted', status: 403 };
-  }
-
-  if (!params?.id) {
-    return { message: 'not enough params: id', status: 404 };
-  }
-
-  const tokenData = await getToken({
-    uid: user?.uid,
-    token: params.id,
-    mode: 'a',
-    json: 1,
-  });
-
-  if (!tokenData && !config.local && !config.debug) {
-    return { message: 'token not allow', status: 403 };
-  }
-
-  const {
-    id, columns, formSetting, table, style,
-  } = await pg.query(`select dataset_id as id, table_name as table, column_list as columns, style, 
-      form_setting as "formSetting" from bi.dataset where dataset_id=$1`, [tokenData?.dataset || params.id])
-    .then(el => el.rows?.[0] || {});
-
-  if (!id) {
-    return { message: 'dataset not found', status: 404 };
-  }
-
-  if (datasetForms[id] && !query.nocache) {
-    return datasetForms[id];
-  }
-
-  const { columns: dbColumns = [] } = await getMeta(table) || {};
-  const isFilesColumn = dbColumns.find((el) => el.name === 'files' && pg.pgType[el.dataTypeID] === 'text');
-
-  const formSchema = formSetting
-    || ((columns || dbColumns).filter((col) => !systemColumns.includes(col.name))).reduce((acc, curr) => Object.assign(acc, {
-      [curr.name]: {
-        type: inputType[curr?.format || curr?.type || 'text'] || 'Text',
-        ua: curr?.description || curr?.title || curr?.name,
-        validators: curr?.is_required ? ['required'] : null,
-        options: curr?.values?.length ? curr?.values : null,
-        data: curr?.data,
-        min: curr?.type || curr?.format === 'number' ? 0 : null,
-      },
-    }), {});
-
-  Object.assign(formSchema, {
-    geom: {
-      type: 'Geom',
-      ua: 'Геометрія',
-      height: '500',
-      geom_edit_controol: [style?.type ? [style.type] : null],
-    },
-  });
-
-  if (isFilesColumn) {
-    Object.assign(formSchema, {
-      files: {
-        type: 'FileList',
-        ua: 'Файли',
-      },
-    });
-  }
-
-  datasetForms[id] = formSchema;
-
-  return formSchema;
-}
+import {
+  config, pgClients, getMeta, getToken,
+} from '@opengis/fastify-table/utils.js';
+
+import datasetForms from '../utils/datasetForms.js';
+
+const inputType = {
+  text: 'Text',
+  autocomplete: 'Autocomplete',
+  select: 'Autocomplete',
+  date: 'DatePicker',
+  'yes/no': 'Switcher',
+  badge: 'Select',
+  number: 'Number',
+  tags: 'Tags',
+  file: 'File',
+};
+
+const systemColumns = [
+  'uid',
+  'cdate',
+  'editor_id',
+  'editor_date',
+  'files',
+];
+
+export default async function datasetForm({
+  pg = pgClients.client, params = {}, query = {}, user = {},
+}) {
+  if (!user?.uid) {
+    return { message: 'access restricted', status: 403 };
+  }
+
+  if (!params?.id) {
+    return { message: 'not enough params: id', status: 404 };
+  }
+
+  const tokenData = await getToken({
+    uid: user?.uid,
+    token: params.id,
+    mode: 'a',
+    json: 1,
+  });
+
+  if (!tokenData && !config.local && !config.debug) {
+    return { message: 'token not allow', status: 403 };
+  }
+
+  const {
+    id, columns, formSetting, table, style,
+  } = await pg.query(`select dataset_id as id, table_name as table, column_list as columns, style, 
+      form_setting as "formSetting" from bi.dataset where dataset_id=$1`, [tokenData?.dataset || params.id])
+    .then(el => el.rows?.[0] || {});
+
+  if (!id) {
+    return { message: 'dataset not found', status: 404 };
+  }
+
+  if (datasetForms[id] && !query.nocache) {
+    return datasetForms[id];
+  }
+
+  const { columns: dbColumns = [] } = await getMeta(table) || {};
+  const isFilesColumn = dbColumns.find((el) => el.name === 'files' && pg.pgType[el.dataTypeID] === 'text');
+
+  const formSchema = formSetting
+    || ((columns || dbColumns).filter((col) => !systemColumns.includes(col.name))).reduce((acc, curr) => Object.assign(acc, {
+      [curr.name]: {
+        type: inputType[curr?.format || curr?.type || 'text'] || 'Text',
+        ua: curr?.description || curr?.title || curr?.name,
+        validators: curr?.is_required ? ['required'] : null,
+        options: curr?.values?.length ? curr?.values : null,
+        data: curr?.data,
+        min: curr?.type || curr?.format === 'number' ? 0 : null,
+      },
+    }), {});
+
+  Object.assign(formSchema, {
+    geom: {
+      type: 'Geom',
+      ua: 'Геометрія',
+      height: '500',
+      geom_edit_controol: [style?.type ? [style.type] : null],
+    },
+  });
+
+  if (isFilesColumn) {
+    Object.assign(formSchema, {
+      files: {
+        type: 'FileList',
+        ua: 'Файли',
+      },
+    });
+  }
+
+  datasetForms[id] = formSchema;
+
+  return formSchema;
+}