@opengis/bi 1.0.21 → 1.0.23

package/server/routes/dataset/utils/convertJSONToCSV.js

@@ -0,0 +1,17 @@
+import path from 'node:path';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+
+export default async function convertJSONToCSV({
+  filepathJSON, send = () => { }, delimiter = ';',
+}) {
+  if (!filepathJSON) return null;
+  const rows = JSON.parse(await readFile(filepathJSON) || '{}');
+  const values = rows.map((row) => Object.values(row).map((el) => `${el};`));
+  values.unshift(Object.keys(rows[0]).map((el) => `${el};`));
+
+  send('Конвертація успішно завершена. Формування файлу...');
+  await mkdir(path.dirname(filepathJSON), { recursive: true });
+  await writeFile(filepathJSON.replace('.json', '.csv'), `\ufeff${values.join('\n').replace(/;,/g, delimiter)}`, 'utf8');
+  send('Файл успішно сформовано. Оновіть сторінку або натисніть кнопку ще раз для завантаження. ', 1);
+  return { message: 'Файл успішно сформовано. Натистіть кнопку ще раз для завантаження даних', status: 200 };
+}

package/server/routes/dataset/utils/convertJSONToXls.js

@@ -0,0 +1,49 @@
+import path from 'path';
+import {
+  mkdir, readFile, writeFile,
+} from 'fs/promises';
+
+import { logger } from '@opengis/fastify-table/utils.js';
+
+import { grpc } from '@opengis/fastify-file/utils.js';
+
+const { jsonToXls } = grpc();
+
+export default async function convertJSONToXls({
+  filepathJSON, colmodel, domain, source, send = () => { },
+}) {
+  if (!filepathJSON) return null;
+
+  const rows = JSON.parse(await readFile(filepathJSON) || '{}');
+
+  if (!rows?.length) {
+    return { message: 'empty rows', status: 400 };
+  }
+
+  send('Конвертація файлу...');
+  try {
+    const { result } = await jsonToXls({
+      json: JSON.stringify(rows),
+      header: source,
+      subheader: `Завантажено з порталу ${domain} - ${new Date().toLocaleDateString()}. Всього: ${rows.length}`,
+      colmodel: JSON.stringify(colmodel),
+    });
+    send('Конвертація успішно завершена. Формування файлу...');
+    await mkdir(path.dirname(filepathJSON), { recursive: true });
+    await writeFile(filepathJSON.replace('.json', '.xlsx'), result, 'base64');
+    send('Файл успішно сформовано. Оновіть сторінку або натисніть кнопку ще раз для завантаження. ', 1);
+    return { message: 'Файл успішно сформовано. Натистіть кнопку ще раз для завантаження даних', status: 200 };
+  }
+  catch (err) {
+    send(`jsonToXLS err: ${err.toString()}`, 1);
+    logger.error({
+      name: 'export/table',
+      filepathJSON,
+      colmodel,
+      domain,
+      rows: rows.length,
+      error: err.toString(),
+    });
+    return { error: err.toString(), status: 500 };
+  }
+}

package/server/routes/dataset/utils/createTableQuery.js

@@ -0,0 +1,59 @@
+import { randomBytes, createHash } from 'node:crypto';
+
+const schema = 'data_user';
+
+const columnType = {
+  text: 'text',
+  select: 'text',
+  date: 'date',
+  'yes/no': 'boolean',
+  badge: 'text',
+  number: 'numeric',
+  tags: 'text[]',
+  geom: 'geom',
+};
+
+export default function createTable(columns, name) {
+  const tableName = randomBytes(64).toString('hex').substring(0, 24).replace(/^\d+/, '');
+  const table = `${schema}.${tableName}`;
+  const pkey = tableName.concat('_id');
+
+  columns?.forEach?.((el, i) => Object.assign(el, {
+    name: `col_${i}`, // el.name
+    type: columnType[el.format || 'text'] || 'text',
+  }));
+
+  const clsQuery = columns
+    ?.filter?.(col => col?.format === 'select' && col.data?.length)
+    ?.map(col => {
+      const data = JSON.stringify(col.data).replace(/'/g, "''");
+      Object.assign(col, { data: createHash('md5').update([tableName, col.name].join()).digest('hex') });
+      return `insert into admin.cls(clsid,name,type,module) values('${col.data}','${col.data}','json', '${table}');
+      insert into admin.cls(code,name,parent,icon,data,module) 
+      select value->>'id',value->>'text','${col.data}',value->>'icon',value->>'data', '${table}' 
+      from json_array_elements('${data}'::json)`;
+    })
+    ?.join(';');
+
+  const createQuery = `create table if not exists ${table} (
+  ${pkey} text not null default public.uuid_generate_v4(), 
+  geom public.geometry, ${columns ? `
+  ${columns?.map?.((el) => `${el.name} ${el.type}`).join(', ')},` : ''} 
+  cdate timestamp without time zone not null default now(), 
+  editor_date timestamp without time zone, 
+  uid text, 
+  editor_id text, 
+  files json, 
+  constraint ${table.replace(/\./g, '_')}_constraint_pkey PRIMARY KEY (${pkey}) )`;
+
+  const commentQuery = columns
+    ?.filter?.((el) => el.title)
+    ?.map((el) => `comment on column ${table}.${el.name} is '${el.title}'`)
+    ?.join(';');
+
+  return {
+    table,
+    pkey,
+    sql: [createQuery, commentQuery, (name ? `comment on table ${table} is '${name}'` : undefined), clsQuery].filter((el) => el).join(';'),
+  };
+}

package/server/routes/dataset/utils/datasetForms.js

@@ -0,0 +1 @@
+export default {};

package/server/routes/dataset/utils/descriptionList.js

@@ -0,0 +1,46 @@
+import { handlebarsSync, handlebars } from '@opengis/fastify-table/utils.js'
+
+function format(d, key, data) {
+    if (!key?.includes) return '';
+    if (d === true) return 'так';
+    if (d === false) return 'ні'
+
+    if (key.includes('{{')) {
+        return handlebarsSync.compile(key)(data);
+    }
+    if (!d) return '-';
+    return d;
+}
+export default async function descriptionList(data, opt) {
+    const { hash } = opt;
+
+    // no data
+    if (hash.nodata && !data) {
+        const noDataText = typeof hash.nodata == 'string' ? hash.nodata : 'no data';
+        return noDataText
+    }
+    if (!data) return 'empty data'
+    if (!hash.columns.split) return 'columns empty' + JSON.stringify(hash.columns)
+    const keys = hash.columns.split(hash.divider || '|').map(el => hash.comma ? el.trim().replace(new RegExp(hash.comma || '#', 'g'), ',') : el.trim());
+
+    const result = [];
+
+    for (let i = 0; i < keys.length; i += 2) {
+        const name = keys[i];
+        const nameHBS = name.includes('{{') ? handlebarsSync.compile(name)({ ...data, hash }) : false
+
+        if (!nameHBS && name.includes('{{')) continue;
+
+        const key = keys[i + 1];
+        const d1 = ['{{format', '{{select', '{{badge', '{{ifCond'].filter(el => key?.includes?.(el)).length ? await handlebars.compile(key)(data) || '-' : null
+        if (d1 || data[key]) {
+            result.push(`<div class="flex py-[16px] text-[14px] border-b border-[#E5E7EB">
+            <div class="text-[#6B7280] dark:text-white  font-[600] w-[50%]">${nameHBS || name}</div>
+            <div class="font-[400] text-black dark:text-[#bdbdbd]  w-[50%]">${d1 || format(data[key], key, data)}</div>
+            </div>
+            `);
+        }
+
+    }
+    return '<div class="w-full">' + result.join('') + '</div>';
+}

package/server/routes/dataset/utils/downloadRemoteFile.js

@@ -0,0 +1,58 @@
+import path from 'node:path';
+import { existsSync } from 'node:fs';
+import { rm, mkdir, writeFile } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+
+import { config, logger } from '@opengis/fastify-table/utils.js';
+
+export default async function downloadRemoteFile({
+  rootDir, url, table,
+}) {
+  if (!rootDir) {
+    return { error: 'param rootDir is required', status: 400 };
+  }
+  if (!url) {
+    return { error: 'param url is required', status: 400 };
+  }
+  if (!table) {
+    return { error: 'param table is required', status: 400 };
+  }
+
+  try {
+    const response = await fetch(url);
+
+    if (response?.status !== 200) {
+      return { message: 'file not found', status: response?.status };
+    }
+
+    const arrayBuffer = await response.arrayBuffer();
+    const buffer = Buffer.from(arrayBuffer);
+
+    const contentType = response.headers.get('Content-Type');
+
+    // eslint-disable-next-line newline-per-chained-call
+    const extName = contentType === 'text/plain' ? path.extname(url) : `.${response.headers.get('Content-Type')?.split(';')?.shift()?.split('/')?.pop()?.replace(/\+|\-|\./g, '')}`; // path.extname(url)
+
+    const filePath = `/files/tmp/${createHash('md5').update([url, table].join()).digest('hex')}${extName}`;
+    const fullPath = path.join(rootDir, filePath);
+
+    if (config?.local) {
+      console.log(url, fullPath, existsSync(fullPath));
+    }
+
+    await mkdir(path.dirname(fullPath), { recursive: true });
+
+    if (existsSync(fullPath) && config?.local) {
+      await rm(fullPath);
+    }
+
+    if (!existsSync(fullPath)) {
+      await writeFile(fullPath, buffer);
+    }
+    return { filePath };
+  }
+  catch (err) {
+    logger.file('dataset/create/error', { url, error: err.toString() });
+    return { error: err.toString(), status: 500 };
+  }
+}

package/server/routes/dataset/utils/executeQuery.js

@@ -0,0 +1,46 @@
+import { pgClients } from '@opengis/fastify-table/utils.js';
+
+import insertDataQuery from './insertDataQuery.js';
+
+const insertDataset = `insert into bi.dataset 
+(name, table_name, dataset_file_path, column_list, pk, data_source, uid)
+values($1,$2,$3,$4,$5,$6,$7) returning dataset_id`;
+
+const updateAppeal = 'update datasets_appeal.appeal set dataset_id=$1, data_key=$2 where ap_id=$3';
+
+export default async function executeQuery({
+  sql, data, name, id, table, relPath, columns, pkey, source, user, url, dataKey,
+}) {
+  const client = await pgClients.client.connect();
+  try {
+    await client.query('BEGIN');
+
+    // create table
+    await client.query(sql);
+
+    // insert data
+    if (data) {
+      const insertData = insertDataQuery(columns, data, table);
+      await client.query(insertData);
+    }
+
+    // create dataset
+    const args = [name || id, table, relPath, columns ? JSON.stringify(columns) : null, pkey, JSON.stringify({ type: source, appeal: id, url }), user?.uid];
+    const datasetId = await client.query(insertDataset, args).then(el => el.rows?.[0]?.dataset_id);
+
+    if (id) {
+      // update appeal for public only, admin w/o appeal
+      await client.query(updateAppeal, [datasetId, dataKey, id]);
+    }
+
+    await client.query('COMMIT');
+    return { datasetId };
+  }
+  catch (err) {
+    await client.query('ROLLBACK');
+    return { error: err.toString(), status: 500 };
+  }
+  finally {
+    client.release();
+  }
+}

package/server/routes/dataset/utils/getLayersData.js

@@ -0,0 +1,107 @@
+import descriptionList from "./descriptionList.js";
+import metaFormat from "./metaFormat.js";
+
+async function getDataByLatLng({ pg, layer, table, id, lat, lng, time = [] }) {
+    // const rclient = getRedis();
+    // const redisKey = `${pg.options.database}:gis-format:${layer}:${lat}:${lng}`;
+    // const cacheData = await rclient.get(redisKey);
+    // if (cacheData && !config.local) return JSON.parse(cacheData);
+
+    const { rows: styles = [] } = await pg.query(`select style_id as id, dataset 
+    from gis.style where style_id=any($1::text[])
+      union all
+    select dataset_id as id, json_build_array(json_build_object('pk', pk, 'table', table_name, 'query', query, 'card', setting->>'card', 'columns', column_list)) as dataset 
+    from bi.dataset a where dataset_id=any($1::text[])`, [layer.split(',')]);
+    time.push(Date.now());
+
+    if (!styles?.length) {
+        return { message: `style not found: ${layer}`, status: 400 };
+    }
+
+    const { srids } = await pg.queryCache?.('select json_agg(srid) as srids from spatial_ref_sys').then((res1) => res1.rows?.[0] || {}) || {};
+
+    const layers = styles.reduce((acc, curr) => {
+        curr.dataset?.filter((el) => el.table && (el.pk || pg.pk?.[el.table]))
+            ?.forEach((el) => acc.push({
+                layer: curr.id,
+                pk: el.pk || pg.pk?.[el.table],
+                key: el.key,
+                table: el.table,
+                query: el.query,
+                template: el.card,
+                column_list: el.column_list,
+                columns: el.columns || el.column_list?.map((el) => el.name)?.join(','),
+                srid: el.srid || '4326',
+                geom: el.geom || 'geom',
+                step: srids?.includes(el.srid - 0) && el.srid !== '4326' ? 10 : 0.0002,
+            }));
+        return acc;
+    }, [])
+        .filter((el) => el.pk && el.layer)
+        .filter((el) => table ? el.table === table : true);
+
+    if (!layers.length) {
+        return { message: 'data not found', status: 404 };
+    }
+
+    const point = `srid=4326;point(${lng} ${lat})`;
+
+    // ${el.columns?.split?.filter?.((el) => el) ? `,${el.columns?.split?.filter?.((el) => el) || ''}` : ''}
+
+    const q = layers.map((el) => `select ${el.pk}, '${el.layer}' as layer,
+    row_to_json(t) as data,
+    st_asgeojson(${el.geom})::json as geom from ${el.table} t 
+    where ${el.query || '1=1'} and ${id ? `${el.pk}::text='${id.replace(/'/g, "''")}'` : '1=1'}
+    and case
+      when ST_GeometryType(geom) in ('ST_Polygon','ST_MultiPolygon')
+      then st_intersects(geom,st_buffer('${point}',${el.step}))
+  
+      when ST_GeometryType(geom) in ('ST_Line','ST_MultiLineString')
+      then st_distance(geom,'${point}') < ${el.step}
+    else false end`).join(' union all ');
+
+    // if (config.local && config.debug) console.log(q);
+    const { rows: objects = [] } = await pg.query(q) || {};
+    time.push(Date.now());
+
+    const rows = objects.map((row) => {
+        const layerData = layers.find((el) => el.layer === row?.layer) || {};
+        const rowData = layerData?.columns?.split?.filter?.((el) => el) ? layerData?.columns?.split?.filter?.((el) => el).reduce((acc, curr) => Object.assign(acc, { [curr]: row.data?.[curr] }), {}) : row.data;
+        return {
+            ...rowData,
+            ...row,
+            ...layerData,
+            data: undefined,
+            id: layerData.pk ? row?.[layerData.pk] : undefined,
+            geom: row.geom,
+        };
+    });
+
+    const { pk, layer: layerId, card, template } = rows[0] || {};
+
+    const dataset = layers.find((el) => el.layer === rows?.[0]?.layer) || {};
+
+    const columnList = (dataset.column_list || dataset.columns)?.reduce?.((acc, curr) => Object.assign(acc, { [curr.name]: curr.title || curr.ua || curr.name }), {}) || {};
+
+    const columns = card
+        ? Object.keys(card).map(el => `${el}| ${card[el]}`).join('|')
+        : Object.keys(columnList)?.map((key) => `${columnList[key]}| ${key}`).join('|');
+
+    const html = columns && rows?.[0] ? await descriptionList(rows[0], { hash: { columns } }) : undefined;
+    time.push(Date.now());
+
+    await Promise.all(rows.map(async () => {
+        const layerData = layers.find((el) => el.layer === rows?.[0]?.layer) || {};
+        const columns = layerData.column_list || layerData.columns;
+        if (columns?.length && Array.isArray(columns)) {
+            await metaFormat({ rows, columns });
+        }
+    }));
+    time.push(Date.now());
+
+    const resp = { id: rows?.[0]?.[pk], layers: layerId, template, data: rows?.[0], rows, html };
+
+    // await rclient.set(redisKey, JSON.stringify(resp), 'EX', 5 * 60);
+    return resp;
+}
+export default getDataByLatLng

package/server/routes/dataset/utils/getTableData.js

@@ -0,0 +1,47 @@
+import { handlebars } from "@opengis/fastify-table/utils.js";
+
+import descriptionList from "./descriptionList.js";
+// import metaFormat from "./metaFormat.js";
+
+async function getDataById({ pg, table, id, time = [] }) {
+    const dataset = await pg.query(`select dataset_id as id, pk, query, table_name, column_list, setting
+    from bi.dataset where dataset_id=$1`, [table]).then((res) => res.rows[0] || {});
+    time.push(Date.now());
+
+    if (!dataset?.id) {
+        return { message: `dataset not found: ${table}`, status: 400 };
+    }
+
+    if (!dataset?.table_name) {
+        return { message: 'invalid dataset: empty params table', status: 400 };
+    }
+
+    const pk = dataset.pk || pg.pk?.[dataset.table_name];
+
+    const { rows = [] } = pk ? await pg.query(`select *, ${pk}, geom::json from ${dataset.table_name} where ${dataset.query || '1=1'} and ${pk}=$1`, [id]) || {} : {};
+    if (pk) rows.forEach((row) => Object.assign(row, { id: row?.[pk] }));
+    time.push(Date.now());
+
+    // sql columns
+    const sql = `select attname as name, pg_catalog.col_description(attrelid,attnum) as title, atttypid::regtype as type from  pg_catalog.pg_attribute a 
+                        where attrelid='${dataset.table_name}'::regclass and attnum>0 
+                        and attname not in ('editor_id','editor_date','cdate','geom','id','uid','cdate')
+                        and atttypid::regtype not in ('json','geometry')`;
+
+    // card or auto
+    const type = dataset.setting?.card_type || 'auto';
+    const list = dataset.setting?.card_list || [];
+    const { rows: all } = await pg.query(sql).then(el => el);
+    const columns = all.filter(el => type === 'list' ? list.includes(el.name) : true)
+        .map(el => `${el.title || el.name}| ${el.name}`).join('|')
+
+
+    const { body } = type === 'html' && dataset?.setting?.card ? await pg.query(`select body->>'body' as body from admin.doc_template where title=$1`, [dataset?.setting?.card]).then(el => el.rows[0] || {}) : {}
+
+    const html = body ? await handlebars.compile(body)(rows[0]) : await descriptionList(rows[0], { hash: { columns } });
+    time.push(Date.now());
+
+
+    return { id: rows?.[0]?.id, template: dataset?.setting?.card, rows, html, body };
+}
+export default getDataById

package/server/routes/dataset/utils/insertDataQuery.js

@@ -0,0 +1,12 @@
+export default function insertDataQuery(columns, data, table) {
+  const columnList = columns?.map((el) => el.name)?.join(',');
+  const features = data?.features || [];
+  if (!features?.length) return null;
+
+  const insertData = `INSERT into ${table} (geom ${columns?.length ? ','.concat(columnList) : ''}) SELECT public.ST_GeomFromGeoJSON((features->>'geometry')::json) AS geom 
+        ${columns?.length ? ','.concat(columns?.map((el) => `features->'properties'->>'${el.title}'`)?.join(',')) : ''} FROM 
+        (SELECT json_array_elements('${JSON.stringify(features).replace(/'/g, "''")}'::json) AS features)q`;
+
+  // console.log(insertData);
+  return insertData;
+}

package/server/routes/dataset/utils/metaFormat.js

@@ -0,0 +1,24 @@
+import { getSelectVal } from '@opengis/fastify-table/utils.js';
+
+export default async function metaFormat({ rows, columns }) {
+  const selectCols = columns?.filter?.((el) => el.data);
+  if (!selectCols?.length) return rows;
+
+  await Promise.all(selectCols.map(async (attr) => {
+    const values = [...new Set(rows?.map((el) => el[attr.name]).flat())].filter((el) => (typeof el === 'boolean' ? true : el));
+    if (!values.length) return null;
+
+    const cls = await getSelectVal({ name: attr.data, values });
+    if (!cls) return null;
+
+    rows.forEach(el => {
+      const val = el[attr.name]?.map?.(c => cls[c.toString()] || cls[c] || c) || cls[el[attr.name]?.toString()] || cls[el[attr.name]] || el[attr.name];
+      if (!val) return;
+      Object.assign(el, { [val?.color ? `${attr.name}_data` : `${attr.name}_text`]: (val.color ? val : val.text || val) });
+    });
+
+    return null;
+  }));
+
+  return rows;
+}

package/server/routes/edit/controllers/dashboard.add.js

@@ -16,9 +16,9 @@ export default async function widgetAdd({ pg, body }) {
 
     return {
       time: Date.now() - time,
-      message: `Added new dashboard, ID: '${res.rows[0].title}'`,
-      status: 200,
-      rows: res.rows,
+      id: res.rows?.[0].dashboard_id,
+      title: res.rows?.[0].title,
+      name: res.rows?.[0].name,
     };
   } catch (err) {
     return { error: err.toString(), status: 500 };

package/server/routes/edit/controllers/widget.add.js

@@ -1,3 +1,4 @@
+import { createHash } from 'node:crypto';
 import { dataInsert, dataUpdate } from "@opengis/fastify-table/utils.js";
 
 /* eslint-disable import/extensions */
@@ -48,24 +49,28 @@ export default async function widgetAdd({ pg, params = {}, body = {} }) {
         data?.metrics,
     });
 
+    data.widget_id = createHash('md5').update([data?.name, dashboardId].join()).digest('hex').substr(0, 10);
+
     const res = await dataUpdate({
       table: 'bi.dashboard',
       id: dashboardId,
       data: {
         widgets: [data].concat(row.widgets || []),
-        panels: [{ widget: data.name, col: data.col || 3 }].concat(
+        panels: [{ widget: data.name, col: data.col || 3, height: data.data?.height }].concat(
           row.panels || []
         ),
       },
     });
     const widgetData = { ...data, data, dashboard_id: dashboardId };
     if (body?.yml) Object.assign(widgetData, { yml: body.yml });
-    await dataInsert({
+    const test = await dataInsert({
       table: 'bi.widget',
       data: widgetData,
     });
     return {
-      message: `Added widget to ${dashboardName}`,
+      dashboard: dashboardName,
+      widgetId: data.widget_id,
+      widgetName: data.name,
       status: 200,
       rows: res,
     };

package/server/routes/edit/controllers/widget.edit.js

@@ -5,7 +5,8 @@ import { getWidget, yamlSafe } from '../../../../utils.js';
 export default async function widgetEdit({ pg, body, params, }) {
   const { widget: widgetName, name: dashboardName } = params;
   const data = body.yml && !body.style ? yamlSafe.load(body.yml) : body;
-
+  const yml = body.yml ?? yamlSafe.dump(body.data);
+  console.log(body.data, data, body.yml, yml);
 
 
   if (!widgetName || !dashboardName) {
@@ -15,8 +16,8 @@ export default async function widgetEdit({ pg, body, params, }) {
     };
   }
 
-  const { widget_id: widgetId } = await pg.query(
-    `select a.widget_id , b.dashboard_id from bi.widget a, bi.dashboard b
+  const { widget_id: widgetId, dashboard_id: dashboardId, widgets } = await pg.query(
+    `select a.widget_id , b.dashboard_id, b.widgets from bi.widget a, bi.dashboard b
     where $2 in (a.widget_id, a.name) and $1 in (b.dashboard_id, b.name) order by 1,2`,
     [dashboardName, widgetName]).then(res1 => res1.rows?.[0] || {});
 
@@ -35,8 +36,8 @@ export default async function widgetEdit({ pg, body, params, }) {
     return { message: 'bad params: table ' + tableName, status: 400 };
   }
 
-  if (body.yml) {
-    Object.assign(widgetData, { yml: body.yml })
+  if (yml) {
+    Object.assign(widgetData, { yml })
   }
 
   Object.assign(widgetData, { table_name: tableName });
@@ -47,6 +48,23 @@ export default async function widgetEdit({ pg, body, params, }) {
     data: widgetData,
   });
 
+  const idx = widgets?.findIndex?.(el => el.name === widgetName) || -1;
+  if (widgetData.hasOwnProperty('data') && idx > -1) {
+    widgets[idx].data = widgetData.data
+  }
+  if (widgetData.hasOwnProperty('yml') && idx > -1) {
+    widgets[idx].yml = widgetData.yml;
+  }
+  if (widgetData.hasOwnProperty('style') && idx > -1) {
+    widgets[idx].style = widgetData.style;
+  }
+
+  await dataUpdate({
+    table: 'bi.dashboard',
+    id: dashboardId,
+    data: { widgets },
+  });
+
   return {
     message: rows,
     status: 200,