@opengis/bi 1.0.21 → 1.0.23

package/server/routes/dataset/controllers/comment.js

@@ -0,0 +1,55 @@
+import { pgClients } from '@opengis/fastify-table/utils.js';
+
+export default async function datasetEditComment({
+  pg = pgClients.client,
+  body = {},
+  params = {},
+}) {
+  if (!params?.id) {
+    return { message: 'not enough params: id', status: 400 };
+  }
+
+  if (!Array.isArray(body.data) || !body.data?.length) {
+    return { message: 'invalid param: body.data not an array / empty', status: 400 };
+  }
+
+  const dataset = await pg.query('select dataset_id as id, table_name as table from bi.dataset where dataset_id=$1', [params.id])
+    .then(el => el.rows?.[0] || {});
+
+  if (!dataset?.table) {
+    return { message: dataset?.id ? 'dataset table not set' : 'dataset not found', status: 404 };
+  }
+
+  const { fields = [] } = pg.pk?.[dataset.table] ? await pg.query(`select * from ${dataset.table} limit 0`) : {};
+
+  if (!fields.length) {
+    return { message: `table not found: ${dataset.table}`, status: 404 };
+  }
+
+  const columnList = fields.map(el => el.name);
+  const validData = body.data?.filter?.(el => el?.name && el?.comment && columnList.includes(el.name));
+
+  const q1 = validData.map(el => `COMMENT ON COLUMN ${dataset.table}."${el.name.replace(/'/g, "''")}" IS '${el.comment.replace(/'/g, "''")}'`).join(';');
+  if (q1) await pg.query(q1);
+
+  const q2 = `select
+    attrelid::regclass,
+    attname,
+    pg_catalog.col_description(attrelid,attnum) as title
+  from pg_catalog.pg_attribute a 
+  where attnum > 0 
+  and attname not in ('editor_id','editor_date','cdate','geom','id','uid','cdate')
+  and atttypid::regtype not in ('json','geometry')
+  and pg_catalog.col_description(attrelid,attnum) is not null`;
+
+  const { rows = [] } = await pg.query(q2);
+
+  const title = rows
+    .filter(el => el.attrelid === dataset.table)
+    .reduce((p, el) => ({ ...p, [el.attname]: el.title }), {});
+
+  const res = await pg.query(`update bi.dataset set setting=coalesce(setting::jsonb, '{}'::jsonb)||$1::jsonb 
+  where dataset_id=$2 returning dataset_id as id, setting`, [{ title }, dataset.id]);
+
+  return res.rows?.[0];
+}

package/server/routes/dataset/controllers/createDatasetPost.js

@@ -0,0 +1,132 @@
+import path from 'node:path';
+import { existsSync } from 'node:fs';
+
+import { config, pgClients, getFolder } from '@opengis/fastify-table/utils.js';
+import { file2json } from '@opengis/fastify-file/utils.js';
+
+import createTableQuery from '../utils/createTableQuery.js';
+import executeQuery from '../utils/executeQuery.js';
+import downloadRemoteFile from '../utils/downloadRemoteFile.js';
+
+const insertDataset = `insert into bi.dataset 
+(name, table_name, dataset_file_path, column_list, pk, data_source, uid)
+values($1,$2,$3,$4,$5,$6,$7) returning dataset_id`;
+
+export default async function createDatasetPost({
+  pg = pgClients.client, body = {}, user = {},
+}) {
+  if (!user?.uid) {
+    return { message: 'access restricted', status: 403 };
+  }
+
+  if (!body?.name) {
+    return { message: 'not enough query params: name', status: 400 };
+  }
+
+  if (!body?.table_name && !body?.file && !body?.column_list?.length && !body?.dataset_url) {
+    return { message: 'not enough query params: table / file / column_list/ url', status: 400 };
+  }
+
+  const {
+    name,
+    table_name: existingTable,
+    column_list: columns = [],
+    dataset_url: datasetUrl,
+  } = body;
+
+  const rootDir = getFolder(config, 'local');
+
+  if (datasetUrl) {
+    const { filePath, error } = await downloadRemoteFile({
+      rootDir, url: datasetUrl, table: name || datasetUrl,
+    });
+    if (error || !filePath) {
+      return { message: error || 'file request URL error', status: 500 };
+    }
+    Object.assign(body, { file: filePath });
+  }
+
+  const { file: relPath } = body;
+
+  if (relPath) {
+    const filepath = path.join(rootDir, relPath);
+    const exists = existsSync(filepath);
+
+    if (!exists) {
+      return { message: 'Файл з вихідними даними не знайдено', status: 404 };
+    }
+
+    const json = await file2json({ filepath });
+
+    // excel sheets fix?
+    const data1 = (['.xls', '.xlsx'].includes(path.extname(filepath)) && !Array.isArray(json)) ? json[Object.keys(json)[0]] : json;
+    const data = path.extname(filepath) === '.json' && body?.key ? data1?.[body.key] : data1;
+
+    const features = ['.csv', '.xlsx', '.xls'].includes(path.extname(filepath))
+      ? data?.map?.((el) => ({ type: 'Feature', properties: Object.keys(el).reduce((acc, curr) => Object.assign(acc, { [curr]: el[curr] }), {}) }))
+      : data?.features || data?.map?.((el) => ({ type: 'Feature', geometry: el.geom, properties: Object.keys(el).filter((key) => key !== 'geom').reduce((acc, curr) => Object.assign(acc, { [curr]: el[curr] }), {}) }));
+
+    if (!Array.isArray(features) || !features?.length) {
+      return { message: 'Файл з вихідними даними порожній', status: 400 };
+    }
+
+    Object.assign(data, { features });
+
+    const fileColumns = Object.keys(data?.features[0]?.properties)
+      ?.filter((el) => !['editor_date', 'cdate', 'uid', 'editor_id', 'files'].includes(el.toLowerCase()))
+      ?.map((el) => ({ title: el, format: 'text' }));
+
+    const { sql, pkey, table } = createTableQuery(fileColumns, name);
+
+    const { datasetId, error } = await executeQuery({
+      sql,
+      data,
+      name,
+      table,
+      relPath,
+      columns: fileColumns,
+      pkey,
+      source: datasetUrl ? 'url' : 'file',
+      url: datasetUrl,
+      user,
+      dataKey: body?.key,
+    });
+
+    if (error) return { error, status: 500 };
+
+    pgClients.client.pk[table] = pkey;
+    pgClients.client.tlist.push(table);
+    return { message: { id: datasetId, table, source: datasetUrl ? 'url' : 'file' }, status: 200 };
+  }
+
+  if (existingTable) {
+    await pg.query(`alter table ${existingTable} add column if not exists geom public.geometry;
+    alter table ${existingTable} add column if not exists files json`);
+    const args = [name, existingTable, null, null, pg.pk?.[existingTable], JSON.stringify({ type: 'table' }), user?.uid];
+    const datasetId = await pg.query(insertDataset, args).then(el => el.rows?.[0]?.dataset_id);
+    return { message: { id: datasetId, table: existingTable, source: 'table' }, status: 200 };
+  }
+
+  if (!columns?.length) {
+    return { message: 'У даній заяві відсутні налаштування структури набору даних', status: 400 };
+  }
+
+  const { sql, pkey, table } = createTableQuery(columns, name);
+
+  const { datasetId, error } = await executeQuery({
+    sql,
+    name,
+    table,
+    relPath,
+    columns,
+    pkey,
+    source: 'newtable',
+    user,
+  });
+
+  if (error) return { error, status: 500 };
+
+  pgClients.client.pk[table] = pkey;
+  pgClients.client.tlist.push(table);
+  return { message: { id: datasetId, table, source: 'newtable' }, status: 200 };
+}

package/server/routes/dataset/controllers/data.js

@@ -0,0 +1,145 @@
+import {
+  config, pgClients, getMeta, getFilterSQL, setToken, getToken,
+} from '@opengis/fastify-table/utils.js';
+
+import metaFormat from '../utils/metaFormat.js';
+import datasetFormData from './table.js';
+
+const maxLimit = 100;
+
+export default async function datasetData(req, reply) {
+  const {
+    pg = pgClients.client, params = {}, query = {}, user = {},
+  } = req;
+  const time = Date.now();
+  const { uid, user_type: userType = 'regular' } = user;
+
+  if (!uid) {
+    return { message: 'access restricted', status: 403 };
+  }
+
+  if (!params?.id) {
+    return { message: 'not enough params: id', status: 404 };
+  }
+
+  const tokenData = await getToken({
+    uid: user?.uid,
+    token: params.id,
+    mode: 'w',
+    json: 1,
+  });
+
+  if (tokenData?.dataset && tokenData?.id) {
+    return datasetFormData(req, reply);
+  }
+
+  const dataset = await pg.query(`select dataset_id as id, table_name as table, column_list as columns, filter_list as filters,
+    sql_list as sql, style, form_setting, pk, access_level from bi.dataset where dataset_id=$1`, [params.id]).then(el => el.rows[0] || {});
+
+  if (!dataset?.id) {
+    return { message: 'dataset not found', status: 404 };
+  }
+
+  const tlist = await pg.query(`select array_agg((select nspname from pg_namespace where oid=relnamespace)||'.'||relname) tlist 
+    from pg_class where relkind in ('r','v')`).then(el => el.rows[0]?.tlist || []);
+
+  if (!dataset.table || !tlist.includes(dataset.table.replace(/"/g, '')) || (!pg.pk?.[dataset.table.replace(/"/g, '')] && !dataset.pk)) {
+    return { message: `table not found: ${dataset.table}`, status: 404 };
+  }
+
+  const { pk = dataset.pk, columns: dbColumns = [] } = await getMeta(dataset.table) || {};
+  const cols = dataset.columns?.filter((el) => el.name !== 'geom')?.map((el) => el.name || el)?.join(',');
+
+  const limit = Math.min(maxLimit, +(query.limit || 20));
+  const offset = query.page && query.page > 0 ? ` offset ${(query.page - 1) * limit}` : '';
+
+  const checkFilter = [query.filter, query.search].filter((el) => el).length;
+
+  const fData = checkFilter ? await getFilterSQL({
+    table: dataset.table,
+    filter: query.filter,
+    search: query.search,
+    json: 1,
+  }) : {};
+
+  const where = [dataset.query, fData.q].filter((el) => el);
+
+  const sqlTable = dataset.sql
+    ?.filter?.((el) => !el?.disabled && el?.sql?.replace)
+    ?.map((el, i) => ` left join lateral (${el.sql.replace('{{uid}}', uid)}) ${el.name || `t${i}`} on 1=1 `)
+    ?.join('') || '';
+
+  const columnList = dbColumns.map((el) => el.name || el).join(',');
+  const order = columnList.includes('cdate') ? (`order by cdate ${query.desc ? 'desc' : ''}`) : '';
+
+  const q = `select 
+  ${cols || '*'} 
+    ${pk ? `,"${pk}" as id` : ''}
+    ${dbColumns.find((el) => el.name === 'files' && pg.pgType[el.dataTypeID] === 'text') ? ',files' : ''} 
+    ${dbColumns.find((el) => el.name === 'geom' && pg.pgType[el.dataTypeID] === 'geometry') ? ',st_asgeojson(geom)::json as geom' : ''} 
+    from (select * from ${dataset.table} where ${sqlTable ? 'true' : (where.join(' and ') || 'true')}  ${order}) t 
+    ${sqlTable} where ${where.join(' and ') || 'true'} ${order} ${offset}  limit ${limit}`.replace(/{{uid}}/g, uid);
+
+  if (query.sql === '1' && (config.debug || userType.includes('admin'))) { return q; }
+
+  // console.log(pg.options.database, q);
+  const { rows = [] } = await pg.query(q);
+
+  const qCount = `select 
+    count(*)::int as total,  
+    count(*) FILTER(WHERE ${[fData.q].filter(el => el).join(' and ') || 'true'})::int as filtered 
+    from ${dataset.table} t ${sqlTable} 
+    where ${[dataset.query].filter(el => el).join(' and ') || 'true'} `.replace(/{{uid}}/g, uid);
+
+  const { total, filtered } = await pg.queryCache(qCount).then(el => el?.rows?.[0] || {});
+
+  await metaFormat({ rows, columns: dataset.columns });
+
+  if (rows?.length && uid) {
+    rows.filter((row) => row[pk] || row.id).forEach((row) => Object.assign(row, {
+      token: setToken({
+        ids: [JSON.stringify({
+          id: row[pk] || row.id,
+          table: dataset.table,
+          form: `${params.id}.form`,
+          dataset: params.id,
+        })],
+        uid,
+        array: 1,
+      })[0],
+    }));
+  }
+  const res = {
+    time: Date.now() - time,
+    access: dataset.access_level,
+    card: dataset.card,
+    actions: ['add', 'edit', 'del', 'get'],
+    total,
+    filtered,
+    count: rows.length,
+    pk,
+    table: dataset.table,
+    form: `${params.id}.form`,
+    rows,
+    columns: dataset.columns || dbColumns.map(col => ({ name: col.name, format: pg.pgType[col.dataTypeID], title: col.name })),
+    filters: dataset.filters,
+  };
+
+  if (uid) {
+    const addTokens = setToken({
+      ids: [JSON.stringify({
+        table: dataset.table,
+        form: `${params.id}.form`,
+        dataset: dataset.id,
+      })],
+      uid,
+      array: 1,
+    });
+
+    Object.assign(res, {
+      addToken: addTokens[0],
+    });
+  }
+
+  return res;
+}

package/server/routes/{db → dataset}/controllers/dbTablePreview.js

@@ -16,28 +16,21 @@ export default async function dbTablePreview({ pg, params = {}, query = {} }) {
   if (query.sql) return q;
   try {
     const { table, columns } = await pg
-      .query(q, [params.name])
-      .then((res) => res.rows?.[0] || {});
+      .query(q, [params.name.replace(/"/g, '')])
+      .then(el => el.rows?.[0] || {});
     if (!table) {
       return { message: 'table not found', status: 404 };
     }
 
-    const { count = 0 } = await pg
-      .query(
-        `select reltuples as count from pg_class where oid  = to_regclass($1)`,
-        [params.name]
-      )
-      .then((res) => res.rows?.[0] || {});
+    const { count = 0 } = await pg.query('select reltuples as count from pg_class where oid  = to_regclass($1)', [params.name])
+      .then(el => el.rows?.[0] || {});
     const geom = columns.find((el) => el.type === 'geometry')?.name;
     const { bounds, extentStr } = geom
-      ? await pg
-          .query(
-            `select count(*),
-      st_asgeojson(st_extent(${geom}))::json as bounds,
-      replace(regexp_replace(st_extent(${geom})::box2d::text,'BOX\\(|\\)','','g'),' ',',') as "extentStr"
-    from ${params.name}`
-          )
-          .then((res) => res.rows?.[0] || {})
+      ? await pg.query(`select 
+        count(*),
+        st_asgeojson(st_extent(${geom}))::json as bounds,
+        replace(regexp_replace(st_extent(${geom})::box2d::text,'BOX\\(|\\)','','g'),' ',',') as "extentStr"
+      from ${params.name}`).then(el => el.rows?.[0] || {})
       : {};
     const extent = extentStr ? extentStr.split(',') : undefined;
 
@@ -49,15 +42,15 @@ export default async function dbTablePreview({ pg, params = {}, query = {} }) {
       'editor_id',
       geom,
     ];
-    const columnList = columns
-      .map((el) => el?.name)
-      .filter((el) => !systemColumns.includes(el));
-    const { rows = [] } = await pg.query(
-      `select ${columnList.join(',')} ${geom ? `, st_asgeojson(geom)::json as geom` : ''} from ${table} limit 10`
-    );
 
-    return { count, geom: !!geom, bounds, extent, columns, rows };
-  } catch (err) {
+    const columnList = columns.map((el) => el?.name).filter((el) => !systemColumns.includes(el));
+    const { rows = [] } = await pg.query(`select ${columnList.map(el => `"${el.replace(/'/g, "''")}"`).join(',')} ${geom ? ', st_asgeojson(geom)::json as geom' : ''} from ${params.name.replace(/'/g, '')} limit 10`);
+
+    return {
+      count, geom: !!geom, bounds, extent, columns, rows,
+    };
+  }
+  catch (err) {
     return { error: err.toString(), status: 500 };
   }
 }

package/server/routes/{db → dataset}/controllers/dbTables.js

@@ -1,6 +1,6 @@
 export default async function dbTables({ pg, query = {} }) {
   const q = `select 
-  t.table_schema ||'.'|| t.table_name as table, 
+  t.table_schema ||'."'|| t.table_name ||'"' as table, 
   obj_description(to_regclass(t.table_schema ||'."'|| t.table_name||'"')) as description,
   t.table_schema as schema,
   (select reltuples from pg_class where oid  = to_regclass(t.table_schema ||'."'|| t.table_name||'"') ) as total,
@@ -22,15 +22,11 @@ export default async function dbTables({ pg, query = {} }) {
     INNER JOIN pg_catalog.pg_namespace nsp ON nsp.oid = connamespace 
     WHERE nsp.nspname = t.table_schema AND rel.relname = t.table_name and contype='p'
   )
-  and ${query.schema ? `t.table_schema=$1` : '1=1'} order by total desc`;
+  and isgeom
+  order by total desc`;
+
   if (query.sql) return q;
-  try {
-    const { rows = [] } = await pg.query(
-      q,
-      [query.schema].filter((el) => el)
-    );
-    return { rows };
-  } catch (err) {
-    return { error: err.toString(), status: 500 };
-  }
+
+  const { rows = [] } = await pg.queryCache(q, { time: 0 });
+  return { rows };
 }

package/server/routes/dataset/controllers/delete.js

@@ -0,0 +1,39 @@
+import {
+  config, getToken, dataDelete, pgClients,
+} from '@opengis/fastify-table/utils.js';
+
+export default async function datasetDataDelete({
+  pg = pgClients.client, params = {}, user = {},
+}) {
+  if (!user?.uid) {
+    return { message: 'access restricted', status: 403 };
+  }
+
+  const tokenData = await getToken({
+    uid: user?.uid,
+    token: params.id,
+    mode: 'a',
+    json: 1,
+  });
+
+  if (!tokenData && !config.local && !config.debug) {
+    return { message: 'token not allow', status: 403 };
+  }
+
+  const dataset = await pg.query('select dataset_id as id, table_name as table from bi.dataset where dataset_id=$1', [tokenData?.dataset || params.id])
+    .then(el => el.rows[0] || {});
+
+  const table = tokenData?.table || dataset.table;
+
+  if (!dataset.id) {
+    return { message: 'dataset not found', status: 404 };
+  }
+
+  const res = await dataDelete({
+    id: tokenData?.id || params?.object_id,
+    table,
+    uid: user?.uid,
+  });
+
+  return { rowCount: res.rowCount, msg: !res.rowCount ? res : null };
+}

package/server/routes/dataset/controllers/{bi.dataset.edit.js → editDataset.js}

@@ -1,4 +1,4 @@
-import { dataUpdate, pgClients, getPG, initPG } from "@opengis/fastify-table/utils.js";
+import { dataUpdate, pgClients } from '@opengis/fastify-table/utils.js';
 
 const columnTypeMatch = {
   text: 'text',
@@ -12,12 +12,12 @@ const columnTypeMatch = {
 };
 
 /**
- * Додавання нового набору даних BI
+ * Редагування структури набору даних
  *
- * @method POST
- * @summary Додавання нового набору даних BI
+ * @method PUT
+ * @summary Редагування структури набору даних
  * @priority 4
- * @alias biDatasetAdd
+ * @alias editDataset
  * @type api
  * @tag bi
  * @param {Object} params.id Dashboard ID
@@ -27,47 +27,53 @@ const columnTypeMatch = {
  * @returns {Object} rows Масив з колонками таблиці
  */
 
-export default async function biDatasetEdit(req) {
-  const { params = {}, body = {}, session = {} } = req;
-  const { uid } = session?.passport?.user || {};
-
-  if (!uid) {
+export default async function editDataset(req) {
+  const {
+    pg = pgClients.client, query = {}, body = {}, user = {},
+  } = req;
+  if (!user?.uid) {
     return { message: 'access restricted', status: 403 };
   }
 
-  if (!body.columns?.length) {
+  if (!query?.id) {
+    return { message: 'not enough params: id', status: 404 };
+  }
+
+  if (!body.column_list?.length) {
     return { message: 'not enough params: columns', status: 400 };
   }
 
-  const dataset = await pgClients.client.query('select db, dataset_id as id, table_name as table from bi.dataset where dataset_id=$1', [params.id])
-    .then(el => el.rows[0]);
+  const dataset = await pg.query('select dataset_id as id, table_name as table, column_list as columns from bi.dataset where dataset_id=$1', [query.id])
+    .then(el => el.rows[0] || {});
 
-  if (!dataset.id) {
+  if (!dataset?.id) {
     return { message: 'dataset not found', status: 404 };
   }
 
-  const pg = dataset.db ? getPG({ db: dataset.db }) : pgClients.client;
-  if (!pg.pk) await initPG(pg);
-
   if (!dataset.table || !pg.pk?.[dataset.table]) {
     return { message: `table not found: ${dataset.table}`, status: 404 };
   }
 
+  if (!dataset.table.startsWith('data_user.')) {
+    return { message: 'access restricted: source', status: 403 };
+  }
+
   const { fields = [] } = await pg.query(`select * from ${dataset.table} limit 0`);
   const columnList = fields.map((col) => col.name);
 
-  const columns = (dataset.columns || body.columns).map((col, idx) => ({
-    ...col,
-    name: col.name && columnList.includes(col.name) ? col.name : `col_${idx}`,
-    disabled: col.name && !body.columns.find((item) => item.name === col.name),
-  }));
+  const columns = (dataset.columns || [])
+    .concat(body.column_list.filter((col) => !col.name || !columnList.includes(col.name)))
+    .map((col, idx) => ({
+      ...col,
+      name: col.name && columnList.includes(col.name) ? col.name : `col_${idx}`,
+      disabled: col.name && !body.column_list.find((item) => item.name === col.name),
+    }));
 
   await dataUpdate({
-    pg: pgClients.client,
     table: 'bi.dataset',
     data: { column_list: columns },
     id: dataset.id,
-    uid,
+    uid: user.uid,
   });
 
   const sqlList = columns
@@ -79,5 +85,5 @@ export default async function biDatasetEdit(req) {
     await pg.query(sqlList.join(';'));
   }
 
-  return { message: { id: dataset?.id, action: 'edit', columns }, status: 200 };
-};
+  return { message: { id: dataset?.id, table: dataset.table, columns }, status: 200 };
+}