@opengis/bi 1.0.14 → 1.0.15

package/server/routes/dataset/controllers/bi.dataset.demo.add.js

@@ -0,0 +1,97 @@
+import path from 'node:path';
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+
+import { getPG } from '@opengis/fastify-table/utils.js';
+
+const host = 'https://cdn.softpro.ua/demo/bi';
+
+
+const root = process.cwd();
+const send = (msg) => console.log(msg)
+async function downloadFile(fileName) {
+    const pg = getPG();
+    send(`${fileName} - checking if already downloaded...`);
+
+    const demoPath = path.join(root, '/log/temp/', fileName);
+    // const fileExists = await isFileExists(demoPath);
+
+    // send(`${host}/${fileName}` + demoPath)
+    const response = await fetch(`${host}/${fileName}`);
+    const body = await response.text();
+    //send(body)
+    if (response?.status !== 200) {
+        return { [fileName]: { result: 'file not found', status: 404 } };
+    }
+
+    await mkdir(path.dirname(demoPath), { recursive: true });
+
+    await writeFile(demoPath, body, 'utf8');
+
+    await pg.query(`drop table if exists ${path.parse(fileName).name}`);
+    const { exists } = await pg.query(`select to_regclass($1) is not null as exists`, [path.parse(fileName).name])
+        .then((res1) => res1.rows?.[0] || {});
+
+
+    if (exists) {
+        send(`${fileName} - dataset table already exists!`);
+        return { [fileName]: { result: 'dataset table already exists', status: 200 } };
+    }
+
+    const sql = await readFile(demoPath, 'utf8');
+    await pg.query(sql);
+    return { [fileName]: { result: 'success', status: 200 } };
+}
+
+/**
+ * Імпорт demo даних dashboard
+ *
+ * @method GET
+ * @summary Імпорт demo даних dashboard
+ * @priority 4
+ * @type api
+ * @tag bi
+ * @errors 400,500
+ * @returns {Number} status Номер помилки
+ * @returns {String} error Опис помилки
+ * @returns {Object} rows Масив з колонками таблиці
+ */
+
+const demoList = {
+    'demo.orders.sql': 'Статистика продажів (demo)',
+    'demo.cleaned_sales_data.sql': 'Статистика продажів (demo 2)',
+    'demo.video_game_sales.sql': 'Статистика продажів відеоігор (demo)',
+};
+
+export default async function biDatasetDemoAdd(req) {
+    const { pg, query = {} } = req;
+    //    const send = () => { }; // eventStream?
+
+
+
+    const sqlList = [];
+
+    const tables = Object.keys(demoList)?.map((el) => {
+        const table = path.parse(el).name;
+        sqlList.push(`insert into bi.dataset(dataset_id,name,table_name)
+            select '${el}', '${demoList?.[el]}', '${table}' on conflict(dataset_id) do update set 
+            name=excluded.name, table_name=excluded.table_name
+            returning dataset_id as id`);
+        return table;
+    });
+
+    const { count } = await pg.query(`select count(*) from pg_catalog.pg_tables where schemaname||'.'||tablename = any($1)`, [tables]).then((res1) => res1.rows?.[0] || {});
+
+    if (count === '3' && !query.nocache) {
+        const result = await pg.query(sqlList.join(';\n'));
+        const ids = (Array.isArray(result) ? result : [result]).filter((el) => el).map((el) => el?.rows?.[0]?.id);
+        return { message: { ids, message: 'all datasets already downloaded' }, status: 200 };
+    }
+
+    const res = {};
+    const resultAll = await Promise.all(Object.keys(demoList).map(async (fileName) => await downloadFile(fileName))).catch(err => console.log(err));
+
+    const result = await pg.query(sqlList.join(';\n'));
+    const ids = (Array.isArray(result) ? result : [result]).filter((el) => el).map((el) => el?.rows?.[0]?.id);
+    return { message: { ids, message: res, resultAll }, status: 200 };
+
+};

package/server/routes/dataset/controllers/bi.dataset.import.js

@@ -0,0 +1,67 @@
+import { randomBytes } from 'node:crypto';
+
+import createTableFunc from './util/create.table.js';
+import prepareData from './util/prepare.data.js';
+
+/**
+ * Імпорт даних до BI набору
+ *
+ * @method POST
+ * @summary Імпорт даних до BI набору
+ * @priority 4
+ * @alias biDatasetPost
+ * @type api
+ * @tag bi
+ * @param {Object} params.id Dashboard ID
+ * @errors 400,500
+ * @returns {Number} status Номер помилки
+ * @returns {String} error Опис помилки
+ * @returns {Object} rows Масив з колонками таблиці
+ */
+
+export default async function biDatasetPost(req) {
+  const {
+    pg, params = {}, body = {}, session = {},
+  } = req;
+  const { uid } = session?.passport?.user || {};
+
+  if (!uid) {
+    return { message: 'access restricted', status: 403 };
+  }
+
+  if (!Array.isArray(body?.data) && body?.data?.features?.[0]?.type !== 'Feature') {
+    return { message: 'body data param is invalid', status: 400 };
+  }
+
+  // await pg.query(`insert into bi.dataset(dataset_id,name, uid) values($1,$1,$2) on conflict (dataset_id) do update set table_name=null`, [params?.id, uid]);
+  const dataset = await pg.query('select dataset_id as id, name, table_name as table from bi.dataset where dataset_id=$1', [params?.id]).then((res) => res.rows?.[0] || {});
+  if (!dataset.id) {
+    return { message: `dataset not found: ${params?.id}`, status: 404 };
+  }
+
+  const tableName = randomBytes(64).toString('hex').substring(0, 24).replace(/^\d+/, '');
+  const pkey = tableName.concat('_id');
+  const table = `bi_data.${tableName}`;
+  const { columns, insertData } = await prepareData({ req, table, data: body?.data });
+  const createTable = createTableFunc({ table, pkey, columns });
+  const updateDataset = `update bi.dataset set table_name='${table}', editor_id='${uid}' where dataset_id='${params?.id}'`;
+  const sqlList = [createTable, insertData, updateDataset];
+  if (dataset.name) {
+    sqlList.push(`comment on table ${table} is '${dataset.name}'`);
+  }
+  const q = sqlList.filter((el) => el).join(';');
+  await pg.query('create extension if not exists postgis;create schema if not exists bi_data;');
+
+  // await pg.query(createTable);
+
+  //const result = {}
+  const result = await pg.query(q);
+
+  return {
+    message: {
+      id: params?.id, action: 'import', result: 'success', total: result?.find((el) => el.command === 'INSERT')?.rowCount,
+    },
+    status: 200,
+  };
+
+};

package/server/routes/dataset/controllers/util/create.table.js

@@ -0,0 +1,22 @@
+export default function createTableFunc({
+  table, pkey, columns,
+}) {
+  const columnType1 = {
+    text: 'text', select: 'text', date: 'date', 'yes/no': 'boolean', badge: 'text', number: 'numeric', tags: 'text[]', geom: 'geom',
+  };
+
+  const createQuery = `create table if not exists ${table} (
+  ${pkey} text not null default public.uuid_generate_v4(), 
+  geom public.geometry, ${columns ? `
+  ${columns?.map((el) => `${el.name} ${columnType1[el.format] || 'text'}`).join(', ')},` : ''} 
+  cdate timestamp without time zone not null default now(), 
+  editor_date timestamp without time zone, 
+  uid text, 
+  editor_id text, 
+  files json, 
+  constraint ${table.replace(/\./g, '_')}_constraint_pkey PRIMARY KEY (${pkey}) )`;
+
+  const commentQuery = columns?.filter((el) => el.ua)?.map((el) => `comment on column ${table}.${el.name} is '${el.ua}'`).join(';');
+
+  return [createQuery, commentQuery].join(';');
+};

package/server/routes/dataset/controllers/util/prepare.data.js

@@ -0,0 +1,49 @@
+import path from 'node:path';
+
+import { getFolder, isFileExists } from '@opengis/fastify-table/utils.js';
+
+import { file2json } from '@opengis/fastify-file/utils.js';
+
+function getFileColumns({ data }) {
+  if (data?.features) {
+    return Object.keys(data?.features[0]?.properties); // geojson / shapefile
+  }
+  if (Array.isArray(data)) {
+    return Object.keys(data[0])?.filter((el) => el); // csv / xls
+  }
+  return Object.values(data[Object.keys(data)?.[0]]?.[0]); // else?
+}
+
+export default async function prepareData({
+  req, table, file_path: filePath, data: data1,
+}) {
+  const rootDir = getFolder(req);
+  const fullPath = path.join(rootDir, filePath || '');
+  const fileExists = await isFileExists(fullPath);
+  if (!fileExists && !data1) {
+    return { message: `file not found: ${filePath}`, status: 404 };
+  }
+
+  const data11 = data1 || await file2json(fullPath);
+  const data = (['.xls', '.xlsx'].includes(path.extname(fullPath)) && !Array.isArray(data11)) ? data11[Object.keys(data11)[0]] : data11;
+
+  /* generate column list based on input data */
+  const columns = getFileColumns({ data })
+    ?.filter((el) => !['editor_date', 'cdate', 'uid', 'editor_id', 'files'].includes(el.toLowerCase()))
+    ?.map((el, index) => ({ name: `col_${index}`, ua: el, format: 'text' }));
+
+  const columnList1 = columns?.map((el) => el.name)?.join(',');
+  const columns1 = ','.concat(columns?.map((el) => `features->'properties'->>'${el?.ua}'`)?.join(','));
+  const columns2 = columns?.map((el) => `d->>'${el.ua}'`).join(',');
+  const q1 = data?.features ? `(SELECT json_array_elements('${JSON.stringify(data.features).replace(/'/g, "''")}'::json) AS features)q` : null;
+  const q2 = !data?.features ? `( SELECT json_array_elements('${JSON.stringify(Array.isArray(data)
+    ? data
+    : data[Object?.keys(data)?.[0]])
+    .replace(/'/g, "''")
+    .replace(/\\"/g, '`')}'::json) AS d )q`
+    : null;
+  const insertData = data?.features
+    ? `INSERT into ${table} (geom ${','.concat(columnList1)}) SELECT public.ST_GeomFromGeoJSON((features->>'geometry')::json) AS geom ${columns1} FROM ${q1}`
+    : `INSERT into ${table} (${columnList1}) SELECT ${columns2} FROM ${q2}`;
+  return { columns, insertData };
+};

package/server/routes/dataset/index.mjs

@@ -0,0 +1,19 @@
+import biDatasetPost from './controllers/bi.dataset.import.js';
+import biDatasetDemoAdd from './controllers/bi.dataset.demo.add.js';
+
+const biSchema = {};
+
+export default async function route(fastify, opts) {
+  fastify.route({
+    method: 'POST',
+    url: '/bi-dataset/:id',
+    schema: biSchema,
+    handler: biDatasetPost,
+  });
+  fastify.route({
+    method: 'GET',
+    url: '/bi-dataset-demo',
+    schema: biSchema,
+    handler: biDatasetDemoAdd,
+  });
+}

package/server/routes/db/controllers/dbTablePreview.js

@@ -1,63 +1,63 @@
-const q = `select nspname||'.'||relname as table, json_agg(json_build_object('name',attname, 'type', a.atttypid::regtype, 'description', coalesce(col_description(attrelid, attnum),attname))) as columns
-from pg_attribute a
-left join pg_catalog.pg_attrdef d ON (a.attrelid, a.attnum) = (d.adrelid, d.adnum)
-JOIN pg_class AS i
-  ON i.oid = a.attrelid 
-JOIN pg_namespace AS NS ON i.relnamespace = NS.OID
-where a.attnum > 0 and nspname||'.'||relname = $1
-and not a.attisdropped
-group by nspname||'.'||relname limit 1`;
-
-export default async function dbTablePreview({ pg, params = {}, query = {} }) {
-  if (!params?.name) {
-    return { message: 'not enough params: name', status: 400 };
-  }
-
-  if (query.sql) return q;
-  try {
-    const { table, columns } = await pg
-      .query(q, [params.name])
-      .then((res) => res.rows?.[0] || {});
-    if (!table) {
-      return { message: 'table not found', status: 404 };
-    }
-
-    const { count = 0 } = await pg
-      .query(
-        `select reltuples as count from pg_class where oid  = to_regclass($1)`,
-        [params.name]
-      )
-      .then((res) => res.rows?.[0] || {});
-    const geom = columns.find((el) => el.type === 'geometry')?.name;
-    const { bounds, extentStr } = geom
-      ? await pg
-          .query(
-            `select count(*),
-      st_asgeojson(st_extent(${geom}))::json as bounds,
-      replace(regexp_replace(st_extent(${geom})::box2d::text,'BOX\\(|\\)','','g'),' ',',') as "extentStr"
-    from ${params.name}`
-          )
-          .then((res) => res.rows?.[0] || {})
-      : {};
-    const extent = extentStr ? extentStr.split(',') : undefined;
-
-    const systemColumns = [
-      'uid',
-      'files',
-      'editor_date',
-      'cdate',
-      'editor_id',
-      geom,
-    ];
-    const columnList = columns
-      .map((el) => el?.name)
-      .filter((el) => !systemColumns.includes(el));
-    const { rows = [] } = await pg.query(
-      `select ${columnList.join(',')} ${geom ? `, st_asgeojson(geom)::json as geom` : ''} from ${table} limit 10`
-    );
-
-    return { count, geom: !!geom, bounds, extent, columns, rows };
-  } catch (err) {
-    return { error: err.toString(), status: 500 };
-  }
-}
+const q = `select nspname||'.'||relname as table, json_agg(json_build_object('name',attname, 'type', a.atttypid::regtype, 'description', coalesce(col_description(attrelid, attnum),attname))) as columns
+from pg_attribute a
+left join pg_catalog.pg_attrdef d ON (a.attrelid, a.attnum) = (d.adrelid, d.adnum)
+JOIN pg_class AS i
+  ON i.oid = a.attrelid 
+JOIN pg_namespace AS NS ON i.relnamespace = NS.OID
+where a.attnum > 0 and nspname||'.'||relname = $1
+and not a.attisdropped
+group by nspname||'.'||relname limit 1`;
+
+export default async function dbTablePreview({ pg, params = {}, query = {} }) {
+  if (!params?.name) {
+    return { message: 'not enough params: name', status: 400 };
+  }
+
+  if (query.sql) return q;
+  try {
+    const { table, columns } = await pg
+      .query(q, [params.name])
+      .then((res) => res.rows?.[0] || {});
+    if (!table) {
+      return { message: 'table not found', status: 404 };
+    }
+
+    const { count = 0 } = await pg
+      .query(
+        `select reltuples as count from pg_class where oid  = to_regclass($1)`,
+        [params.name]
+      )
+      .then((res) => res.rows?.[0] || {});
+    const geom = columns.find((el) => el.type === 'geometry')?.name;
+    const { bounds, extentStr } = geom
+      ? await pg
+          .query(
+            `select count(*),
+      st_asgeojson(st_extent(${geom}))::json as bounds,
+      replace(regexp_replace(st_extent(${geom})::box2d::text,'BOX\\(|\\)','','g'),' ',',') as "extentStr"
+    from ${params.name}`
+          )
+          .then((res) => res.rows?.[0] || {})
+      : {};
+    const extent = extentStr ? extentStr.split(',') : undefined;
+
+    const systemColumns = [
+      'uid',
+      'files',
+      'editor_date',
+      'cdate',
+      'editor_id',
+      geom,
+    ];
+    const columnList = columns
+      .map((el) => el?.name)
+      .filter((el) => !systemColumns.includes(el));
+    const { rows = [] } = await pg.query(
+      `select ${columnList.join(',')} ${geom ? `, st_asgeojson(geom)::json as geom` : ''} from ${table} limit 10`
+    );
+
+    return { count, geom: !!geom, bounds, extent, columns, rows };
+  } catch (err) {
+    return { error: err.toString(), status: 500 };
+  }
+}

package/server/routes/db/controllers/dbTables.js

@@ -1,36 +1,36 @@
-export default async function dbTables({ pg, query = {} }) {
-  const q = `select 
-  t.table_schema ||'.'|| t.table_name as table, 
-  obj_description(to_regclass(t.table_schema ||'."'|| t.table_name||'"')) as description,
-  t.table_schema as schema,
-  (select reltuples from pg_class where oid  = to_regclass(t.table_schema ||'."'|| t.table_name||'"') ) as total,
-  coalesce(isgeom,false) as isgeom
-  
-  from information_schema.tables t
-  left join lateral(
-    select true as isgeom from information_schema.columns c 
-    where c.table_name = t.table_name 
-    and c.table_schema = t.table_schema and 'geometry'=c.udt_name limit 1
-  )c on 1=1
-  
-  where t.table_type = 'BASE TABLE'
-  and t.table_schema not in ('public','log','admin','feature_ir','gis', 'setting')
-  and t.table_name not like '%.%'
-  and regexp_replace(t.table_name, '^[[:digit:]]', '', 'g') = t.table_name
-  and 1=(SELECT count(*) FROM pg_catalog.pg_constraint con 
-    INNER JOIN pg_catalog.pg_class rel ON rel.oid = con.conrelid
-    INNER JOIN pg_catalog.pg_namespace nsp ON nsp.oid = connamespace 
-    WHERE nsp.nspname = t.table_schema AND rel.relname = t.table_name and contype='p'
-  )
-  and ${query.schema ? `t.table_schema=$1` : '1=1'} order by total desc`;
-  if (query.sql) return q;
-  try {
-    const { rows = [] } = await pg.query(
-      q,
-      [query.schema].filter((el) => el)
-    );
-    return { rows };
-  } catch (err) {
-    return { error: err.toString(), status: 500 };
-  }
-}
+export default async function dbTables({ pg, query = {} }) {
+  const q = `select 
+  t.table_schema ||'.'|| t.table_name as table, 
+  obj_description(to_regclass(t.table_schema ||'."'|| t.table_name||'"')) as description,
+  t.table_schema as schema,
+  (select reltuples from pg_class where oid  = to_regclass(t.table_schema ||'."'|| t.table_name||'"') ) as total,
+  coalesce(isgeom,false) as isgeom
+  
+  from information_schema.tables t
+  left join lateral(
+    select true as isgeom from information_schema.columns c 
+    where c.table_name = t.table_name 
+    and c.table_schema = t.table_schema and 'geometry'=c.udt_name limit 1
+  )c on 1=1
+  
+  where t.table_type = 'BASE TABLE'
+  and t.table_schema not in ('public','log','admin','feature_ir','gis', 'setting')
+  and t.table_name not like '%.%'
+  and regexp_replace(t.table_name, '^[[:digit:]]', '', 'g') = t.table_name
+  and 1=(SELECT count(*) FROM pg_catalog.pg_constraint con 
+    INNER JOIN pg_catalog.pg_class rel ON rel.oid = con.conrelid
+    INNER JOIN pg_catalog.pg_namespace nsp ON nsp.oid = connamespace 
+    WHERE nsp.nspname = t.table_schema AND rel.relname = t.table_name and contype='p'
+  )
+  and ${query.schema ? `t.table_schema=$1` : '1=1'} order by total desc`;
+  if (query.sql) return q;
+  try {
+    const { rows = [] } = await pg.query(
+      q,
+      [query.schema].filter((el) => el)
+    );
+    return { rows };
+  } catch (err) {
+    return { error: err.toString(), status: 500 };
+  }
+}

package/server/routes/db/index.mjs

@@ -1,17 +1,17 @@
-import dbTables from './controllers/dbTables.js';
-import dbTablePreview from './controllers/dbTablePreview.js';
-
-export default async function route(fastify, opts) {
-  fastify.route({
-    method: 'GET',
-    url: '/db-tables',
-    schema: {},
-    handler: dbTables,
-  });
-  fastify.route({
-    method: 'GET',
-    url: '/db-tables/:name',
-    schema: {},
-    handler: dbTablePreview,
-  });
-}
+import dbTables from './controllers/dbTables.js';
+import dbTablePreview from './controllers/dbTablePreview.js';
+
+export default async function route(fastify, opts) {
+  fastify.route({
+    method: 'GET',
+    url: '/db-tables',
+    schema: {},
+    handler: dbTables,
+  });
+  fastify.route({
+    method: 'GET',
+    url: '/db-tables/:name',
+    schema: {},
+    handler: dbTablePreview,
+  });
+}

package/server/routes/edit/controllers/dashboard.add.js

@@ -1,24 +1,26 @@
-export default async function widgetAdd({ pg, funcs, params = {}, body }) {
-  try {
-    const time = Date.now();
-    const tableName = body.table_name;
-    const checkTable = await pg.query(
-      `select * from bi.dashboard where $1 in (table_name)`,
-      [tableName]
-    );
-    if (!checkTable.rows.length) return { message: 'bad params', status: 401 };
-    const res = await funcs.dataInsert({
-      table: 'bi.dashboard',
-      data: body,
-    });
-
-    return {
-      time: Date.now() - time,
-      message: `Added new dashboard, ID: '${res.rows[0].title}'`,
-      status: 200,
-      rows: res.rows,
-    };
-  } catch (err) {
-    return { error: err.toString(), status: 500 };
-  }
-}
+import { dataInsert } from "@opengis/fastify-table/utils.js";
+
+export default async function widgetAdd({ pg, funcs, params = {}, body }) {
+  try {
+    const time = Date.now();
+    const tableName = body.table_name;
+    const checkTable = await pg.query(
+      `select * from bi.dashboard where $1 in (table_name)`,
+      [tableName]
+    );
+    if (!checkTable.rows.length) return { message: 'bad params', status: 401 };
+    const res = await dataInsert({
+      table: 'bi.dashboard',
+      data: body,
+    });
+
+    return {
+      time: Date.now() - time,
+      message: `Added new dashboard, ID: '${res.rows[0].title}'`,
+      status: 200,
+      rows: res.rows,
+    };
+  } catch (err) {
+    return { error: err.toString(), status: 500 };
+  }
+}

package/server/routes/edit/controllers/dashboard.edit.js

@@ -1,44 +1,46 @@
-export default async function dashboardEdit(
-  { pg, funcs, params = {}, body = {} },
-  reply
-) {
-  try {
-    if (!params.name) {
-      return {
-        message: 'not enough params: dashboard name required',
-        status: 400,
-      };
-    }
-    const tableName = body.table_name;
-    const checkTable = await pg.query(
-      `select * from bi.dashboard where $1 in (table_name)`,
-      [tableName]
-    );
-    if (!checkTable.rows.length) return { message: 'bad params', status: 401 };
-    const { name: dashboardName } = params;
-    const row = await pg
-      .query(
-        `select dashboard_id from bi.dashboard where $1 in (dashboard_id, name)`,
-        [dashboardName]
-      )
-      .then((res1) => res1.rows?.[0] || {});
-    const { dashboard_id: dashboardId } = row;
-
-    const res = await funcs.dataUpdate({
-      table: 'bi.dashboard',
-      id: dashboardId,
-      data: body,
-    });
-    if (!Object.keys(res)?.length) {
-      return { message: 'not found data', status: 404 };
-    }
-
-    return {
-      message: `updated ${dashboardName}`,
-      status: 200,
-      rows: res,
-    };
-  } catch (err) {
-    return reply.status(500).send(err.toString());
-  }
-}
+import { dataUpdate } from "@opengis/fastify-table/utils.js";
+
+export default async function dashboardEdit(
+  { pg, params = {}, body = {} },
+  reply
+) {
+  try {
+    if (!params.name) {
+      return {
+        message: 'not enough params: dashboard name required',
+        status: 400,
+      };
+    }
+    const tableName = body.table_name;
+    const checkTable = await pg.query(
+      `select * from bi.dashboard where $1 in (table_name)`,
+      [tableName]
+    );
+    if (!checkTable.rows.length) return { message: 'bad params', status: 401 };
+    const { name: dashboardName } = params;
+    const row = await pg
+      .query(
+        `select dashboard_id from bi.dashboard where $1 in (dashboard_id, name)`,
+        [dashboardName]
+      )
+      .then((res1) => res1.rows?.[0] || {});
+    const { dashboard_id: dashboardId } = row;
+
+    const res = await dataUpdate({
+      table: 'bi.dashboard',
+      id: dashboardId,
+      data: body,
+    });
+    if (!Object.keys(res)?.length) {
+      return { message: 'not found data', status: 404 };
+    }
+
+    return {
+      message: `updated ${dashboardName}`,
+      status: 200,
+      rows: res,
+    };
+  } catch (err) {
+    return reply.status(500).send(err.toString());
+  }
+}