@opengis/bi 1.2.0 → 1.2.2

package/server/routes/data/controllers/data.js

@@ -0,0 +1,230 @@
+import yaml from 'js-yaml';
+
+import {
+  config,
+  autoIndex,
+  pgClients,
+  getSelect,
+  getSelectVal,
+  getFilterSQL,
+  getMeta,
+  logger,
+} from '@opengis/fastify-table/utils.js';
+
+import chartSQL from './util/chartSQL.js';
+import normalizeData from './util/normalizeData.js';
+
+import { getWidget } from '../../../../utils.js';
+
+const maxLimit = 100;
+
+export default async function dataAPI(req, reply) {
+  const time = Date.now();
+
+  const { query = {}, user = {}, unittest } = req;
+
+  query.metric = Array.isArray(query.metric) ? query.metric.pop() : query.metric;
+
+  const { dashboard, widget, filter, search, samples } = query;
+
+  const widgetData = await getWidget({ pg: req.pg, dashboard, widget });
+
+  if (widgetData.status) return widgetData;
+
+  const { type, text, data = {}, controls, style, options } = widgetData;
+
+  const pg = widgetData.pg || req.pg || pgClients.client;
+
+  const error1 = {};
+  const { fields: cols = [] } = await pg.query(
+    `select * from ${data.table} t ${widgetData.tableSQL || data.tableSQL || ''} limit 0`
+  ).catch(err => Object.assign(error1, { error: err.toString() })) || {};
+  const columnTypes = cols?.map?.((el) => ({
+    name: el.name,
+    type: pg.pgType?.[el.dataTypeID],
+  }));
+
+  // data param
+  const { x, cls, groupbyCls, metric, table, where, tableSQL, groupby, xName, yName, xType, yType, error = error1 } =
+    normalizeData(widgetData, query, columnTypes);
+
+  const limit = Math.min(query.limit || widgetData.limit || maxLimit, maxLimit);
+  // if (error) { return reply.status(400).send(error); }
+
+  // auto Index
+  if (pg.pk?.[data.table]) {
+    autoIndex({
+      table: data.table,
+      pg,
+      columns: [data?.time]
+        .concat([xName])
+        .concat([groupby])
+        .filter((el) => el),
+    }).catch((err) => console.log(err));
+  }
+
+  const { pk, columns = [], view } = await getMeta({ pg, table: data.table });
+
+  if (!view && !pk) {
+    return { message: `table not found: ${data.table} (${pg.options?.database})`, status: 404 };
+  }
+
+  // const columnList = columns.map(col => col.name);
+  const groupbyColumnNotExists = groupby?.split?.(',')?.filter?.(el => !columnTypes.map(el => el.name).includes(el.trim()));
+
+  if (groupby && groupbyColumnNotExists?.length) {
+    return { message: `groupby column not found: ${groupbyColumnNotExists} (${data.table}/${pg.options?.database})`, status: 404 };
+  }
+
+  // get group
+  const groupData = groupby
+    ? await pg
+      .query(
+        `select ${groupby} as name ,count(*) from ${tableSQL || table} group by ${groupby} order by count(*) desc 20`
+      )
+      .then((el) => el.rows)
+    : null;
+
+  if (query.sql === '2') return { x, metric, table, tableSQL, data, groupData };
+
+  const order = data.order || (type === 'listbar' && cols.find(el => el.name === 'metric') ? 'metric desc' : null);
+
+  const fData =
+    filter || search
+      ? await getFilterSQL({
+        pg,
+        table,
+        filter,
+        search,
+        filterList: widgetData.filters,
+      })
+      : {};
+
+  const optimizedSQL = widgetData?.sql
+    ? `${widgetData.sql} ${fData?.q && false ? fData?.q : ''} limit ${limit}`
+    : (fData?.optimizedSQL || `select * from ${tableSQL || table}`);
+
+  if (type?.includes('bar') && !metric?.length) {
+    return { message: 'empty widget params: metrics', status: 400 };
+  }
+
+  const sql = widgetData.sql ? optimizedSQL : (chartSQL[type] || chartSQL.chart)({
+    where: config.local && user?.user_type === 'superadmin' ? 'true' : where, // test
+    metric,
+    yType, // metric type
+    columns: widgetData.columns,
+    table: `(${optimizedSQL})q`,
+    x,
+    groupData,
+    groupby,
+    order,
+    samples,
+    limit,
+    xType,
+    fx: widgetData.fx,
+  });
+
+  if (query.sql) return sql;
+
+  if (!sql || sql?.includes('undefined')) {
+    return {
+      message: {
+        error: 'invalid sql',
+        type,
+        sql,
+        where,
+        metric,
+        table: `(${optimizedSQL})q`,
+        x,
+        groupData,
+        groupby,
+      },
+      status: 500,
+    };
+  }
+
+  if (config.trace) console.log(sql, user?.uid);
+
+  const { rows = [], fields = [], errorSql } = await pg.query(sql.replace('{{uid}}', user?.uid)).catch(err => {
+    logger.file('bi/data', { error: err.toString(), sql });
+    return { errorSql: err.toString() };
+  }); // test with limit
+
+  if (groupbyCls) {
+    const { arr = [] } = await getSelect(groupbyCls, pg) || {};
+    if (arr.length) {
+      const ids = arr.map(el => el.id);
+      const text = arr.reduce((acc, curr) => ({ ...acc, [curr.id]: curr.text }), {});
+      rows.forEach(row => {
+        ids.reduce((acc, curr) => {
+          Object.assign(row, { [text[curr]]: row[curr] });
+          delete row[curr];
+          return acc;
+        }, {});
+      });
+    }
+  }
+
+  if (cls) {
+    const values = rows
+      .map((row) => row[x])
+      ?.filter((el, idx, arr) => el && arr.indexOf(el) === idx);
+    const vals = await getSelectVal({ pg, name: cls, values });
+    rows
+      .filter((row) => row[x])
+      .forEach((row) => {
+        Object.assign(row, { [x]: vals?.[row[x]]?.text || vals?.[row[x]] || row[x] });
+      });
+  }
+
+  const metaTitles = columns.reduce((acc, curr) => Object.assign(acc, { [curr.name]: curr.title || curr.ua }), {});
+  const titles = Array.isArray(widgetData?.columns)
+    ? widgetData.columns.reduce((acc, curr) => Object.assign(acc, { [curr.name]: curr.title || curr.ua }), {})
+    : Object.keys(widgetData?.columns || {}).reduce((acc, curr) => Object.assign(acc, { [curr]: widgetData?.columns?.[curr] }), {});
+
+  const rows1 = type === 'table' ? rows.map(row => Object.keys(row || {}).reduce((acc, curr) => Object.assign(acc, { [titles?.[curr] || metaTitles?.[curr] || curr]: row?.[curr] }), {})) : rows;
+
+  const yml = widgetData.yml || yaml.dump(extractYml(widgetData));
+  const dimensions = fields.map((el) => el.name);
+
+  const res = {
+    time: Date.now() - time,
+    error: error || errorSql || (!widgetData.sql ? widgetData.error : undefined),
+    dimensions,
+    filter: xName,
+    dimensionsType: [xType, yType].filter((el) => el)?.length
+      ? [xType, yType].filter((el) => el)
+      : fields.map((el) => pg.pgType?.[el.dataTypeID]),
+    type,
+
+    text: text || widgetData?.title || data.text,
+    // data: query.format === 'data' ? dimensions.map(el => rows.map(r => r[el])) : undefined,
+    source:
+      query.format === 'array'
+        ? dimensions.map((el) => rows1.map((r) => r[el]))
+        : rows1,
+    style,
+    options,
+    controls,
+    yml,
+    data: widgetData.data,
+    id: query.widget,
+    columns: columnTypes.map(el => Object.assign(el, { title: titles[el.name] || metaTitles?.[el.name] || el.name })),
+    params: config?.local || unittest ? {
+      x,
+      cls,
+      metric,
+      table,
+      tableSQL,
+      where,
+      groupby,
+      sql,
+    } : undefined,
+  };
+  return res;
+}
+
+function extractYml(sourceData) {
+  const { title, description, type, data, style, controls } = sourceData;
+  return { title, description, type, data, style, controls };
+}

package/server/routes/data/controllers/util/chartSQL.js

@@ -0,0 +1,49 @@
+function number({ metric, where, table, samples, fx }) {
+  const sql = `select ${fx || metric} from ${table} where ${where} ${samples ? 'limit 10' : ''}`;
+  return sql;
+}
+function table({ columns = [], table, where, samples }) {
+  const cols = Array.isArray(columns)
+    ? columns.map((el) => `"${(el.name || el).replace(/'/g, "''")}"`).join(',')
+    : Object.keys(columns).map(key => `"${key.replace(/'/g, "''")}"`).join(',');
+  return `select ${cols || '*'} from ${table} where ${where} ${samples ? 'limit 10' : 'limit 20'} `;
+}
+
+function chart({
+  metric,
+  yType, // metric type
+  where,
+  table,
+  x,
+  groupby,
+  groupData,
+  order,
+  samples,
+  limit = 100,
+  xType,
+  fx, // agg function
+}) {
+  const xCol = x && xType?.includes('[]') ? `unnest(${x})` : x;
+
+  const metricData =
+    groupData
+      ?.filter(el => el.name)
+      ?.map(
+        (el) =>
+          `${metric} filter (where '${el.name.toString().replace(/'/g, "''")}'=${yType?.includes('[]') ? `any(${groupby.replace(/'/g, "''")}::text[])` : groupby.replace(/'/g, "''")}) as "${el.name.toString().replace(/'/g, "''")}"`
+      )
+      .join(',') || `${fx || metric} as metric`;
+  const sql = `select ${xCol} ${x && xType?.includes('[]') ? `as ${x}` : ''}, ${metricData} 
+    from ${table}
+    where ${where} 
+    ${xCol ? `group by ${xCol}` : ''}
+    ${order || xCol ? `order by ${order || xCol}` : ''}
+    ${samples ? 'limit 10' : `limit ${limit}`}`;
+  return sql;
+}
+
+function text() {
+  return undefined;
+}
+
+export default { number, chart, table };

package/server/routes/data/controllers/util/normalizeData.js

@@ -0,0 +1,65 @@
+function normalizeData(data, query = {}, columnTypes = []) {
+  const skip = [];
+  ['x', 'groupby', 'granularity'].forEach((el) => {
+    // console.log(el, query[el], columnTypes.find(col => col.name == query[el]))
+    if (!columnTypes.find((col) => col.name == query[el])) {
+      if (query[el] && query[el] !== 'null') {
+        if (el === 'granularity' && !['week', 'month', 'quarter', 'year'].includes(query[el])) {
+          skip.push(`invalid granularity option: ${query[el]}`);
+        } else if (el !== 'granularity') { skip.push(`column not found: ${query[el]}`); }
+      }
+      if (!(el === 'granularity' || (el === 'groupby' && query[el] === 'null'))) delete query[el];
+    }
+  });
+
+  if (
+    !columnTypes.find(
+      (col) => col.type === 'numeric' && col.name == query.metric
+    )
+  ) {
+    delete query.metric;
+  }
+
+  const xName = query.x || (Array.isArray(data.x) ? data.x[0] : data.x);
+  const xType = columnTypes.find((el) => el.name == xName)?.type;
+
+  const granularity =
+    xType === 'date' || xType?.includes('timestamp')
+      ? query.granularity || data.granularity || 'year'
+      : null;
+
+  const x =
+    (granularity
+      ? `date_trunc('${granularity}',${xName})::date::text`
+      : null) || xName;
+
+  const metrics = Array.isArray(data.metrics || data.metric) ? (data.metrics || data.metric) : [data.metrics || data.metric];
+  const metric =
+    (query.metric ? `sum(${query.metric})` : null) ||
+    (metrics.length
+      ? (metrics
+        ?.filter((el) => el && columnTypes.find((col) => col.name == (el?.name || el)))
+        ?.map((el) => el.fx || `${el.operator || 'sum'}(${el.name || el})`)?.join(',') || 'count(*)')
+      : 'count(*)');
+
+  const yName = metrics?.[0]?.name || metrics?.[0];
+  const yType = columnTypes.find((el) => el.name == yName)?.type;
+
+  const { cls, groupbyCls, table, filterCustom } = data;
+  const groupby = (query.groupby || data.groupby) === 'null' ? null : (query.groupby || data.groupby);
+  // const orderby = query.orderby || data.orderby || 'count(*)';
+
+  const custom = query?.filterCustom
+    ?.split(',')
+    ?.map((el) => filterCustom?.find((item) => item?.name === el)?.sql)
+    ?.filter((el) => el)
+    ?.join(' and ');
+  const where = `${data.query || '1=1'} and ${custom || 'true'}`;
+
+  const tableSQL = data.tableSQL?.length
+    ? `(select * from ${data?.table} t ${data.tableSQL || ''} where ${where})q`
+    : undefined;
+
+  return { x, cls, groupbyCls, metric, table, where, tableSQL, groupby, xName, xType, yName, yType, error: skip.length ? skip.join(',') : undefined };
+}
+export default normalizeData;

package/server/routes/data/index.mjs

@@ -0,0 +1,32 @@
+import data from './controllers/data.js';
+
+const biSchema = {
+  type: 'object',
+  properties: {
+    querystring: {
+      widget: { type: 'string', pattern: '^([\\d\\w_]+)$' },
+      dashboard: { type: 'string', pattern: '^([\\d\\w_]+)$' },
+      sql: { type: 'string', pattern: '^([\\d])$' },
+      // metric: { type: 'string', pattern: '^([\\d\\w_]+)$' },
+      x: { type: 'string', pattern: '^([\\d\\w_]+)$' },
+      granularity: { type: 'string', pattern: '^(week|month|quarter|year)$' },
+      groupby: { type: 'string', pattern: '^([\\d\\w_]+)$' },
+      filterCustom: { type: 'string', pattern: '^([\\d\\w_,]+)$' },
+    },
+    params: {
+      id: { type: 'string', pattern: '^([\\d\\w]+)$' },
+    },
+  },
+};
+
+const policy = ['public'];
+
+export default async function route(fastify, opts) {
+  fastify.route({
+    method: 'GET',
+    url: '/bi-data',
+    schema: biSchema,
+    config: { policy },
+    handler: data,
+  });
+}

package/server/routes/dataset/controllers/bi.dataset.list.js

@@ -0,0 +1,29 @@
+import { pgClients } from "@opengis/fastify-table/utils.js";
+
+const maxLimit = 100;
+
+/**
+ * Отримання списку наборів даних BI
+ *
+ * @method GET
+ * @summary Отримання списку наборів даних BI
+ * @priority 4
+ * @alias biDatasetList
+ * @type api
+ * @tag bi
+ * @errors 400,500
+ * @returns {Number} status Номер помилки
+ * @returns {String} error Опис помилки
+ * @returns {Object} rows Масив з колонками таблиці
+ */
+
+export default async function biDatasetList({ pg = pgClients.client, query = {} }) {
+
+  const limit = Math.min(maxLimit, +(query.limit || 20));
+  const offset = query.page && query.page > 0 ? (query.page - 1) * limit : 0;
+
+  const { rows = [] } = await pg.query(`select dataset_id as id, name, source_type as source, table_name as table, dashboard_list 
+  from bi.dataset order by name limit ${limit} offset ${offset}`);
+
+  return { rows };
+};

package/server/routes/dataset/controllers/bi.db.list.js

@@ -0,0 +1,19 @@
+import { config } from '@opengis/fastify-table/utils.js';
+
+/**
+ * Список БД
+ *
+ * @method POST
+ * @priority 4
+ * @alias biDbList
+ * @type api
+ * @tag bi
+ * @errors 400,500
+ * @returns {Number} status Номер помилки
+ * @returns {String} error Опис помилки
+ * @returns {Object} rows Масив з колонками таблиці
+ */
+
+export default async function biDbList(req) {
+    return { db_list: config.db_list?.map((el) => ({ id: el.key, text: el.title })) || [] };
+};

package/server/routes/dataset/controllers/comment.js

@@ -0,0 +1,55 @@
+import { pgClients } from '@opengis/fastify-table/utils.js';
+
+export default async function datasetEditComment({
+  pg = pgClients.client,
+  body = {},
+  params = {},
+}) {
+  if (!params?.id) {
+    return { message: 'not enough params: id', status: 400 };
+  }
+
+  if (!Array.isArray(body.data) || !body.data?.length) {
+    return { message: 'invalid param: body.data not an array / empty', status: 400 };
+  }
+
+  const dataset = await pg.query('select dataset_id as id, table_name as table from bi.dataset where dataset_id=$1', [params.id])
+    .then(el => el.rows?.[0] || {});
+
+  if (!dataset?.table) {
+    return { message: dataset?.id ? 'dataset table not set' : 'dataset not found', status: 404 };
+  }
+
+  const { fields = [] } = pg.pk?.[dataset.table] ? await pg.query(`select * from ${dataset.table} limit 0`) : {};
+
+  if (!fields.length) {
+    return { message: `table not found: ${dataset.table}`, status: 404 };
+  }
+
+  const columnList = fields.map(el => el.name);
+  const validData = body.data?.filter?.(el => el?.name && el?.comment && columnList.includes(el.name));
+
+  const q1 = validData.map(el => `COMMENT ON COLUMN ${dataset.table}."${el.name.replace(/'/g, "''")}" IS '${el.comment.replace(/'/g, "''")}'`).join(';');
+  if (q1) await pg.query(q1);
+
+  const q2 = `select
+    attrelid::regclass,
+    attname,
+    pg_catalog.col_description(attrelid,attnum) as title
+  from pg_catalog.pg_attribute a 
+  where attnum > 0 
+  and attname not in ('editor_id','editor_date','cdate','geom','id','uid','cdate')
+  and atttypid::regtype not in ('json','geometry')
+  and pg_catalog.col_description(attrelid,attnum) is not null`;
+
+  const { rows = [] } = await pg.query(q2);
+
+  const title = rows
+    .filter(el => el.attrelid === dataset.table)
+    .reduce((p, el) => ({ ...p, [el.attname]: el.title }), {});
+
+  const res = await pg.query(`update bi.dataset set setting=coalesce(setting::jsonb, '{}'::jsonb)||$1::jsonb 
+  where dataset_id=$2 returning dataset_id as id, setting`, [{ title }, dataset.id]);
+
+  return res.rows?.[0];
+}

package/server/routes/dataset/controllers/createDatasetPost.js

@@ -0,0 +1,134 @@
+import path from 'node:path';
+import { existsSync } from 'node:fs';
+
+import { config, pgClients, getFolder, file2json } from '@opengis/fastify-table/utils.js';
+
+import createTableQuery from '../utils/createTableQuery.js';
+import executeQuery from '../utils/executeQuery.js';
+import downloadRemoteFile from '../utils/downloadRemoteFile.js';
+
+const insertDataset = `insert into bi.dataset 
+(name, table_name, dataset_file_path, column_list, pk, data_source, uid)
+values($1,$2,$3,$4,$5,$6,$7) returning dataset_id`;
+
+export default async function createDatasetPost({
+  pg = pgClients.client, body = {}, user = {},
+}) {
+  if (!user?.uid) {
+    return { message: 'access restricted', status: 403 };
+  }
+
+  if (!body?.name) {
+    return { message: 'not enough query params: name', status: 400 };
+  }
+
+  if (!body?.table_name && !body?.file && !body?.column_list?.length && !body?.dataset_url) {
+    return { message: 'not enough query params: table / file / column_list/ url', status: 400 };
+  }
+
+  const {
+    name,
+    table_name: existingTable,
+    column_list: columns = [],
+    dataset_url: datasetUrl,
+    encoding,
+  } = body;
+
+  const rootDir = getFolder(config, 'local');
+
+  if (datasetUrl) {
+    const { filePath, error } = await downloadRemoteFile({
+      rootDir, url: datasetUrl, table: name || datasetUrl,
+    });
+    if (error || !filePath) {
+      return { message: error || 'file request URL error', status: 500 };
+    }
+    Object.assign(body, { file: filePath });
+  }
+
+  const { file: relPath } = body;
+
+  if (relPath) {
+    const filepath = path.join(rootDir, relPath);
+    const exists = existsSync(filepath);
+
+    if (!exists) {
+      return { message: 'Файл з вихідними даними не знайдено', status: 404 };
+    }
+
+    const json = await file2json({ filepath, encoding });
+
+    // excel sheets fix?
+    const data1 = (['.xls', '.xlsx'].includes(path.extname(filepath)) && !Array.isArray(json)) ? json[Object.keys(json)[0]] : json;
+    const data = path.extname(filepath) === '.json' && body?.key ? data1?.[body.key] : data1;
+
+    const features = ['.csv', '.xlsx', '.xls'].includes(path.extname(filepath))
+      ? data?.map?.((el) => ({ type: 'Feature', properties: Object.keys(el).reduce((acc, curr) => Object.assign(acc, { [curr]: el[curr] }), {}) }))
+      : data?.features || data?.map?.((el) => ({ type: 'Feature', geometry: el.geom, properties: Object.keys(el).filter((key) => key !== 'geom').reduce((acc, curr) => Object.assign(acc, { [curr]: el[curr] }), {}) }));
+
+    if (!Array.isArray(features) || !features?.length) {
+      return { message: 'Файл з вихідними даними порожній', status: 400 };
+    }
+
+    Object.assign(data, { features });
+
+    const fileColumns = Object.keys(data?.features[0]?.properties)
+      ?.filter((el) => !['editor_date', 'cdate', 'uid', 'editor_id', 'files'].includes(el.toLowerCase()))
+      ?.map((el) => ({ title: el, format: 'text' }));
+
+    const { sql, pkey, table } = createTableQuery(fileColumns, name);
+
+    const { datasetId, error } = await executeQuery({
+      pg,
+      sql,
+      data,
+      name,
+      table,
+      relPath,
+      columns: fileColumns,
+      pkey,
+      source: datasetUrl ? 'url' : 'file',
+      url: datasetUrl,
+      user,
+      dataKey: body?.key,
+    });
+
+    if (error) return { error, status: 500 };
+
+    pg.pk[table] = pkey;
+    pg.tlist.push(table);
+    return { message: { id: datasetId, table, source: datasetUrl ? 'url' : 'file' }, status: 200 };
+  }
+
+  if (existingTable) {
+    await pg.query(`alter table ${existingTable} add column if not exists geom public.geometry;
+    alter table ${existingTable} add column if not exists files json`);
+    const args = [name, existingTable, null, null, pg.pk?.[existingTable], JSON.stringify({ type: 'table' }), user?.uid];
+    const datasetId = await pg.query(insertDataset, args).then(el => el.rows?.[0]?.dataset_id);
+    return { message: { id: datasetId, table: existingTable, source: 'table' }, status: 200 };
+  }
+
+  if (!columns?.length) {
+    return { message: 'У даній заяві відсутні налаштування структури набору даних', status: 400 };
+  }
+
+  const { sql, pkey, table } = createTableQuery(columns, name);
+
+  const { datasetId, error } = await executeQuery({
+    pg,
+    sql,
+    name,
+    table,
+    relPath,
+    columns,
+    pkey,
+    source: 'newtable',
+    user,
+  });
+
+  if (error) return { error, status: 500 };
+
+  pg.pk[table] = pkey;
+  pg.tlist.push(table);
+  return { message: { id: datasetId, table, source: 'newtable' }, status: 200 };
+}