@opengis/bi 1.0.13 → 1.0.15

package/server/routes/dashboard/controllers/utils/yaml.js

@@ -1,12 +1,11 @@
-import yaml from 'js-yaml';
-
-yaml.loadSafe = (yml) => {
-  try {
-    return yaml.load(yml);
-  }
-  catch (err) {
-    return { error: err.toString() };
-  }
-};
-
-export default yaml;
+import yaml from 'js-yaml';
+
+yaml.loadSafe = (yml) => {
+  try {
+    return yaml.load(yml);
+  } catch (err) {
+    return { error: err.toString() };
+  }
+};
+
+export default yaml;

package/server/routes/dashboard/index.mjs

@@ -1,24 +1,25 @@
-import config from '../../../config.js';
-
-import dashboard from './controllers/dashboard.js';
-import dashboardList from './controllers/dashboard.list.js';
-import dashboardDelete from './controllers/dashboard.delete.js';
-const biSchema = {
-  querystring: {
-    widget: { type: 'string', pattern: '^([\\d\\w]+)$' },
-    dashboard: { type: 'string', pattern: '^([\\d\\w]+)$' },
-    list: { type: 'string', pattern: '^([\\d])$' },
-    sql: { type: 'string', pattern: '^([\\d])$' }
-  },
-  params: {
-    id: { type: 'string', pattern: '^([\\d\\w]+)$' },
-  },
-};
-
-
-export default async function route(fastify) {
-  fastify.get(`/bi-dashboard/:id`, { schema: biSchema }, dashboard);
-  fastify.get(`/bi-dashboard`, dashboardList);
-  fastify.delete(`/bi-dashboard/:id`, dashboardDelete);
-  fastify.get(`/bi-test`, () => { return { test: '2' }});
-}
+import config from '../../../config.js';
+
+import dashboard from './controllers/dashboard.js';
+import dashboardList from './controllers/dashboard.list.js';
+import dashboardDelete from './controllers/dashboard.delete.js';
+const biSchema = {
+  querystring: {
+    widget: { type: 'string', pattern: '^([\\d\\w]+)$' },
+    dashboard: { type: 'string', pattern: '^([\\d\\w]+)$' },
+    list: { type: 'string', pattern: '^([\\d])$' },
+    sql: { type: 'string', pattern: '^([\\d])$' },
+  },
+  params: {
+    id: { type: 'string', pattern: '^([\\d\\w]+)$' },
+  },
+};
+
+export default async function route(fastify) {
+  fastify.get(`/bi-dashboard/:id`, { schema: biSchema }, dashboard);
+  fastify.get(`/bi-dashboard`, dashboardList);
+  fastify.delete(`/bi-dashboard/:id`, dashboardDelete);
+  fastify.get(`/bi-test`, () => {
+    return { test: '2' };
+  });
+}

package/server/routes/data/controllers/data.js

@@ -1,97 +1,168 @@
-
-
-//import autoIndex from '@opengis/fastify-table/pg/funcs/autoIndex.js';
-// import pgClients from '@opengis/fastify-table/pg/pgClients.js';
-
-import { getPGAsync, autoIndex, pgClients, getSelectVal } from '@opengis/fastify-table/utils.js';
-import chartSQL from './util/chartSQL.js';
-import normalizeData from './util/normalizeData.js';
-
-import { getWidget } from '../../../../utils.js';
-
-export default async function data({
-  funcs = {}, query = {},
-}) {
-  const time = Date.now();
-  const { dashboard, widget, filter, search, samples } = query;
-
-  const widgetData = await getWidget({ dashboard, widget });
-
-  if (widgetData.status) return widgetData;
-
-  const { type, text, data = {}, controls, style, options } = widgetData;
-
-  const pg = data.db ? await getPGAsync(data.db) : pgClients.client
-
-  const { fields: cols } = await pg.query(`select * from ${data.table} limit 0`);
-  const columnTypes = cols.map(el => ({ name: el.name, type: pg.pgType?.[el.dataTypeID] }))
-
-  // data param 
-  const { x, cls, metric, table, where, tableSQL, groupby, xName } = normalizeData(widgetData, query, columnTypes);
-
-  // auto Index
-  if (pg.pk?.[data.table]) {
-    autoIndex({ table: data.table, pg, columns: [data?.time].concat([xName]).concat([groupby]).filter(el => el) }).catch(err => console.log(err))
-  }
-
-  // get group 
-  const groupData = groupby ? await pg.query(`select ${groupby} as name ,count(*) from ${tableSQL || table} group by ${groupby} order by count(*) desc limit 20`).then(el => el.rows) : null;
-
-  if (query.sql === '2') return { x, metric, table, tableSQL, data, groupData };
-
-  const order = data.order || (type === 'listbar' ? 'metric desc' : null);
-
-  const { optimizedSQL = `select * from ${tableSQL || table}` } = filter || search ? await funcs.getFilterSQL({
-    pg, table, filter, search,
-  }) : {};
-
-  const sql = (chartSQL[type] || chartSQL['chart'])({ where, metric, table: `(${optimizedSQL})q`, x, groupData, groupby, order, samples });
-
-  if (query.sql) return sql;
-
-  if (!sql || sql?.includes('undefined')) {
-    return { message: { error: 'invalid sql', type, sql, where, metric, table: `(${optimizedSQL})q`, x, groupData, groupby }, status: 500 };
-  }
-
-  const { rows, fields } = await pg.query(sql); // test with limit
-
-  if (cls) {
-    const values = rows.map((row) => row[x])?.filter((el, idx, arr) => el && arr.indexOf(el) === idx);
-    const vals = await getSelectVal({ pg, name: cls, values });
-    rows.filter((row) => row[x]).forEach((row) => {
-      Object.assign(row, { [x]: vals?.[row[x]] || row[x] });
-    });
-  }
-
-  const dimensions = fields.map(el => el.name);
-
-  const res = {
-    time: Date.now() - time,
-    dimensions,
-
-    dimensionsType: fields.map(el => pg.pgType?.[el.dataTypeID]),
-    type,
-
-    text: text ? text : (widgetData?.title || data.text),
-    //data: query.format === 'data' ? dimensions.map(el => rows.map(r => r[el])) : undefined,
-    source: query.format === 'array' ? dimensions.map(el => rows.map(r => r[el])) : rows,
-    style,
-    options,
-    controls,
-    params: {
-      x,
-      cls,
-      metric,
-      table,
-      tableSQL,
-      where,
-      groupby,
-      sql,
-    },
-    columnTypes,
-  };
-  return res;
-
-
-
-}
+import yaml from 'js-yaml';
+
+import {
+  config,
+  getPGAsync,
+  autoIndex,
+  pgClients,
+  getSelectVal,
+  getFilterSQL,
+} from '@opengis/fastify-table/utils.js';
+
+import chartSQL from './util/chartSQL.js';
+import normalizeData from './util/normalizeData.js';
+
+import { getWidget } from '../../../../utils.js';
+
+export default async function dataAPI({ funcs = {}, query = {} }) {
+  const time = Date.now();
+
+  query.metric = Array.isArray(query.metric) ? query.metric.pop() : query.metric;
+
+  const { dashboard, widget, filter, search, samples } = query;
+
+  const widgetData = await getWidget({ dashboard, widget });
+
+  if (widgetData.status) return widgetData;
+
+  const { type, text, data = {}, controls, style, options } = widgetData;
+
+  const pg = data.db ? await getPGAsync(data.db) : pgClients.client;
+
+  const { fields: cols } = await pg.query(
+    `select * from ${data.table} limit 0`
+  );
+  const columnTypes = cols.map((el) => ({
+    name: el.name,
+    type: pg.pgType?.[el.dataTypeID],
+  }));
+
+  // data param
+  const { x, cls, metric, table, where, tableSQL, groupby, xName, yName, xType, yType } =
+    normalizeData(widgetData, query, columnTypes);
+
+  // auto Index
+  if (pg.pk?.[data.table]) {
+    autoIndex({
+      table: data.table,
+      pg,
+      columns: [data?.time]
+        .concat([xName])
+        .concat([groupby])
+        .filter((el) => el),
+    }).catch((err) => console.log(err));
+  }
+
+  // get group
+  const groupData = groupby
+    ? await pg
+      .query(
+        `select ${groupby} as name ,count(*) from ${tableSQL || table} group by ${groupby} order by count(*) desc limit 20`
+      )
+      .then((el) => el.rows)
+    : null;
+
+  if (query.sql === '2') return { x, metric, table, tableSQL, data, groupData };
+
+  const order = data.order || (type === 'listbar' ? 'metric desc' : null);
+
+  const { optimizedSQL = `select * from ${tableSQL || table}` } =
+    filter || search
+      ? await getFilterSQL({
+        pg,
+        table,
+        filter,
+        search,
+      })
+      : {};
+
+  if (type?.includes('bar') && !metric?.length) {
+    return { message: 'empty widget params: metrics', status: 400 };
+  }
+
+  const sql = (chartSQL[type] || chartSQL.chart)({
+    where,
+    metric,
+    table: `(${optimizedSQL} ${samples ? 'limit 10' : ''})q`,
+    x,
+    groupData,
+    groupby,
+    order,
+    samples,
+    xType,
+  });
+
+  if (query.sql) return sql;
+
+  if (!sql || sql?.includes('undefined')) {
+    return {
+      message: {
+        error: 'invalid sql',
+        type,
+        sql,
+        where,
+        metric,
+        table: `(${optimizedSQL})q`,
+        x,
+        groupData,
+        groupby,
+      },
+      status: 500,
+    };
+  }
+
+  const { rows, fields } = await pg.query(sql); // test with limit
+
+  if (cls) {
+    const values = rows
+      .map((row) => row[x])
+      ?.filter((el, idx, arr) => el && arr.indexOf(el) === idx);
+    const vals = await getSelectVal({ pg, name: cls, values });
+    rows
+      .filter((row) => row[x])
+      .forEach((row) => {
+        Object.assign(row, { [x]: vals?.[row[x]] || row[x] });
+      });
+  }
+
+  const yml = widgetData.yml || yaml.dump(extractYml(widgetData));
+  const dimensions = fields.map((el) => el.name);
+
+  const res = {
+    time: Date.now() - time,
+    dimensions,
+    filter: xName,
+    dimensionsType: [xType, yType].filter((el) => el)?.length
+      ? [xType, yType].filter((el) => el)
+      : fields.map((el) => pg.pgType?.[el.dataTypeID]),
+    type,
+
+    text: text || widgetData?.title || data.text,
+    // data: query.format === 'data' ? dimensions.map(el => rows.map(r => r[el])) : undefined,
+    source:
+      query.format === 'array'
+        ? dimensions.map((el) => rows.map((r) => r[el]))
+        : rows,
+    style,
+    options,
+    controls,
+    yml,
+    params: config?.local ? {
+      x,
+      cls,
+      metric,
+      table,
+      tableSQL,
+      where,
+      groupby,
+      sql,
+    } : undefined,
+    columnTypes,
+  };
+  return res;
+}
+
+function extractYml(sourceData) {
+  const { title, description, type, data, style, controls } = sourceData;
+  return { title, description, type, data, style, controls };
+}

package/server/routes/data/controllers/util/chartSQL.js

@@ -1,25 +1,42 @@
-function number({ metric, where, table, samples }) {
-    const sql = `select ${metric} from ${table} where ${where} ${samples ? 'limit 10' : ''}`
-    return sql;
-}
-function table({ columns, table, where, samples }) {
-    return `select ${columns.map(el => el.name || el)}::text from ${table} where ${where} ${samples ? 'limit 10' : 'limit 20'} `
-}
-
-function chart({ metric, where, table, x, groupby, groupData, order, samples }) {
-
-    const metricData = groupData?.map(el => `${metric} filter (where ${groupby}='${el.name}') as "${el.name}"`).join(',') || `${metric} as metric`
-    const sql = `select ${x}, ${metricData} 
-    from ${table}
-    where ${where} 
-    group by ${x}
-    order by ${order || x}
-    ${samples ? 'limit 10' : 'limit 100'}`;
-    return sql;
-}
-
-function text() {
-    return undefined;
-}
-
-export default { number, chart, }
+function number({ metric, where, table, samples }) {
+  const sql = `select ${metric} from ${table} where ${where} ${samples ? 'limit 10' : ''}`;
+  return sql;
+}
+function table({ columns, table, where, samples }) {
+  return `select ${columns.map((el) => el.name || el)}::text from ${table} where ${where} ${samples ? 'limit 10' : 'limit 20'} `;
+}
+
+function chart({
+  metric,
+  where,
+  table,
+  x,
+  groupby,
+  groupData,
+  order,
+  samples,
+  xType,
+}) {
+  const xCol = x && xType?.includes('[]') ? `unnest(${x})` : x;
+
+  const metricData =
+    groupData
+      ?.map(
+        (el) =>
+          `${metric} filter (where ${groupby}='${el.name}') as "${el.name}"`
+      )
+      .join(',') || `${metric} as metric`;
+  const sql = `select ${xCol}, ${metricData} 
+    from ${table}
+    where ${where} 
+    ${xCol ? `group by ${xCol}` : ''}
+    ${order || xCol ? `order by ${order || xCol}` : ''}
+    ${samples ? 'limit 10' : 'limit 100'}`;
+  return sql;
+}
+
+function text() {
+  return undefined;
+}
+
+export default { number, chart };

package/server/routes/data/controllers/util/normalizeData.js

@@ -1,34 +1,59 @@
-function normalizeData(data, query = {}, columnTypes = []) {
-    ['x', 'groupby', 'granularity'].forEach(el => {
-        //console.log(el, query[el], columnTypes.find(col => col.name == query[el]))
-        if (!columnTypes.find(col => col.name == query[el])) { delete query[el]; }
-    });
-
-    if (!columnTypes.find(col => col.type === 'numeric' && col.name == query.metric)) { delete query.metric; }
-
-    const xName = query.x || (Array.isArray(data.x) ? data.x[0] : data.x);
-    const xTYpe = columnTypes.find(el => el.name == xName)?.type;
-
-    const granularity = xTYpe === 'date' || xTYpe?.includes('timestamp') ? (query.granularity || data.granularity || 'year') : null;
-
-    const x = (granularity ? `date_trunc('${granularity}',${xName})::date::text` : null) || xName;
-
-    const metrics = Array.isArray(data.metrics) ? data.metrics : [data.metrics];
-    const metric = (query.metric ? `sum(${query.metric})` : null) || (metrics.length ? metrics?.filter(el => el).map(el => el.fx || `${el.operator || 'sum'}(${el.name || el})`) : 'count(*)');
-
-    const { cls, table, filterCustom } = data;
-    const groupby = query.groupby || data.groupby;
-    //const orderby = query.orderby || data.orderby || 'count(*)';
-
-    const custom = query?.filterCustom?.split(',')
-    ?.map((el) => filterCustom?.find((item) => item?.name === el)?.sql)
-    ?.filter((el) => el)?.join(' and ');
-    const where = `${data.query || '1=1'} and ${custom || 'true'}`;
-
-    const tableSQL = data.tableSQL?.length
-        ? `(select * from ${data?.table} t ${data.tableSQL.join(' \n ')} where ${where})q`
-        : undefined;
-
-    return { x, cls, metric, table, where, tableSQL, groupby, xName }
-}
-export default normalizeData;
+function normalizeData(data, query = {}, columnTypes = []) {
+  ['x', 'groupby', 'granularity'].forEach((el) => {
+    // console.log(el, query[el], columnTypes.find(col => col.name == query[el]))
+    if (!columnTypes.find((col) => col.name == query[el])) {
+      delete query[el];
+    }
+  });
+
+  if (
+    !columnTypes.find(
+      (col) => col.type === 'numeric' && col.name == query.metric
+    )
+  ) {
+    delete query.metric;
+  }
+
+  const xName = query.x || (Array.isArray(data.x) ? data.x[0] : data.x);
+  const xType = columnTypes.find((el) => el.name == xName)?.type;
+
+  const granularity =
+    xType === 'date' || xType?.includes('timestamp')
+      ? query.granularity || data.granularity || 'year'
+      : null;
+
+  const x =
+    (granularity
+      ? `date_trunc('${granularity}',${xName})::date::text`
+      : null) || xName;
+
+  const metrics = Array.isArray(data.metrics) ? data.metrics : [data.metrics];
+  const metric =
+    (query.metric ? `sum(${query.metric})` : null) ||
+    (metrics.length
+      ? (metrics
+        ?.filter((el) => el && columnTypes.find((col) => col.name == (el?.name || el)))
+        ?.map((el) => el.fx || `${el.operator || 'sum'}(${el.name || el})`)?.join(',') || 'count(*)')
+      : 'count(*)');
+  
+  const yName = metrics?.[0]?.name || metrics?.[0];
+  const yType = columnTypes.find((el) => el.name == yName)?.type;
+
+  const { cls, table, filterCustom } = data;
+  const groupby = query.groupby || data.groupby;
+  // const orderby = query.orderby || data.orderby || 'count(*)';
+
+  const custom = query?.filterCustom
+    ?.split(',')
+    ?.map((el) => filterCustom?.find((item) => item?.name === el)?.sql)
+    ?.filter((el) => el)
+    ?.join(' and ');
+  const where = `${data.query || '1=1'} and ${custom || 'true'}`;
+
+  const tableSQL = data.tableSQL?.length
+    ? `(select * from ${data?.table} t ${data.tableSQL.join(' \n ')} where ${where})q`
+    : undefined;
+
+  return { x, cls, metric, table, where, tableSQL, groupby, xName, xType, yName, yType };
+}
+export default normalizeData;

package/server/routes/data/index.mjs

@@ -1,26 +1,29 @@
-import config from '../../../config.js';
-
-import data from './controllers/data.js';
-
-
-const biSchema = {
-  querystring: {
-    widget: { type: 'string', pattern: '^([\\d\\w]+)$' },
-    dashboard: { type: 'string', pattern: '^([\\d\\w]+)$' },
-    sql: { type: 'string', pattern: '^([\\d])$' }
-  },
-  params: {
-    id: { type: 'string', pattern: '^([\\d\\w]+)$' },
-  },
-};
-
-
-export default async function route(fastify, opts) {
-  const prefix = opts?.prefix || config.prefix || '/api';
-  fastify.route({
-    method: 'GET',
-    url: '/bi-data',
-    schema: biSchema,
-    handler: data,
-  });
-}
+import config from '../../../config.js';
+
+import data from './controllers/data.js';
+
+const biSchema = {
+  querystring: {
+    widget: { type: 'string', pattern: '^([\\d\\w_]+)$' },
+    dashboard: { type: 'string', pattern: '^([\\d\\w_]+)$' },
+    sql: { type: 'string', pattern: '^([\\d])$' },
+    // metric: { type: 'string', pattern: '^([\\d\\w_]+)$' },
+    x: { type: 'string', pattern: '^([\\d\\w_]+)$' },
+    granularity: { type: 'string', pattern: '^(week|month|quarter|year)$' },
+    groupby: { type: 'string', pattern: '^([\\d\\w_]+)$' },
+    filterCustom: { type: 'string', pattern: '^([\\d\\w_,]+)$' },
+  },
+  params: {
+    id: { type: 'string', pattern: '^([\\d\\w]+)$' },
+  },
+};
+
+export default async function route(fastify, opts) {
+  const prefix = opts?.prefix || config.prefix || '/api';
+  fastify.route({
+    method: 'GET',
+    url: '/bi-data',
+    schema: biSchema,
+    handler: data,
+  });
+}

package/server/routes/dataset/controllers/bi.dataset.demo.add.js

@@ -0,0 +1,97 @@
+import path from 'node:path';
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+
+import { getPG } from '@opengis/fastify-table/utils.js';
+
+const host = 'https://cdn.softpro.ua/demo/bi';
+
+
+const root = process.cwd();
+const send = (msg) => console.log(msg)
+async function downloadFile(fileName) {
+    const pg = getPG();
+    send(`${fileName} - checking if already downloaded...`);
+
+    const demoPath = path.join(root, '/log/temp/', fileName);
+    // const fileExists = await isFileExists(demoPath);
+
+    // send(`${host}/${fileName}` + demoPath)
+    const response = await fetch(`${host}/${fileName}`);
+    const body = await response.text();
+    //send(body)
+    if (response?.status !== 200) {
+        return { [fileName]: { result: 'file not found', status: 404 } };
+    }
+
+    await mkdir(path.dirname(demoPath), { recursive: true });
+
+    await writeFile(demoPath, body, 'utf8');
+
+    await pg.query(`drop table if exists ${path.parse(fileName).name}`);
+    const { exists } = await pg.query(`select to_regclass($1) is not null as exists`, [path.parse(fileName).name])
+        .then((res1) => res1.rows?.[0] || {});
+
+
+    if (exists) {
+        send(`${fileName} - dataset table already exists!`);
+        return { [fileName]: { result: 'dataset table already exists', status: 200 } };
+    }
+
+    const sql = await readFile(demoPath, 'utf8');
+    await pg.query(sql);
+    return { [fileName]: { result: 'success', status: 200 } };
+}
+
+/**
+ * Імпорт demo даних dashboard
+ *
+ * @method GET
+ * @summary Імпорт demo даних dashboard
+ * @priority 4
+ * @type api
+ * @tag bi
+ * @errors 400,500
+ * @returns {Number} status Номер помилки
+ * @returns {String} error Опис помилки
+ * @returns {Object} rows Масив з колонками таблиці
+ */
+
+const demoList = {
+    'demo.orders.sql': 'Статистика продажів (demo)',
+    'demo.cleaned_sales_data.sql': 'Статистика продажів (demo 2)',
+    'demo.video_game_sales.sql': 'Статистика продажів відеоігор (demo)',
+};
+
+export default async function biDatasetDemoAdd(req) {
+    const { pg, query = {} } = req;
+    //    const send = () => { }; // eventStream?
+
+
+
+    const sqlList = [];
+
+    const tables = Object.keys(demoList)?.map((el) => {
+        const table = path.parse(el).name;
+        sqlList.push(`insert into bi.dataset(dataset_id,name,table_name)
+            select '${el}', '${demoList?.[el]}', '${table}' on conflict(dataset_id) do update set 
+            name=excluded.name, table_name=excluded.table_name
+            returning dataset_id as id`);
+        return table;
+    });
+
+    const { count } = await pg.query(`select count(*) from pg_catalog.pg_tables where schemaname||'.'||tablename = any($1)`, [tables]).then((res1) => res1.rows?.[0] || {});
+
+    if (count === '3' && !query.nocache) {
+        const result = await pg.query(sqlList.join(';\n'));
+        const ids = (Array.isArray(result) ? result : [result]).filter((el) => el).map((el) => el?.rows?.[0]?.id);
+        return { message: { ids, message: 'all datasets already downloaded' }, status: 200 };
+    }
+
+    const res = {};
+    const resultAll = await Promise.all(Object.keys(demoList).map(async (fileName) => await downloadFile(fileName))).catch(err => console.log(err));
+
+    const result = await pg.query(sqlList.join(';\n'));
+    const ids = (Array.isArray(result) ? result : [result]).filter((el) => el).map((el) => el?.rows?.[0]?.id);
+    return { message: { ids, message: res, resultAll }, status: 200 };
+
+};