@opengis/bi 1.0.14 → 1.0.15

package/server/routes/data/controllers/data.js

@@ -1,157 +1,168 @@
-// import autoIndex from '@opengis/fastify-table/pg/funcs/autoIndex.js';
-// import pgClients from '@opengis/fastify-table/pg/pgClients.js';
-import yaml from 'js-yaml';
-
-import {
-  getPGAsync,
-  autoIndex,
-  pgClients,
-  getSelectVal,
-} from '@opengis/fastify-table/utils.js';
-import chartSQL from './util/chartSQL.js';
-import normalizeData from './util/normalizeData.js';
-
-import { getWidget } from '../../../../utils.js';
-
-export default async function dataAPI({ funcs = {}, query = {} }) {
-  const time = Date.now();
-  const { dashboard, widget, filter, search, samples } = query;
-
-  const widgetData = await getWidget({ dashboard, widget });
-
-  if (widgetData.status) return widgetData;
-
-  const { type, text, data = {}, controls, style, options } = widgetData;
-
-  const pg = data.db ? await getPGAsync(data.db) : pgClients.client;
-
-  const { fields: cols } = await pg.query(
-    `select * from ${data.table} limit 0`
-  );
-  const columnTypes = cols.map((el) => ({
-    name: el.name,
-    type: pg.pgType?.[el.dataTypeID],
-  }));
-
-  // data param
-  const { x, cls, metric, table, where, tableSQL, groupby, xName } =
-    normalizeData(widgetData, query, columnTypes);
-
-  // auto Index
-  if (pg.pk?.[data.table]) {
-    autoIndex({
-      table: data.table,
-      pg,
-      columns: [data?.time]
-        .concat([xName])
-        .concat([groupby])
-        .filter((el) => el),
-    }).catch((err) => console.log(err));
-  }
-
-  // get group
-  const groupData = groupby
-    ? await pg
-      .query(
-        `select ${groupby} as name ,count(*) from ${tableSQL || table} group by ${groupby} order by count(*) desc limit 20`
-      )
-      .then((el) => el.rows)
-    : null;
-
-  if (query.sql === '2') return { x, metric, table, tableSQL, data, groupData };
-
-  const order = data.order || (type === 'listbar' ? 'metric desc' : null);
-
-  const { optimizedSQL = `select * from ${tableSQL || table}` } =
-    filter || search
-      ? await funcs.getFilterSQL({
-        pg,
-        table,
-        filter,
-        search,
-      })
-      : {};
-
-  const sql = (chartSQL[type] || chartSQL.chart)({
-    where,
-    metric,
-    table: `(${optimizedSQL})q`,
-    x,
-    groupData,
-    groupby,
-    order,
-    samples,
-  });
-
-  if (query.sql) return sql;
-
-  if (!sql || sql?.includes('undefined')) {
-    return {
-      message: {
-        error: 'invalid sql',
-        type,
-        sql,
-        where,
-        metric,
-        table: `(${optimizedSQL})q`,
-        x,
-        groupData,
-        groupby,
-      },
-      status: 500,
-    };
-  }
-
-  const { rows, fields } = await pg.query(sql); // test with limit
-
-  if (cls) {
-    const values = rows
-      .map((row) => row[x])
-      ?.filter((el, idx, arr) => el && arr.indexOf(el) === idx);
-    const vals = await getSelectVal({ pg, name: cls, values });
-    rows
-      .filter((row) => row[x])
-      .forEach((row) => {
-        Object.assign(row, { [x]: vals?.[row[x]] || row[x] });
-      });
-  }
-
-  const yml = widgetData.yml || yaml.dump(extractYml(widgetData));
-  const dimensions = fields.map((el) => el.name);
-
-  const res = {
-    time: Date.now() - time,
-    dimensions,
-
-    dimensionsType: fields.map((el) => pg.pgType?.[el.dataTypeID]),
-    type,
-
-    text: text || widgetData?.title || data.text,
-    // data: query.format === 'data' ? dimensions.map(el => rows.map(r => r[el])) : undefined,
-    source:
-      query.format === 'array'
-        ? dimensions.map((el) => rows.map((r) => r[el]))
-        : rows,
-    style,
-    options,
-    controls,
-    yml,
-    params: {
-      x,
-      cls,
-      metric,
-      table,
-      tableSQL,
-      where,
-      groupby,
-      sql,
-    },
-    columnTypes,
-  };
-  return res;
-}
-
-function extractYml(sourceData) {
-  const { title, description, type, data, style, controls } = sourceData;
-  return { title, description, type, data, style, controls };
+import yaml from 'js-yaml';
+
+import {
+  config,
+  getPGAsync,
+  autoIndex,
+  pgClients,
+  getSelectVal,
+  getFilterSQL,
+} from '@opengis/fastify-table/utils.js';
+
+import chartSQL from './util/chartSQL.js';
+import normalizeData from './util/normalizeData.js';
+
+import { getWidget } from '../../../../utils.js';
+
+export default async function dataAPI({ funcs = {}, query = {} }) {
+  const time = Date.now();
+
+  query.metric = Array.isArray(query.metric) ? query.metric.pop() : query.metric;
+
+  const { dashboard, widget, filter, search, samples } = query;
+
+  const widgetData = await getWidget({ dashboard, widget });
+
+  if (widgetData.status) return widgetData;
+
+  const { type, text, data = {}, controls, style, options } = widgetData;
+
+  const pg = data.db ? await getPGAsync(data.db) : pgClients.client;
+
+  const { fields: cols } = await pg.query(
+    `select * from ${data.table} limit 0`
+  );
+  const columnTypes = cols.map((el) => ({
+    name: el.name,
+    type: pg.pgType?.[el.dataTypeID],
+  }));
+
+  // data param
+  const { x, cls, metric, table, where, tableSQL, groupby, xName, yName, xType, yType } =
+    normalizeData(widgetData, query, columnTypes);
+
+  // auto Index
+  if (pg.pk?.[data.table]) {
+    autoIndex({
+      table: data.table,
+      pg,
+      columns: [data?.time]
+        .concat([xName])
+        .concat([groupby])
+        .filter((el) => el),
+    }).catch((err) => console.log(err));
+  }
+
+  // get group
+  const groupData = groupby
+    ? await pg
+      .query(
+        `select ${groupby} as name ,count(*) from ${tableSQL || table} group by ${groupby} order by count(*) desc limit 20`
+      )
+      .then((el) => el.rows)
+    : null;
+
+  if (query.sql === '2') return { x, metric, table, tableSQL, data, groupData };
+
+  const order = data.order || (type === 'listbar' ? 'metric desc' : null);
+
+  const { optimizedSQL = `select * from ${tableSQL || table}` } =
+    filter || search
+      ? await getFilterSQL({
+        pg,
+        table,
+        filter,
+        search,
+      })
+      : {};
+
+  if (type?.includes('bar') && !metric?.length) {
+    return { message: 'empty widget params: metrics', status: 400 };
+  }
+
+  const sql = (chartSQL[type] || chartSQL.chart)({
+    where,
+    metric,
+    table: `(${optimizedSQL} ${samples ? 'limit 10' : ''})q`,
+    x,
+    groupData,
+    groupby,
+    order,
+    samples,
+    xType,
+  });
+
+  if (query.sql) return sql;
+
+  if (!sql || sql?.includes('undefined')) {
+    return {
+      message: {
+        error: 'invalid sql',
+        type,
+        sql,
+        where,
+        metric,
+        table: `(${optimizedSQL})q`,
+        x,
+        groupData,
+        groupby,
+      },
+      status: 500,
+    };
+  }
+
+  const { rows, fields } = await pg.query(sql); // test with limit
+
+  if (cls) {
+    const values = rows
+      .map((row) => row[x])
+      ?.filter((el, idx, arr) => el && arr.indexOf(el) === idx);
+    const vals = await getSelectVal({ pg, name: cls, values });
+    rows
+      .filter((row) => row[x])
+      .forEach((row) => {
+        Object.assign(row, { [x]: vals?.[row[x]] || row[x] });
+      });
+  }
+
+  const yml = widgetData.yml || yaml.dump(extractYml(widgetData));
+  const dimensions = fields.map((el) => el.name);
+
+  const res = {
+    time: Date.now() - time,
+    dimensions,
+    filter: xName,
+    dimensionsType: [xType, yType].filter((el) => el)?.length
+      ? [xType, yType].filter((el) => el)
+      : fields.map((el) => pg.pgType?.[el.dataTypeID]),
+    type,
+
+    text: text || widgetData?.title || data.text,
+    // data: query.format === 'data' ? dimensions.map(el => rows.map(r => r[el])) : undefined,
+    source:
+      query.format === 'array'
+        ? dimensions.map((el) => rows.map((r) => r[el]))
+        : rows,
+    style,
+    options,
+    controls,
+    yml,
+    params: config?.local ? {
+      x,
+      cls,
+      metric,
+      table,
+      tableSQL,
+      where,
+      groupby,
+      sql,
+    } : undefined,
+    columnTypes,
+  };
+  return res;
+}
+
+function extractYml(sourceData) {
+  const { title, description, type, data, style, controls } = sourceData;
+  return { title, description, type, data, style, controls };
 }

package/server/routes/data/controllers/util/chartSQL.js

@@ -1,39 +1,42 @@
-function number({ metric, where, table, samples }) {
-  const sql = `select ${metric} from ${table} where ${where} ${samples ? 'limit 10' : ''}`;
-  return sql;
-}
-function table({ columns, table, where, samples }) {
-  return `select ${columns.map((el) => el.name || el)}::text from ${table} where ${where} ${samples ? 'limit 10' : 'limit 20'} `;
-}
-
-function chart({
-  metric,
-  where,
-  table,
-  x,
-  groupby,
-  groupData,
-  order,
-  samples,
-}) {
-  const metricData =
-    groupData
-      ?.map(
-        (el) =>
-          `${metric} filter (where ${groupby}='${el.name}') as "${el.name}"`
-      )
-      .join(',') || `${metric} as metric`;
-  const sql = `select ${x}, ${metricData} 
-    from ${table}
-    where ${where} 
-    group by ${x}
-    order by ${order || x}
-    ${samples ? 'limit 10' : 'limit 100'}`;
-  return sql;
-}
-
-function text() {
-  return undefined;
-}
-
-export default { number, chart };
+function number({ metric, where, table, samples }) {
+  const sql = `select ${metric} from ${table} where ${where} ${samples ? 'limit 10' : ''}`;
+  return sql;
+}
+function table({ columns, table, where, samples }) {
+  return `select ${columns.map((el) => el.name || el)}::text from ${table} where ${where} ${samples ? 'limit 10' : 'limit 20'} `;
+}
+
+function chart({
+  metric,
+  where,
+  table,
+  x,
+  groupby,
+  groupData,
+  order,
+  samples,
+  xType,
+}) {
+  const xCol = x && xType?.includes('[]') ? `unnest(${x})` : x;
+
+  const metricData =
+    groupData
+      ?.map(
+        (el) =>
+          `${metric} filter (where ${groupby}='${el.name}') as "${el.name}"`
+      )
+      .join(',') || `${metric} as metric`;
+  const sql = `select ${xCol}, ${metricData} 
+    from ${table}
+    where ${where} 
+    ${xCol ? `group by ${xCol}` : ''}
+    ${order || xCol ? `order by ${order || xCol}` : ''}
+    ${samples ? 'limit 10' : 'limit 100'}`;
+  return sql;
+}
+
+function text() {
+  return undefined;
+}
+
+export default { number, chart };

package/server/routes/data/controllers/util/normalizeData.js

@@ -1,56 +1,59 @@
-function normalizeData(data, query = {}, columnTypes = []) {
-  ['x', 'groupby', 'granularity'].forEach((el) => {
-    // console.log(el, query[el], columnTypes.find(col => col.name == query[el]))
-    if (!columnTypes.find((col) => col.name == query[el])) {
-      delete query[el];
-    }
-  });
-
-  if (
-    !columnTypes.find(
-      (col) => col.type === 'numeric' && col.name == query.metric
-    )
-  ) {
-    delete query.metric;
-  }
-
-  const xName = query.x || (Array.isArray(data.x) ? data.x[0] : data.x);
-  const xTYpe = columnTypes.find((el) => el.name == xName)?.type;
-
-  const granularity =
-    xTYpe === 'date' || xTYpe?.includes('timestamp')
-      ? query.granularity || data.granularity || 'year'
-      : null;
-
-  const x =
-    (granularity
-      ? `date_trunc('${granularity}',${xName})::date::text`
-      : null) || xName;
-
-  const metrics = Array.isArray(data.metrics) ? data.metrics : [data.metrics];
-  const metric =
-    (query.metric ? `sum(${query.metric})` : null) ||
-    (metrics.length
-      ? metrics
-          ?.filter((el) => el)
-          .map((el) => el.fx || `${el.operator || 'sum'}(${el.name || el})`)
-      : 'count(*)');
-
-  const { cls, table, filterCustom } = data;
-  const groupby = query.groupby || data.groupby;
-  // const orderby = query.orderby || data.orderby || 'count(*)';
-
-  const custom = query?.filterCustom
-    ?.split(',')
-    ?.map((el) => filterCustom?.find((item) => item?.name === el)?.sql)
-    ?.filter((el) => el)
-    ?.join(' and ');
-  const where = `${data.query || '1=1'} and ${custom || 'true'}`;
-
-  const tableSQL = data.tableSQL?.length
-    ? `(select * from ${data?.table} t ${data.tableSQL.join(' \n ')} where ${where})q`
-    : undefined;
-
-  return { x, cls, metric, table, where, tableSQL, groupby, xName };
-}
-export default normalizeData;
+function normalizeData(data, query = {}, columnTypes = []) {
+  ['x', 'groupby', 'granularity'].forEach((el) => {
+    // console.log(el, query[el], columnTypes.find(col => col.name == query[el]))
+    if (!columnTypes.find((col) => col.name == query[el])) {
+      delete query[el];
+    }
+  });
+
+  if (
+    !columnTypes.find(
+      (col) => col.type === 'numeric' && col.name == query.metric
+    )
+  ) {
+    delete query.metric;
+  }
+
+  const xName = query.x || (Array.isArray(data.x) ? data.x[0] : data.x);
+  const xType = columnTypes.find((el) => el.name == xName)?.type;
+
+  const granularity =
+    xType === 'date' || xType?.includes('timestamp')
+      ? query.granularity || data.granularity || 'year'
+      : null;
+
+  const x =
+    (granularity
+      ? `date_trunc('${granularity}',${xName})::date::text`
+      : null) || xName;
+
+  const metrics = Array.isArray(data.metrics) ? data.metrics : [data.metrics];
+  const metric =
+    (query.metric ? `sum(${query.metric})` : null) ||
+    (metrics.length
+      ? (metrics
+        ?.filter((el) => el && columnTypes.find((col) => col.name == (el?.name || el)))
+        ?.map((el) => el.fx || `${el.operator || 'sum'}(${el.name || el})`)?.join(',') || 'count(*)')
+      : 'count(*)');
+  
+  const yName = metrics?.[0]?.name || metrics?.[0];
+  const yType = columnTypes.find((el) => el.name == yName)?.type;
+
+  const { cls, table, filterCustom } = data;
+  const groupby = query.groupby || data.groupby;
+  // const orderby = query.orderby || data.orderby || 'count(*)';
+
+  const custom = query?.filterCustom
+    ?.split(',')
+    ?.map((el) => filterCustom?.find((item) => item?.name === el)?.sql)
+    ?.filter((el) => el)
+    ?.join(' and ');
+  const where = `${data.query || '1=1'} and ${custom || 'true'}`;
+
+  const tableSQL = data.tableSQL?.length
+    ? `(select * from ${data?.table} t ${data.tableSQL.join(' \n ')} where ${where})q`
+    : undefined;
+
+  return { x, cls, metric, table, where, tableSQL, groupby, xName, xType, yName, yType };
+}
+export default normalizeData;

package/server/routes/data/index.mjs

@@ -1,24 +1,29 @@
-import config from '../../../config.js';
-
-import data from './controllers/data.js';
-
-const biSchema = {
-  querystring: {
-    widget: { type: 'string', pattern: '^([\\d\\w]+)$' },
-    dashboard: { type: 'string', pattern: '^([\\d\\w]+)$' },
-    sql: { type: 'string', pattern: '^([\\d])$' },
-  },
-  params: {
-    id: { type: 'string', pattern: '^([\\d\\w]+)$' },
-  },
-};
-
-export default async function route(fastify, opts) {
-  const prefix = opts?.prefix || config.prefix || '/api';
-  fastify.route({
-    method: 'GET',
-    url: '/bi-data',
-    schema: biSchema,
-    handler: data,
-  });
-}
+import config from '../../../config.js';
+
+import data from './controllers/data.js';
+
+const biSchema = {
+  querystring: {
+    widget: { type: 'string', pattern: '^([\\d\\w_]+)$' },
+    dashboard: { type: 'string', pattern: '^([\\d\\w_]+)$' },
+    sql: { type: 'string', pattern: '^([\\d])$' },
+    // metric: { type: 'string', pattern: '^([\\d\\w_]+)$' },
+    x: { type: 'string', pattern: '^([\\d\\w_]+)$' },
+    granularity: { type: 'string', pattern: '^(week|month|quarter|year)$' },
+    groupby: { type: 'string', pattern: '^([\\d\\w_]+)$' },
+    filterCustom: { type: 'string', pattern: '^([\\d\\w_,]+)$' },
+  },
+  params: {
+    id: { type: 'string', pattern: '^([\\d\\w]+)$' },
+  },
+};
+
+export default async function route(fastify, opts) {
+  const prefix = opts?.prefix || config.prefix || '/api';
+  fastify.route({
+    method: 'GET',
+    url: '/bi-data',
+    schema: biSchema,
+    handler: data,
+  });
+}