@opengis/bi 1.0.11 → 1.0.12

package/server/routes/edit/controllers/widget.del.js

@@ -14,7 +14,7 @@ export default async function widgetDel({ pg = {}, params = {} }) {
     const row = await pg
       .query(
         `select a.widget_id, b.dashboard_id from bi.widget a, bi.dashboard b
-      where a.name=$2 and $1 in (b.dashboard_id, b.name) order by 1,2`,
+      where $2 in (a.widget_id, a.name) and $1 in (b.dashboard_id, b.name) order by 1,2`,
         [dashboardName, widgetName]
       )
       .then((res1) => res1.rows?.[0] || {});

package/server/routes/edit/controllers/widget.edit.js

@@ -1,7 +1,7 @@
 import dataUpdate from '@opengis/fastify-table/crud/funcs/dataUpdate.js';
 export default async function widgetEdit({ pg = {}, funcs = {}, body = {}, params = {} }) {
   const { widget: widgetName, name: dashboardName } = params;
-
+  const tableName = body.table_name;
   if (!widgetName || !dashboardName) {
     return {
       message: 'not enough params: dashboard and widget name',
@@ -13,12 +13,15 @@ export default async function widgetEdit({ pg = {}, funcs = {}, body = {}, param
     const row = await pg
       .query(
         `select a.widget_id, b.dashboard_id from bi.widget a, bi.dashboard b
-    where a.name=$2 and $1 in (b.dashboard_id, b.name) order by 1,2`,
+    where $2 in (a.widget_id, a.name) and $1 in (b.dashboard_id, b.name) order by 1,2`,
         [dashboardName, widgetName]
       )
       .then((res1) => res1.rows?.[0] || {});
-
+    const checkTable = await pg.query(`select * from bi.widget where $1 in (table_name)`, [
+      tableName
+    ]);
     const { widget_id: widgetId, dashboard_id: dashboardId } = row || {};
+    if (!checkTable.rows.length) return { message: 'bad params', status: 401 };
 
     if (!widgetId) {
       return { message: `widget not found ${widgetName}`, status: 404 };
@@ -47,7 +50,7 @@ export default async function widgetEdit({ pg = {}, funcs = {}, body = {}, param
       Array.isArray(bodyDashboard.panels) && bodyDashboard.panels?.length
         ? bodyDashboard.panels?.map((panel) => {
             if (panel.widget === widgetName) {
-              panel.col = body.col; 
+              panel.col = body.col;
             }
             return panel;
           })
@@ -56,7 +59,7 @@ export default async function widgetEdit({ pg = {}, funcs = {}, body = {}, param
       Array.isArray(bodyDashboard.widgets) && bodyDashboard?.widgets?.length
         ? bodyDashboard.widgets?.map((widget) => {
             if (widget.name === widgetName) {
-              return body; 
+              return body;
             }
             return widget;
           })

package/server/routes/map/controllers/cluster.js

@@ -0,0 +1,90 @@
+import path from 'node:path';
+import { existsSync, readFileSync } from 'node:fs';
+
+import yaml from '../../dashboard/controllers/utils/yaml.js';
+
+export default async function cluster({
+    pg, funcs, query = {}, log,
+}) {
+    const {
+        widget, filter, dashboard, search,
+    } = query;
+
+    if (!widget) {
+        return { message: 'not enough params: widget', status: 400 };
+    }
+
+    const { config = {} } = funcs;
+
+    const cwd = process.cwd();
+    const widgetDir = dashboard ? path.join(cwd, 'server/templates/dashboard', dashboard) : path.join(cwd, 'server/templates/widget');
+
+    const filePath = dashboard ? path.join(widgetDir, (widget || 'index') + '.yml') : path.join(widgetDir, widget + '.yml');
+
+    if (!existsSync(filePath)) {
+        return { message: { root: config.local ? widgetDir : undefined, error: `not found`, widget }, status: 404 };
+    }
+
+    try {
+        const fileData = readFileSync(filePath, 'utf-8');
+        const { data = {} } = filePath.includes('.yml') ? yaml.loadSafe(fileData) : {};
+
+        if (!data?.table) {
+            return { message: `invalid ${widget ? 'widget' : 'dashboard'}: table not specified`, status: 400 };
+        }
+
+        const pkey = pg.pk?.[data?.table];
+
+        if (!pkey) {
+            return { message: `invalid ${widget ? 'widget' : 'dashboard'}: table pk not found (${data?.table})`, status: 400 };
+        }
+
+        // data param 
+        const { table, query: where = '1=1', metrics = [], cluster, clusterTable = {} } = data;
+
+        if (!cluster) {
+            return { message: `invalid ${widget ? 'widget' : 'dashboard'}: cluster column not specified`, status: 400 };
+        }
+
+        if (!metrics.length) {
+            return { message: `invalid ${widget ? 'widget' : 'dashboard'}: metric columns not found`, status: 400 };
+        }
+
+        if (!clusterTable?.name) {
+            return { message: 'not enough widget params: clusterTable name', status: 400 };
+        }
+
+        if (clusterTable?.name && !pg.pk?.[clusterTable?.name]) {
+            return { message: 'invalid widget params: clusterTable pkey not found', status: 404 };
+        }
+
+        const { total = 0 } = pg.pk?.[table]
+            ? await pg.queryCache(`select oid::regclass as table, reltuples AS total from pg_class`)
+                .then((res) => res.rows?.find((row) => row.table === table))
+            : {};
+        
+        const { bbox } = await pg.query(`select st_asgeojson(box2d(geom))::json as bbox from ${table} where ${where || '1=1'}`).then((res1) => res1.rows?.[0] || {});
+
+        // get sql
+        const { optimizedSQL } = filter || search
+            ? await funcs.getFilterSQL({ pg, table, filter, search })
+            : {};
+        
+        const q = `select "${cluster}" as name, sum("${metrics[0]}") as metric 
+        from ${optimizedSQL ? `(${optimizedSQL})` : table} q 
+        left join lateral (select "${pg.pk?.[clusterTable?.name]}" as id from ${clusterTable?.name} where ${clusterTable?.title}=q."${cluster}" limit 1)b on 1=1
+        where ${where} group by ${cluster}, b.id`;
+
+        if (query.sql === '1') return q;
+        
+        // auto Index
+        funcs.autoIndex({ table, columns: (metrics || []).concat([cluster]) });
+
+        const { rows = [] } = await pg.query(q);
+
+        return { rows, bbox, count: rows.length, total };
+    } catch (err) {
+        log.error('bi/cluster', { error: err.toString(), query });
+        return { error: err.toString(), status: 500 };
+    }
+}

package/server/routes/map/controllers/clusterVtile.js

@@ -0,0 +1,144 @@
+import Sphericalmercator from '@mapbox/sphericalmercator';
+
+import path from 'path';
+import { createHash } from 'crypto';
+import { writeFile, mkdir } from 'fs/promises';
+import { existsSync, readFileSync } from 'fs';
+
+import yaml from '../../dashboard/controllers/utils/yaml.js';
+
+const mercator = new Sphericalmercator({ size: 256 });
+
+export default async function clusterVtile(req, reply) {
+    const {
+        pg, funcs, params = {}, query = {}, log,
+    } = req;
+    const { z, y } = params;
+    const x = params.x?.split('.')[0] - 0;
+
+    if (!x || !y || !z) {
+        return { message: 'not enough params: xyz', status: 400 };
+    }
+
+    const {
+        widget, filter, dashboard, search, clusterZoom, nocache, pointZoom,
+    } = query;
+
+    if (!widget) {
+        return { message: 'not enough params: widget', status: 400 };
+    }
+
+    const { config = {} } = funcs;
+
+    const cwd = process.cwd();
+    const widgetDir = dashboard ? path.join(cwd, 'server/templates/dashboard', dashboard) : path.join(cwd, 'server/templates/widget');
+
+    const filePath = dashboard ? path.join(widgetDir, (widget || 'index') + '.yml') : path.join(widgetDir, widget + '.yml');
+
+    if (!existsSync(filePath)) {
+        return { message: { root: config.local ? widgetDir : undefined, error: `not found`, widget }, status: 404 };
+    }
+
+    const headers = {
+        'Content-Type': 'application/x-protobuf',
+        'Cache-Control': nocache || query.sql || config.local
+        ? 'no-cache'
+        : 'public, max-age=86400',
+    };
+
+    const hash = [pointZoom, filter].filter((el) => el).join();
+
+    const root = funcs.getFolder(req);
+    const file = path.join(root, `/map/vtile/${widget}/${hash ? `${createHash('sha1').update(hash).digest('base64')}/` : ''}${z}/${x}/${y}.mvt`);
+
+    try {
+        const fileData = readFileSync(filePath, 'utf-8');
+        const { data = {} } = filePath.includes('.yml') ? yaml.loadSafe(fileData) : {};
+
+        if (!data?.table) {
+        return { message: `invalid ${widget ? 'widget' : 'dashboard'}: table not specified`, status: 400 };
+        }
+
+        const pkey = pg.pk?.[data?.table];
+
+        if (!pkey) {
+        return { message: `invalid ${widget ? 'widget' : 'dashboard'}: table pk not found (${data?.table})`, status: 400 };
+        }
+
+        // data param 
+        const { table, query: where = '1=1', metrics = [], cluster, clusterTable = {} } = data;
+
+        if (!cluster) {
+            return { message: `invalid ${widget ? 'widget' : 'dashboard'}: cluster column not specified`, status: 400 };
+        }
+
+        if (!metrics.length) {
+            return { message: `invalid ${widget ? 'widget' : 'dashboard'}: metric columns not found`, status: 400 };
+        }
+
+        // get sql
+        const { optimizedSQL } = filter || search
+            ? await funcs.getFilterSQL({ pg, table, filter, search })
+            : {};
+        
+        const q = `select "${cluster}" as name, sum("${metrics[0]}") as metric, 
+        ${clusterTable?.name ? `b.*` : `q.${data?.geom || 'geom'} as geom`} 
+        from ${optimizedSQL ? `(${optimizedSQL})` : table} q 
+        ${clusterTable?.name
+            ? `left join lateral (select ${pg.pk?.[clusterTable?.name]
+                ? `"${pg.pk?.[clusterTable?.name]}" as id,`
+                : ''}
+                ${clusterTable?.geom || 'geom'} as geom from ${clusterTable?.name} where ${clusterTable?.title}=q."${cluster}" limit 1)b on 1=1`
+            : ''} 
+        where ${where} group by 
+        ${cluster}, 
+        ${pg.pk?.[clusterTable?.name] ? `b.id,` : ''}
+        ${clusterTable?.name ? `b.${clusterTable?.geom || 'geom'}` : `q.${data?.geom || 'geom'}`}`;
+
+        if (query.sql === '1') return q;
+
+        const geomCol = parseInt(z, 10) < parseInt(pointZoom, 10) ? `ST_Centroid(${clusterTable?.geom || data?.geom || 'geom' })` : clusterTable?.geom || data?.geom || 'geom';
+        
+        const bbox = mercator.bbox(+y, +x, +z, false/* , '900913' */);
+        const bbox2d = `'BOX(${bbox[0]} ${bbox[1]},${bbox[2]} ${bbox[3]})'::box2d`;
+
+        const q1 = `SELECT ST_AsMVT(q, '${table}', 4096, 'geom','row') as tile
+            FROM (
+                SELECT  
+                floor(random() * 100000 + 1)::int + row_number() over() as row,
+
+                ${pg.pk?.[clusterTable?.name] ? 'id,' : ''} name, metric,
+
+                ST_AsMVTGeom(st_transform(${geomCol}, 3857),ST_TileEnvelope(${z},${y},${x})::box2d,4096,256,false) as geom
+
+                FROM (select * from (${q})q where geom && ${bbox2d} 
+                
+                and geom is not null and st_srid(geom) >0
+
+                and ST_GeometryType(geom) = any ('{ "ST_Polygon", "ST_MultiPolygon" }')
+                
+                limit 3000)q
+            ) q`;
+
+        if (query.sql === '2') return q1;
+        
+        // auto Index
+        funcs.autoIndex({ table, columns: (metrics || []).concat([cluster]) });
+
+        const { rows = [] } = await pg.query(q1);
+
+        if (query.sql === '3') return rows.map((el) => el.tile);
+
+        const buffer = Buffer.concat(rows.map((el) => Buffer.from(el.tile)));
+
+        if (!nocache && !config.local) {
+        await mkdir(path.dirname(file), { recursive: true });
+        await writeFile(file, buffer, 'binary');
+        }
+
+        return reply.headers(headers).send(buffer);
+    } catch (err) {
+        log.error('bi/clusterVtile', { error: err.toString(), query, params });
+        return { error: err.toString(), status: 500 };
+    }
+}

package/server/routes/map/controllers/geojson.js

@@ -28,7 +28,7 @@ export default async function geojson(req, reply) {
   const cwd = process.cwd();
   const widgetDir = dashboard ? path.join(cwd, 'server/templates/dashboard', dashboard) : path.join(cwd, 'server/templates/widget');
 
-  const filePath = dashboard ? path.join(widgetDir, widget || 'index' + '.yml') : path.join(widgetDir, widget + '.yml');
+  const filePath = dashboard ? path.join(widgetDir, (widget || 'index') + '.yml') : path.join(widgetDir, widget + '.yml');
 
   if (!existsSync(filePath)) {
     return { message: { root: config?.local ? widgetDir : undefined, error: `not found`, widget }, status: 404 };

package/server/routes/map/controllers/map.js

@@ -0,0 +1,63 @@
+import path from 'path';
+import { existsSync, readFileSync } from 'fs';
+
+import yaml from '../../dashboard/controllers/utils/yaml.js';
+
+export default async function map({
+    pg, funcs = {}, query = {},
+}) {
+    const { config = {} } = funcs;
+    const { widget, dashboard } = query;
+
+    const cwd = process.cwd();
+    const widgetDir = dashboard ? path.join(cwd, 'server/templates/dashboard', dashboard) : path.join(cwd, 'server/templates/widget');
+
+    const filePath = dashboard ? path.join(widgetDir, (widget || 'index') + '.yml') : path.join(widgetDir, widget + '.yml');
+  
+    if (!existsSync(filePath)) {
+      return { message: { root: config?.local ? widgetDir : undefined, error: `not found`, widget }, status: 404 };
+    }
+
+    try {
+        const fileData = readFileSync(filePath, 'utf-8');
+        const { data = {}, type } = filePath.includes('.yml') ? yaml.loadSafe(fileData) : {};
+        if (!['map' /*, 'mapCluster' */].includes(type)) {
+            return { message: 'access restricted: invalid widget type', status: 403 };
+        }
+        if (!data?.table) {
+            return { message: 'invalid widget: param table is required', status: 400 };
+        }
+        if (!pg.pk[data?.table]) {
+            return { message: 'invalid widget: table pkey not found', status: 400 };
+        }
+
+        const { q = '' } = await funcs.getFilterSQL({
+            pg, table: data?.table, filter: query.filter,
+        });
+
+        const res = {};
+        if (data?.color) {
+            const { rows = [] } = await pg.query(`select count(*), "${data.color}" as val from ${data.table} where ${data.query || '1=1'} group by "${data.color}"`);
+            Object.assign(res, { colors: rows }); // кольори для легенди
+        }
+        if (data?.metrics?.length) {
+            const metric = data?.metrics[0];
+            const q1 = `select PERCENTILE_CONT(0)  WITHIN GROUP (ORDER BY "${metric}") as "0",
+                PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY "${metric}") as "25",
+                PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY "${metric}") as "50",
+                PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY "${metric}") as "75",
+                PERCENTILE_CONT(1) WITHIN GROUP (ORDER BY "${metric}") as "100" from ${data.table} where ${data.query || '1=1'} and ${q || '1=1'}`;
+            const sizes = await pg.query(q1).then((res1) => Object.values(res1.rows?.[0] || {}));
+            Object.assign(res, { sizes }); // розміри для легенди
+        }
+        const { bbox } = await pg.query(`select st_asgeojson(box2d(geom))::json as bbox from ${data.table} where ${data.query || '1=1'}`).then((res1) => res1.rows?.[0] || {});
+        Object.assign(res, {
+            bbox,                                   // Map bounds
+            top: [],                                // 10 найкращих 
+            bottom: [],                             // 10 найгірших
+        }); 
+        return res;
+    } catch (err) {
+        return { error: err.toString(), status: 500 };
+    }
+}

package/server/routes/map/controllers/vtile.js

@@ -91,7 +91,7 @@ export default async function vtile(req, reply) {
     }
 
     // get sql
-    const filterQ = filter ? await funcs.getFilterSQL({ pg, table, filter, query }) : undefined;
+    const filterQ = filter ? await funcs.getFilterSQL({ pg, table, filter }) : undefined;
     const q = `select "${pkey}", "${xName?.name}", /* st_asgeojson(geom)::json as */ ${geom} as geom from ${filterQ ? `(${filterQ})` : table} q where ${where}`;
 
     if (sql === '1') return q;

package/server/routes/map/index.mjs

@@ -1,8 +1,10 @@
-// import config from '../../../config.js';
-
+import map from './controllers/map.js';
 import geojson from './controllers/geojson.js';
 import vtile from './controllers/vtile.js';
 
+import cluster from './controllers/cluster.js';
+import clusterVtile from './controllers/clusterVtile.js';
+
 const biSchema = {
   querystring: {
     widget: { type: 'string', pattern: '^([\\d\\w]+)$' },
@@ -14,9 +16,10 @@ const biSchema = {
   },
 };
 
-
 export default async function route(fastify, opts) {
-  // const prefix = opts?.prefix || config.prefix || '/api';
+  fastify.get('/bi-map', { schema: biSchema }, map);
   fastify.get('/bi-geojson', { schema: biSchema }, geojson);
   fastify.get('/bi-vtile/:z/:y/:x', { schema: biSchema }, vtile);
+  fastify.get('/bi-cluster', { schema: biSchema }, cluster);
+  fastify.get('/bi-cluster-vtile/:z/:y/:x', { schema: biSchema }, clusterVtile);
 }

package/server/templates/cls/demo.parcel.object_type.json

@@ -0,0 +1,12 @@
+[
+    {
+        "id": "1",
+        "text": "Теплиця",
+        "color": "#4682B4"
+    },
+    {
+        "id": "2",
+        "text": "Сад",
+        "color": "#228B22"
+    }
+]

package/server/templates/dashboard/demo/funnel.yml

@@ -0,0 +1,18 @@
+type: funnel
+title: Funnel
+
+data:
+  table: demo.cleaned_sales_data # Назва таблиці
+  query: 1=1 # Запит
+
+  metrics: # Групування
+    - name: sales
+      operator: sum
+      title: sales
+
+  x: order_date
+  granularity: quarter
+  groupby: product_line
+
+controls:
+style:

package/server/templates/dashboard/demo/heatmap.yml

@@ -0,0 +1,18 @@
+type: heatmap
+title: Heatmap
+
+data:
+  table: demo.cleaned_sales_data # Назва таблиці
+  query: 1=1 # Запит
+
+  metrics: # Групування
+    - name: sales
+      operator: sum
+      title: sales
+
+  x: order_date
+  granularity: quarter
+  groupby: product_line
+
+controls:
+style:

package/server/templates/dashboard/demo/index.yml

@@ -0,0 +1,58 @@
+title: Sales Dashboard
+description: This example dashboard provides insight into the business operations of vehicle seller
+table_name: demo.cleaned_sales_data
+panels:
+  # row 1
+  - type: column
+    col: 3
+    widgets:
+      - total_revenue
+      - total_products_sold
+
+  - widget: quarterly_revenue
+    col: 6
+  - widget: vehicle_sales_info
+    col: 3
+
+  # row 2
+  - widget: total_products_sold_by_product_line
+    col: 3
+  - widget: quarterly_revenue_by_product_line
+    col: 6
+  - widget: total_revenue_by_product_line
+    col: 3
+  #row 3
+  - widget: line
+    col: 3
+  - widget: pivot
+    col: 6
+  - widget: stat
+    col: 3
+  #row 4
+  - widget: progress
+    col: 3
+  - widget: funnel
+    col: 3
+  - widget: heatmap
+    col: 6
+  #row 5
+  - widget: waterfall
+    col: 3
+  - widget: map
+    col: 8
+grid: # сітка якщо є
+  - { 'x': 0, 'y': 0, 'w': 2, 'h': 2, 'i': 0, widget: 'quarterly_revenue' }
+  - { 'x': 1, 'y': 0, 'w': 2, 'h': 2, 'i': 0, widget: 'total_revenue' }
+
+widgets: null # самі віджети
+
+filters: null # фільтри
+
+style:
+  color:
+  stack: true # only for bar
+  orientation: vertical # only for bar, line
+  date_format: smart_date # d3 format number
+  number_format: smart_date # d3 format number
+  label: 222
+  tooltip: 333

package/server/templates/dashboard/demo/line.yml

@@ -0,0 +1,19 @@
+type: line
+title: Line bar
+
+data:
+  table: demo.cleaned_sales_data # Назва таблиці
+  query: 1=1 # Запит
+
+  metrics: # Групування
+    - name: sales
+      operator: sum
+      title: sales
+
+  x: order_date
+  granularity: quarter
+  orderby: year
+
+controls:
+style:
+  area: true

package/server/templates/dashboard/demo/map.yml

@@ -0,0 +1,13 @@
+type: map
+title: Регіональний розподіл
+
+data:
+  table: demo.parcel_object_culture # Назва таблиці
+  query: 1=1 # Запит
+
+  metrics: # Розмір точки
+    - obj_area
+  color: object_type # color
+  clusterZoom: 15
+  columns: # додаткові колонки
+    - obj_area

package/server/templates/dashboard/demo/pivot.yml

@@ -0,0 +1,18 @@
+type: pivot
+title: Pivot
+
+data:
+  table: demo.cleaned_sales_data # Назва таблиці
+  query: 1=1 # Запит
+
+  metrics: # Групування
+    - name: sales
+      operator: sum
+      title: sales
+
+  x: order_date
+  granularity: quarter
+  groupby: product_line
+
+controls:
+style:

package/server/templates/dashboard/demo/progress.yml

@@ -0,0 +1,15 @@
+type: progress
+title: Progress bar
+
+data:
+  table: demo.cleaned_sales_data # Назва таблиці
+  query: 1=1 # Запит
+
+  metrics: # Групування
+    - name: sales
+      operator: sum
+      title: sales
+  x: product_line
+
+controls:
+style:

package/server/templates/dashboard/demo/quarterly_revenue.yml

@@ -0,0 +1,17 @@
+type: bar
+title: Quarterly Revenue
+
+data:
+  table: demo.cleaned_sales_data # Назва таблиці
+  query: 1=1 # Запит
+
+  metrics: # Групування
+    - name: sales
+      operator: sum
+      title: sales
+
+  x: order_date
+  granularity: quarter
+  orderby: year
+
+controls:

package/server/templates/dashboard/demo/quarterly_revenue_by_product_line.yml

@@ -0,0 +1,19 @@
+type: bar
+title: Quarterly Revenue (By Product Line)
+
+data:
+  table: demo.cleaned_sales_data # Назва таблиці
+  query: 1=1 # Запит
+
+  metrics: # Групування
+    - name: sales
+      operator: sum
+      title: sales
+
+  x: order_date
+  granularity: quarter
+  groupby: product_line
+
+controls:
+style:
+  stack: true

package/server/templates/dashboard/demo/stat.yml

@@ -0,0 +1,15 @@
+type: stat
+title: Stat bar
+
+data:
+  table: demo.cleaned_sales_data # Назва таблиці
+  query: 1=1 # Запит
+
+  metrics: # Групування
+    - name: sales
+      operator: sum
+      title: sales
+  x: product_line
+
+controls:
+style:

package/server/templates/dashboard/demo/total_products_sold.yml

@@ -0,0 +1,9 @@
+type: number
+title: Total Products Sold
+
+data:
+  table: demo.cleaned_sales_data # Назва таблиці
+  query: 1=1 # Запит
+  metrics:
+    - quantity_ordered
+controls:

package/server/templates/dashboard/demo/total_products_sold_by_product_line.yml

@@ -0,0 +1,12 @@
+type: listbar
+title: Total Products Sold (By Product Line)
+
+data:
+  table: demo.cleaned_sales_data # Назва таблиці
+  query: 1=1 # Запит
+
+  metrics: # Групування
+    - name: quantity_ordered
+      operator: sum
+      title: quantity_sold
+  x: product_line

package/server/templates/dashboard/demo/total_revenue.yml

@@ -0,0 +1,10 @@
+type: number
+title: Total Revenue
+
+data:
+  table: demo.cleaned_sales_data # Назва таблиці
+  query: 1=1 # Запит
+  metrics: sales
+controls:
+style:
+  prefix: '$'