@opengis/bi 1.2.0 → 1.2.2

package/server/routes/map/controllers/vtile.js

@@ -0,0 +1,183 @@
+import Sphericalmercator from '@mapbox/sphericalmercator';
+
+import path from 'path';
+import { createHash } from 'crypto';
+import { writeFile, mkdir } from 'fs/promises';
+import { existsSync, /* readdirSync, */ readFileSync } from 'fs';
+import { getWidget } from '../../../../utils.js';
+
+import { getFolder, getFilterSQL, logger, pgClients } from '@opengis/fastify-table/utils.js';
+
+import normalizeData from '../../data/controllers/util/normalizeData.js';
+
+const mercator = new Sphericalmercator({ size: 256 });
+
+const types = {
+  point: 'ST_Point' /* ,ST_MultiPoint */,
+  polygon: 'ST_Polygon,ST_MultiPolygon',
+};
+
+const area = {
+  1: 1000000,
+  2: 100000,
+  3: 100000,
+  4: 100000,
+  5: 100000,
+  6: 100000,
+  7: 100000,
+  8: 100000,
+  9: 100000,
+  10: 50000,
+  11: 40000,
+  12: 20000,
+  13: 20000,
+};
+
+export default async function vtile(req, reply) {
+  const { params = {}, query = {} } = req;
+
+  const {
+    filter,
+    widget,
+    dashboard,
+    sql,
+    cluster,
+    type,
+    nocache,
+    geom = 'geom',
+    pointZoom = 0,
+  } = query;
+
+  if (!widget) {
+    return { message: 'not enough params: widget', status: 400 };
+  }
+
+  const { y, z } = params;
+  const x = params.x?.split('.')[0] - 0;
+
+  if (!x || !y || !z) {
+    return { message: 'not enough params: xyz', status: 400 };
+  }
+
+  const { pg = req.pg || pgClients.client, data } = await getWidget({ pg: req.pg, widget, dashboard });
+
+  const headers = {
+    'Content-Type': 'application/x-protobuf',
+    'Cache-Control': nocache || sql ? 'no-cache' : 'public, max-age=86400',
+  };
+
+  const hash = [pointZoom, filter].filter((el) => el).join();
+
+  const root = getFolder(req);
+  const file = path.join(
+    root,
+    `/map/vtile/${widget}/${hash ? `${createHash('sha1').update(hash).digest('base64')}/` : ''}${z}/${x}/${y}.mvt`
+  );
+
+  try {
+    const pkey = pg.pk?.[data?.table];
+    if (!pkey) {
+      return {
+        message: `invalid ${widget ? 'widget' : 'dashboard'}: table pk not found (${data?.table})`,
+        status: 400,
+      };
+    }
+
+    // data param
+    const {
+      table,
+      where = '1=1',
+      xName = {},
+      metric,
+    } = normalizeData(data, query);
+
+    // get sql
+    const columns = data.columns?.map((el) => el.name || el)?.join(',') || '1';
+    const filterQ = filter
+      ? await getFilterSQL({ pg, table, filter })
+      : undefined;
+    const q = `select "${pkey}", 
+                      ${data?.color ? `"${data?.color}"` : '0'} as x, 
+                      ${data.metrics?.[0] ? `"${data.metrics[0]}"::float` : '0'} as metric, 
+                      ${columns}, 
+                      ${geom} as geom 
+                      from ${filterQ ? `(${filterQ})` : table} q where ${where}`;
+
+    if (sql === '1') return q;
+
+    const koef =
+      {
+        10: 0.1,
+        11: 0.05,
+        12: 0.005,
+        13: 0.0002,
+        14: 0.00005,
+      }[z] || 0.000001;
+
+    const geomCol =
+      parseInt(z, 10) < parseInt(pointZoom, 10) || true
+        ? `ST_Centroid(${geom})`
+        : geom;
+
+    const bbox = mercator.bbox(+y, +x, +z, false /* , '900913' */);
+    const bbox2d = `'BOX(${bbox[0]} ${bbox[1]},${bbox[2]} ${bbox[3]})'::box2d`;
+
+    const areaZoom =
+      area[z] && false
+        ? ` and (st_area(st_transform(${geom},3857)))>'${area[z]}'`
+        : '';
+
+    const q1 =
+      cluster > z
+        ? `SELECT ST_AsMVT(q, 'bi', 4096, 'geom','row') as tile
+        FROM (
+          SELECT floor(random() * 100000 + 1)::int + row_number() over() as row, count(*) as point_count,
+          ST_AsMVTGeom(st_transform(st_centroid(ST_Union(geom)),3857),ST_TileEnvelope(${z},${y},${x})::box2d,4096,256,false) as geom
+          FROM (
+          SELECT geom, ST_ClusterDBSCAN(geom,${koef},1) OVER () AS cluster 
+          FROM (${q})q where ${geom} && ${bbox2d}  ) j
+          WHERE cluster IS NOT NULL
+          GROUP BY cluster
+          ORDER BY 1 DESC
+        )q`
+        : `SELECT ST_AsMVT(q, 'bi', 4096, 'geom','row') as tile
+          FROM (
+            SELECT  
+              floor(random() * 100000 + 1)::int + row_number() over() as row,
+
+              ${pkey} as id, 
+              x,
+              metric,
+              ${columns},
+              ST_AsMVTGeom(st_transform(${geomCol}, 3857),ST_TileEnvelope(${z},${y},${x})::box2d,4096,256,false) as geom
+
+              FROM (select * from (${q})q where ${geom} && ${bbox2d} 
+              
+              and ${geom} is not null and st_srid(${geom}) >0
+              
+              ${areaZoom}
+
+            ${types[type] ? ` and ST_GeometryType(geom) = any ('{ ${types[type]} }') ` : ''}
+            
+            limit 3000)q
+          ) q`;
+
+    if (sql === '2') return q1;
+
+    const { rows = [] } = await pg.query(q1);
+
+    if (sql === '3') return rows.map((el) => el.tile);
+
+    const buffer = Buffer.concat(rows.map((el) => Buffer.from(el.tile)));
+
+    if (!nocache) {
+      await mkdir(path.dirname(file), { recursive: true });
+      await writeFile(file, buffer, 'binary');
+    }
+
+    return reply.headers(headers).send(buffer);
+  } catch (err) {
+    logger.file('bi/vtile', { level: 'ERROR', error: err.toString(), query, params });
+    return { error: err.toString(), status: 500 };
+  }
+}

package/server/routes/map/index.mjs

@@ -0,0 +1,32 @@
+import map from './controllers/map.js';
+import geojson from './controllers/geojson.js';
+import vtile from './controllers/vtile.js';
+
+import cluster from './controllers/cluster.js';
+import clusterVtile from './controllers/clusterVtile.js';
+import heatmap from './controllers/heatmap.js';
+
+const biSchema = {
+  type: 'object',
+  properties: {
+    querystring: {
+      widget: { type: 'string', pattern: '^([\\d\\w]+)$' },
+      dashboard: { type: 'string', pattern: '^([\\d\\w]+)$' },
+      sql: { type: 'string', pattern: '^([\\d])$' },
+    },
+    params: {
+      id: { type: 'string', pattern: '^([\\d\\w]+)$' },
+    },
+  },
+};
+
+const policy = ['public'];
+
+export default async function route(fastify, opts) {
+  fastify.get('/bi-map', { config: { policy }, schema: biSchema }, map);
+  fastify.get('/bi-geojson', { config: { policy }, schema: biSchema }, geojson);
+  fastify.get('/bi-vtile/:z/:y/:x', { config: { policy }, schema: biSchema }, vtile);
+  fastify.get('/bi-cluster', { config: { policy }, schema: biSchema }, cluster);
+  fastify.get('/bi-cluster-vtile/:z/:y/:x', { config: { policy }, schema: biSchema }, clusterVtile);
+  fastify.get('/bi-heatmap', { config: { policy } }, heatmap);
+}

package/server/templates/page/login.html

@@ -0,0 +1,59 @@
+<!DOCTYPE html>
+<html lang="en" dir="">
+
+<head>
+    <!-- scripts -->
+    <script src="https://cdn.tailwindcss.com"></script>
+</head>
+
+<body  class="bg-[#f3f4f6] flex items-center flex-1 w-full h-[100vh] overflow-x-hidden min-h-full ">
+    <main class="w-full">
+    <div class="flex w-full max-w-sm mx-auto overflow-hidden bg-white rounded-lg shadow-lg dark:bg-gray-800 lg:max-w-4xl">
+<div class="hidden bg-left bg-cover lg:block lg:w-1/2" style="background-image: url('https://media.istockphoto.com/id/1296381479/vector/set-of-simple-infographic-graphs-and-charts-data-visualization-statistics-and-business.jpg?s=612x612&w=0&k=20&c=JpEoP-6w16OwVC49Sq2PEI8-dHF1CZnm7rHyL5DoxqA=');"></div>
+
+<div class="w-full px-6 py-8 md:px-8 lg:w-1/2">
+<div class="flex justify-center mx-auto">
+    <img  width="300" height="100"  src="https://cdn.softpro.ua/data/icons/logo-softpro.png" alt="logo">
+</div>
+
+<p class="mt-3 mb-[32px] text-xl text-center text-gray-600 dark:text-gray-200">
+    Аналітичний модуль
+</p>
+
+<form action="/api/login" method="get" >
+<div class="mt-4">
+    <label class="block mb-2 text-sm font-medium text-gray-600 dark:text-gray-200" for="LoggingEmailAddress">Електронна пошта</label>
+    <input name="username" id="LoggingEmailAddress" class="block w-full px-4 py-2 text-gray-700 bg-white border rounded-lg dark:bg-gray-800 dark:text-gray-300 dark:border-gray-600 focus:border-blue-400 focus:ring-opacity-40 dark:focus:border-blue-300 focus:outline-none focus:ring focus:ring-blue-300" type="text">
+</div>
+
+<div class="mt-4">
+    <div class="flex justify-between">
+        <label class="block mb-2 text-sm font-medium text-gray-600 dark:text-gray-200" for="loggingPassword">Пароль</label>
+        <!-- <a class="text-xs text-gray-500 dark:text-gray-300 hover:underline">Forget Password?</a> -->
+    </div>
+
+    <input id="loggingPassword" name="password"   class="block w-full px-4 py-2 text-gray-700 bg-white border rounded-lg dark:bg-gray-800 dark:text-gray-300 dark:border-gray-600 focus:border-blue-400 focus:ring-opacity-40 dark:focus:border-blue-300 focus:outline-none focus:ring focus:ring-blue-300" type="password">
+</div>
+
+<div class="mt-6">
+    <button type="submit"   class="w-full px-6 py-3 text-sm font-medium tracking-wide text-white capitalize transition-colors duration-300 transform bg-gray-800 rounded-lg hover:bg-gray-700 focus:outline-none focus:ring focus:ring-gray-300 focus:ring-opacity-50">
+         Увійти
+    </button>
+</div>
+
+</form>
+</div>
+</div>
+    </main>
+
+</body>
+
+
+<script>
+    const error = location.search.includes('wrong_pass')
+    if (error) {
+        document.getElementById('login_error').innerHTML = 'Не вірний користувач або пароль';
+    }
+</script>
+
+</html>

package/server/utils/getWidget.js

@@ -0,0 +1,117 @@
+import { getTemplate, getPGAsync, pgClients, getMeta } from '@opengis/fastify-table/utils.js';
+
+import mdToHTML from '../helpers/mdToHTML.js';
+
+async function getWidget({ pg: pg1 = pgClients.client, dashboard, widget }) {
+  if (!dashboard && !widget) {
+    return { message: `not enough params: dashboard / widget`, status: 400 };
+  }
+  const dashboardData = dashboard
+    ? await getTemplate('dashboard', dashboard)
+    : null;
+
+  const dashboardData1 = await pg1.query(`select db, table_name, widgets as "dashboardWidgets" from bi.dashboard where dashboard_id=$1`, [dashboard])
+    .then(el => el.rows?.[0] || {});
+  const { dashboardWidgets = [] } = dashboardData1;
+  const dashboardDb = typeof dashboardData1?.db === 'string' && dashboardData1?.db?.startsWith('{') ? JSON.parse(dashboardData1?.db) : dashboardData1?.db;
+
+  const dashboardIndex = dashboardData?.find((el) => el[0] == 'index.yml')?.[1]; // dashboardData?.find((el) => el[2] == 'index')?.[1]
+  const pg = dashboardDb || dashboardIndex?.db ? await getPGAsync(dashboardDb || dashboardIndex?.db) : pg1;
+
+  const { id, tableName } =
+    !dashboardData && !dashboardData1 && pg.pk['bi.dashboard'] && dashboard
+      ? await pg1.query(`select dashboard_id as id, table_name as "tableName" from bi.dashboard where $1 in (dashboard_id, name)`, [dashboard])
+        .then(el => el.rows?.[0] || {})
+      : {};
+
+  if (!dashboardData && !dashboardData1 && dashboard && !id) {
+    return { message: `dashboard not found: ${dashboard}`, status: 404 };
+  }
+
+  dashboardData?.forEach((el) => {
+    el[2] = el[0].split('.')[0];
+  });
+
+  const widgetData = dashboard
+    ? dashboardData?.find((el) => el[2] === (widget || 'index'))?.[1]
+    || dashboardWidgets?.find(el => el.name === widget)
+    : await getTemplate('widget', widget);
+
+  if (typeof widgetData === 'string' || typeof widgetData?.html === 'string' || (widgetData?.type === 'text' && typeof widgetData?.data?.text === 'string')) {
+    const html = widgetData?.html || mdToHTML(widgetData?.data?.text || widgetData);
+    return { source: html, status: 200 };
+  }
+
+  if (!id && !dashboardData && !dashboardData1 && !widgetData) {
+    return { message: `not found ${widget} ${dashboard}`, status: 404 };
+  }
+
+  const q = `select *, title as text, coalesce(data::jsonb, '{}'::jsonb) || jsonb_build_object('table', table_name) as data from bi.widget where dashboard_id=$1 and name=$2`;
+
+  const {
+    type,
+    text,
+    data = {},
+    controls,
+    style,
+    options,
+    yml
+  } = widgetData ||
+    (await pg
+      .query(q, [id || dashboard, widget])
+      .then(el => el.rows?.[0] || {}));
+
+  if (!type) {
+    return { message: `widget not found: ${widget}`, status: 404 };
+  }
+
+  Object.assign(data, {
+    table:
+      data.table ||
+      tableName ||
+      widgetData?.table_name ||
+      dashboardIndex?.table ||
+      dashboardIndex?.table_name ||
+      dashboardData1?.table_name,
+    db: dashboardDb?.db || dashboardDb || dashboardIndex?.db || widgetData?.db || pg?.options?.database,
+  });
+  const main = { ...(dashboardIndex || {}), ...widgetData, ...data, ...data?.data || {} };
+  const widgetDb = widgetData?.db ? await getPGAsync(widgetData?.db) : pg;
+
+  if (!main?.table) {
+    return {
+      message: /* json.error || */ `invalid ${widget ? 'widget' : 'dashboard'}: empty table`,
+      status: 404,
+    };
+  }
+
+  const loadTemplate = pg.pk?.['admin.doc_template'] ? await pg.query(
+    'select body from admin.doc_template where doc_type=5 and title=$1',
+    [main.table]
+  ).then(el => el.rows?.[0]?.body) : null;
+
+  const { pk, view, columns = [] } = await getMeta({ pg: widgetDb, table: loadTemplate?.table || main?.table });
+  if (!pk && !view) {
+    return {
+      message: /* json.error || */ `invalid ${widget ? 'widget' : 'dashboard'}: table not found (${loadTemplate?.table || main?.table})`,
+      status: 404,
+    };
+  }
+
+  const tableSQL = Array.isArray(main?.tableSQL) ? main?.tableSQL?.map?.(
+    (el, i) => `left join lateral(${el})t${i + 1} on 1=1`
+  )?.join?.(' ') : main?.tableSQL;
+
+  const columnList = columns.map(col => col.name);
+
+  const res = { ...main, pg: widgetDb, sql: widgetData?.sql, limit: widgetData?.limit, tableSQL, data, type, text, controls, style, options, yml };
+  if (res.x && !columnList.includes(res.x) && !loadTemplate?.table && !tableSQL?.includes?.(res.x)) {
+    Object.assign(res, { x: null, error: `column does not exists: ${res.x} at table ${res.table}` });
+  }
+  if (loadTemplate?.table) {
+    Object.assign(res.data || {}, { table: loadTemplate?.table });
+    Object.assign(res, { table: loadTemplate?.table, table_name: loadTemplate?.table, tableSQL: loadTemplate.sql?.filter?.(el => !el.disabled && el?.sql?.replace)?.map?.((el, i) => `left join lateral(${el.sql})t${i + 1} on 1=1`)?.join?.(' ') });
+  }
+  return res;
+}
+export default getWidget;

package/utils.js

@@ -0,0 +1,12 @@
+// This file contains code that we reuse
+// between our tests.
+
+// import getTemplatePath from '@opengis/fastify-table/table/controllers/utils/getTemplatePath.js';
+import getWidget from './server/utils/getWidget.js';
+import yamlSafe from './server/routes/dashboard/controllers/utils/yaml.js';
+
+export {
+  // getTemplatePath,
+  yamlSafe,
+  getWidget,
+};