@opengis/bi 1.0.13 → 1.0.15

package/server/routes/map/controllers/clusterVtile.js

@@ -1,143 +1,166 @@
-import Sphericalmercator from '@mapbox/sphericalmercator';
-
-import path from 'path';
-import { createHash } from 'crypto';
-import { writeFile, mkdir } from 'fs/promises';
-
-import { getWidget } from '../../../../utils.js';
-
-const mercator = new Sphericalmercator({ size: 256 });
-
-const clusterExists = {};
-
-export default async function clusterVtile(req, reply) {
-    const {
-        pg, funcs, params = {}, query = {}, log,
-    } = req;
-    const { z, y } = params;
-    const x = params.x?.split('.')[0] - 0;
-
-    if (!x || !y || !z) {
-        return { message: 'not enough params: xyz', status: 400 };
-    }
-
-    const {
-        widget, filter, dashboard, search, clusterZoom, nocache, pointZoom,
-    } = query;
-
-    if (!widget) {
-        return { message: 'not enough params: widget', status: 400 };
-    }
-
-
-    const { data } = await getWidget({ dashboard, widget })
-
-
-    const headers = {
-        'Content-Type': 'application/x-protobuf',
-        'Cache-Control': nocache || query.sql
-            ? 'no-cache'
-            : 'public, max-age=86400',
-    };
-
-    const hash = [pointZoom, filter].filter((el) => el).join();
-
-    const root = funcs.getFolder(req);
-    const file = path.join(root, `/map/vtile/${widget}/${hash ? `${createHash('sha1').update(hash).digest('base64')}/` : ''}${z}/${x}/${y}.mvt`);
-
-    try {
-
-
-        if (!data?.table) {
-            return { message: `invalid ${widget ? 'widget' : 'dashboard'}: table not specified`, status: 400 };
-        }
-
-        const pkey = pg.pk?.[data?.table];
-
-        if (!pkey) {
-            return { message: `invalid ${widget ? 'widget' : 'dashboard'}: table pk not found (${data?.table})`, status: 400 };
-        }
-
-        // data param 
-        const { table, query: where = '1=1', metrics = [], cluster, clusterTable = {} } = data;
-        if (!clusterTable?.name) {
-            Object.assign(data, { clusterTable: { name: 'bi.cluster', title, query: `type='${data.cluster}'` } });
-        }
-
-        if (!clusterExists[data?.cluster]) {
-            // const res = await fetch('https://cdn.softpro.ua/data/bi/region-ua.json');
-            // if (res?.status !== 200) {
-            // }
-            // await pg.query(`insert into bi.cluster (title,type,geom) values()`);
-            // clusterExists[data?.cluster] = 1;
-        }
-
-        if (!cluster) {
-            return { message: `invalid ${widget ? 'widget' : 'dashboard'}: cluster column not specified`, status: 400 };
-        }
-
-        if (!metrics.length) {
-            return { message: `invalid ${widget ? 'widget' : 'dashboard'}: metric columns not found`, status: 400 };
-        }
-
-        // get sql
-        const { optimizedSQL } = filter || search
-            ? await funcs.getFilterSQL({ pg, table, filter, search })
-            : {};
-
-        const q = `select "${cluster}" as name, sum("${metrics[0]}")::float as metric, b.* 
-        from ${optimizedSQL ? `(${optimizedSQL})` : table} q 
-            left join lateral (select "${pg.pk?.[clusterTable?.name]}" as id,
-                ${clusterTable?.geom || 'geom'} as geom from ${clusterTable?.name} 
-                where ${clusterTable?.query || '1=1'} and ${clusterTable?.title}=q."${cluster}" limit 1
-            )b on 1=1 
-        where ${where} group by 
-        ${cluster}, b.id, b.${clusterTable?.geom || 'geom'}`;
-
-        if (query.sql === '1') return q;
-
-        const geomCol = parseInt(z, 10) < parseInt(pointZoom, 10) ? `ST_Centroid(${clusterTable?.geom || data?.geom || 'geom'})` : clusterTable?.geom || data?.geom || 'geom';
-
-        const bbox = mercator.bbox(+y, +x, +z, false/* , '900913' */);
-        const bbox2d = `'BOX(${bbox[0]} ${bbox[1]},${bbox[2]} ${bbox[3]})'::box2d`;
-
-        const q1 = `SELECT ST_AsMVT(q, 'bi', 4096, 'geom','row') as tile
-            FROM (
-                SELECT  
-                floor(random() * 100000 + 1)::int + row_number() over() as row,
-
-                ${pg.pk?.[clusterTable?.name] ? 'id,' : ''} name, metric,
-
-                ST_AsMVTGeom(st_transform(${geomCol}, 3857),ST_TileEnvelope(${z},${y},${x})::box2d,4096,256,false) as geom
-
-                FROM (select * from (${q})q where geom && ${bbox2d} 
-                
-                and geom is not null and st_srid(geom) >0
-
-                and ST_GeometryType(geom) = any ('{ "ST_Polygon", "ST_MultiPolygon" }')
-                
-                limit 3000)q
-            ) q`;
-
-        if (query.sql === '2') return q1;
-
-        // auto Index
-        funcs.autoIndex({ table, columns: (metrics || []).concat([cluster]) });
-
-        const { rows = [] } = await pg.query(q1);
-
-        if (query.sql === '3') return rows.map((el) => el.tile);
-
-        const buffer = Buffer.concat(rows.map((el) => Buffer.from(el.tile)));
-
-        if (!nocache) {
-            await mkdir(path.dirname(file), { recursive: true });
-            await writeFile(file, buffer, 'binary');
-        }
-
-        return reply.headers(headers).send(buffer);
-    } catch (err) {
-        log.error('bi/clusterVtile', { error: err.toString(), query, params });
-        return { error: err.toString(), status: 500 };
-    }
-}
+import Sphericalmercator from '@mapbox/sphericalmercator';
+
+import path from 'path';
+import { createHash } from 'crypto';
+import { writeFile, mkdir } from 'fs/promises';
+
+import { getFolder, getFilterSQL, autoIndex } from '@opengis/fastify-table/utils.js';
+
+import { getWidget } from '../../../../utils.js';
+
+import downloadClusterData from './utils/downloadClusterData.js';
+
+const mercator = new Sphericalmercator({ size: 256 });
+
+const clusterExists = {};
+
+export default async function clusterVtile(req, reply) {
+  const { pg, funcs, params = {}, query = {}, log } = req;
+  const { z, y } = params;
+  const x = params.x?.split('.')[0] - 0;
+
+  if (!x || !y || !z) {
+    return { message: 'not enough params: xyz', status: 400 };
+  }
+
+  const { widget, filter, dashboard, search, clusterZoom, nocache, pointZoom } =
+    query;
+
+  if (!widget) {
+    return { message: 'not enough params: widget', status: 400 };
+  }
+
+  const { data } = await getWidget({ dashboard, widget });
+
+  const headers = {
+    'Content-Type': 'application/x-protobuf',
+    'Cache-Control':
+      nocache || query.sql ? 'no-cache' : 'public, max-age=86400',
+  };
+
+  const hash = [pointZoom, filter].filter((el) => el).join();
+
+  const root = getFolder(req);
+  const file = path.join(
+    root,
+    `/map/vtile/${widget}/${hash ? `${createHash('sha1').update(hash).digest('base64')}/` : ''}${z}/${x}/${y}.mvt`
+  );
+
+  try {
+    if (!data?.table) {
+      return {
+        message: `invalid ${widget ? 'widget' : 'dashboard'}: table not specified`,
+        status: 400,
+      };
+    }
+
+    const pkey = pg.pk?.[data?.table];
+
+    if (!pkey) {
+      return {
+        message: `invalid ${widget ? 'widget' : 'dashboard'}: table pk not found (${data?.table})`,
+        status: 400,
+      };
+    }
+
+    // data param
+    const {
+      table,
+      query: where = '1=1',
+      metrics = [],
+      cluster,
+      clusterTable = {},
+    } = data;
+    if (!clusterTable?.name) {
+      Object.assign(clusterTable, {
+        name: 'bi.cluster',
+        title: 'title',
+        query: `type='${data.cluster}'`,
+      });
+    }
+
+    if (cluster && !clusterExists[data.cluster]) {
+      const res = await downloadClusterData({ pg, log, cluster });
+      if (res) return res;
+      clusterExists[cluster] = 1;
+    }
+
+    if (!cluster) {
+      return {
+        message: `invalid ${widget ? 'widget' : 'dashboard'}: cluster column not specified`,
+        status: 400,
+      };
+    }
+
+    if (!metrics.length) {
+      return {
+        message: `invalid ${widget ? 'widget' : 'dashboard'}: metric columns not found`,
+        status: 400,
+      };
+    }
+
+    // get sql
+    const { optimizedSQL } =
+      filter || search
+        ? await getFilterSQL({ pg, table, filter, search })
+        : {};
+
+    const q = `select "${cluster}" as name, sum("${metrics[0]}")::float as metric, b.* 
+        from ${optimizedSQL ? `(${optimizedSQL})` : table} q 
+            left join lateral (select "${pg.pk?.[clusterTable?.name]}" as id,
+                ${clusterTable?.geom || 'geom'} as geom from ${clusterTable?.name} 
+                where ${clusterTable?.query || '1=1'} and ${clusterTable?.title}=q."${cluster}" limit 1
+            )b on 1=1 
+        where ${where} group by 
+        ${cluster}, b.id, b.${clusterTable?.geom || 'geom'}`;
+
+    if (query.sql === '1') return q;
+
+    const geomCol =
+      parseInt(z, 10) < parseInt(pointZoom, 10)
+        ? `ST_Centroid(${clusterTable?.geom || data?.geom || 'geom'})`
+        : clusterTable?.geom || data?.geom || 'geom';
+
+    const bbox = mercator.bbox(+y, +x, +z, false /* , '900913' */);
+    const bbox2d = `'BOX(${bbox[0]} ${bbox[1]},${bbox[2]} ${bbox[3]})'::box2d`;
+
+    const q1 = `SELECT ST_AsMVT(q, 'bi', 4096, 'geom','row') as tile
+            FROM (
+                SELECT  
+                floor(random() * 100000 + 1)::int + row_number() over() as row,
+
+                ${pg.pk?.[clusterTable?.name] ? 'id,' : ''} name, metric,
+
+                ST_AsMVTGeom(st_transform(${geomCol}, 3857),ST_TileEnvelope(${z},${y},${x})::box2d,4096,256,false) as geom
+
+                FROM (select * from (${q})q where geom && ${bbox2d} 
+                
+                and geom is not null and st_srid(geom) >0
+
+                and ST_GeometryType(geom) = any ('{ "ST_Polygon", "ST_MultiPolygon" }')
+                
+                limit 3000)q
+            ) q`;
+
+    if (query.sql === '2') return q1;
+
+    // auto Index
+    autoIndex({ table, columns: (metrics || []).concat([cluster]) });
+
+    const { rows = [] } = await pg.query(q1);
+
+    if (query.sql === '3') return rows.map((el) => el.tile);
+
+    const buffer = Buffer.concat(rows.map((el) => Buffer.from(el.tile)));
+
+    if (!nocache) {
+      await mkdir(path.dirname(file), { recursive: true });
+      await writeFile(file, buffer, 'binary');
+    }
+
+    return reply.headers(headers).send(buffer);
+  } catch (err) {
+    log.error('bi/clusterVtile', { error: err.toString(), query, params });
+    return { error: err.toString(), status: 500 };
+  }
+}

package/server/routes/map/controllers/geojson.js

@@ -1,101 +1,127 @@
-import Sphericalmercator from '@mapbox/sphericalmercator';
-
-import path from 'path';
-import { createHash } from 'crypto';
-import { writeFile, mkdir, readFile, stat } from 'fs/promises';
-import { existsSync, /*readdirSync, */ readFileSync } from 'fs';
-
-import yaml from '../../dashboard/controllers/utils/yaml.js';
-import normalizeData from '../../data/controllers/util/normalizeData.js';
-
-import { getWidget } from '../../../../utils.js';
-
-const types = { point: 'ST_Point' /* ,ST_MultiPoint */, polygon: 'ST_Polygon,ST_MultiPolygon' };
-const hourMs = 3.6e+6;
-
-export default async function geojson(req, reply) {
-
-  const {
-    pg, query = {}, funcs = {}, log,
-  } = req;
-
-  const { filter, widget, sql, type, nocache, id, dashboard, geom = 'geom', pointZoom = 0 } = query;
-
-  if (!widget && !dashboard) {
-    return { message: 'not enough params: widget', status: 400 };
-  }
-
-  const data = await getWidget({ dashboard, widget });
-  if (data.status) return data;
-
-  const hash = [pointZoom, filter].filter((el) => el).join();
-
-  const root = funcs.getFolder(req);
-  const file = path.join(root, `/map/geojson/${widget}/${hash ? `${createHash('sha1').update(hash).digest('base64')}/` : ''}.geojson`);
-
-  if (existsSync(file)) {
-    const timeNow = Date.now();
-    const stats = await stat(file);
-    const birthTime = new Date(stats.birthtime).getTime();
-    if (!(birthTime - timeNow > (hourMs * 24)) && !nocache) {
-      const res = JSON.parse(await readFile(file, 'utf-8') || {});
-      return res;
-    }
-  }
-
-  try {
-    const pkey = pg.pk?.[data?.table];
-    if (!pkey) {
-      return { message: `invalid ${widget ? 'widget' : 'dashboard'}: table pk not found (${data?.table})`, status: 400 };
-    }
-
-    // data param 
-    const { table, where = '1=1', xName, x } = normalizeData(data, query);
-
-    if (!xName && !x) {
-      return { message: `invalid ${widget ? 'widget' : 'dashboard'}: x axis column not specified`, status: 400 };
-    }
-
-    // get sql
-    const filterQ = filter ? await funcs.getFilterSQL({ pg, table, filter, query }) : undefined;
-    const q = `select "${pkey}", "${xName || x}", /* st_asgeojson(geom)::json as */ ${geom} as geom from ${filterQ ? `(${filterQ})` : table} q where ${where}`;
-
-    if (sql === '1') return q;
-
-    const { st_geometrytype: geomType = 'point' } = await pg.query(`select st_geometrytype(${geom}), count(*) from ${table} 
-    where ${where} group by st_geometrytype(${geom})`).then((res) => res.rows?.[0] || {});
-
-    const q1 = `SELECT 'FeatureCollection' As type, json_agg(f) As features FROM ( 
-      SELECT 'Feature' As type, row_number() over() as id,
-      st_asgeojson(st_force2d(${query.srid
-      ? `st_transform(${type === 'centroid' ? `st_centroid(${geom})` : geom},${query.srid})`
-      : `${type === 'centroid' || query.point || query.centroid ? `st_centroid(${geom})` : geom}`}), 6, 0)::json as geometry,    
-      (select row_to_json(tc) from (select ${'' ? `${''} as status, ` : ''} 
-      ${xName ? `${xName},` : ''} 
-      ${data.style?.colorAttr ? `${data.style.colorAttr},` : ''}
-      ${pkey} as id,(select file_path from crm.files 
-        where entity_id=q.${pkey}::text and file_status <>'3' and ext in ('png','jpg') limit 1) as image
-      )tc) as properties
-      from (${q})q where ${id && pkey ? ` ${pkey} = '${id}' and ` : ''} ${geom} is not null
-      ${data.query ? ` and ${data.query}` : ''}
-      ${query.extent ? `and ${geom} && 'BOX(${query.extent.split(',').reduce((p, el, i) => p + el + (i % 2 ? ',' : ' '), '')})'::box2d` : ''}
-      ${types[type] ? ` and ST_GeometryType(${geom}) = any ('{ ${types[type]} }') ` : ''}
-      limit ${geomType?.toLowerCase()?.includes('point') ? '15000' : '2500'})f`;
-
-    if (sql === '2') return q1;
-    
-    // auto Index
-    funcs.autoIndex({ table, columns: [xName] });
-
-    const res = await pg.query(q1).then((res) => res.rows?.[0] || {});
-
-    await mkdir(path.dirname(file), { recursive: true });
-    await writeFile(file, JSON.stringify(res));
-
-    return res;
-  }
-  catch (err) {
-    log.error('bi/geojson', { error: err.toString(), query });
-    return { error: err.toString(), status: 500 };
-  }
-}
+import path from 'path';
+import { createHash } from 'crypto';
+import { writeFile, mkdir, readFile, stat } from 'fs/promises';
+import { existsSync, /* readdirSync, */ readFileSync } from 'fs';
+
+
+import { getFolder, getFilterSQL, autoIndex } from '@opengis/fastify-table/utils.js';
+
+import normalizeData from '../../data/controllers/util/normalizeData.js';
+
+import { getWidget } from '../../../../utils.js';
+
+const types = {
+  point: 'ST_Point' /* ,ST_MultiPoint */,
+  polygon: 'ST_Polygon,ST_MultiPolygon',
+};
+const hourMs = 3.6e6;
+
+export default async function geojson(req, reply) {
+  const { pg, query = {}, funcs = {}, log } = req;
+
+  const {
+    filter,
+    widget,
+    sql,
+    type,
+    nocache,
+    id,
+    dashboard,
+    geom = 'geom',
+    pointZoom = 0,
+  } = query;
+
+  if (!widget && !dashboard) {
+    return { message: 'not enough params: widget', status: 400 };
+  }
+
+  const data = await getWidget({ dashboard, widget });
+  if (data.status) return data;
+
+  const hash = [pointZoom, filter].filter((el) => el).join();
+
+  const root = getFolder(req);
+  const file = path.join(
+    root,
+    `/map/geojson/${widget}/${hash ? `${createHash('sha1').update(hash).digest('base64')}/` : ''}.geojson`
+  );
+
+  if (existsSync(file)) {
+    const timeNow = Date.now();
+    const stats = await stat(file);
+    const birthTime = new Date(stats.birthtime).getTime();
+    if (!(birthTime - timeNow > hourMs * 24) && !nocache) {
+      const res = JSON.parse((await readFile(file, 'utf-8')) || {});
+      return res;
+    }
+  }
+
+  try {
+    const pkey = pg.pk?.[data?.table];
+    if (!pkey) {
+      return {
+        message: `invalid ${widget ? 'widget' : 'dashboard'}: table pk not found (${data?.table})`,
+        status: 400,
+      };
+    }
+
+    // data param
+    const { table, where = '1=1', xName, x } = normalizeData(data, query);
+
+    if (!xName && !x) {
+      return {
+        message: `invalid ${widget ? 'widget' : 'dashboard'}: x axis column not specified`,
+        status: 400,
+      };
+    }
+
+    // get sql
+    const filterQ = filter
+      ? await getFilterSQL({ pg, table, filter, query })
+      : undefined;
+    const q = `select "${pkey}", "${xName || x}", /* st_asgeojson(geom)::json as */ ${geom} as geom from ${filterQ ? `(${filterQ})` : table} q where ${where}`;
+
+    if (sql === '1') return q;
+
+    const { st_geometrytype: geomType = 'point' } = await pg
+      .query(
+        `select st_geometrytype(${geom}), count(*) from ${table} 
+    where ${where} group by st_geometrytype(${geom})`
+      )
+      .then((res) => res.rows?.[0] || {});
+
+    const q1 = `SELECT 'FeatureCollection' As type, json_agg(f) As features FROM ( 
+      SELECT 'Feature' As type, row_number() over() as id,
+      st_asgeojson(st_force2d(${
+        query.srid
+          ? `st_transform(${type === 'centroid' ? `st_centroid(${geom})` : geom},${query.srid})`
+          : `${type === 'centroid' || query.point || query.centroid ? `st_centroid(${geom})` : geom}`
+      }), 6, 0)::json as geometry,    
+      (select row_to_json(tc) from (select ${'' ? `${''} as status, ` : ''} 
+      ${xName ? `${xName},` : ''} 
+      ${data.style?.colorAttr ? `${data.style.colorAttr},` : ''}
+      ${pkey} as id,(select file_path from crm.files 
+        where entity_id=q.${pkey}::text and file_status <>'3' and ext in ('png','jpg') limit 1) as image
+      )tc) as properties
+      from (${q})q where ${id && pkey ? ` ${pkey} = '${id}' and ` : ''} ${geom} is not null
+      ${data.query ? ` and ${data.query}` : ''}
+      ${query.extent ? `and ${geom} && 'BOX(${query.extent.split(',').reduce((p, el, i) => p + el + (i % 2 ? ',' : ' '), '')})'::box2d` : ''}
+      ${types[type] ? ` and ST_GeometryType(${geom}) = any ('{ ${types[type]} }') ` : ''}
+      limit ${geomType?.toLowerCase()?.includes('point') ? '15000' : '2500'})f`;
+
+    if (sql === '2') return q1;
+
+    // auto Index
+    autoIndex({ table, columns: [xName] });
+
+    const res = await pg.query(q1).then((res) => res.rows?.[0] || {});
+
+    await mkdir(path.dirname(file), { recursive: true });
+    await writeFile(file, JSON.stringify(res));
+
+    return res;
+  } catch (err) {
+    log.error('bi/geojson', { error: err.toString(), query });
+    return { error: err.toString(), status: 500 };
+  }
+}