@opengis/gis 0.2.8 → 0.2.10

package/module/test/cls/doc_status.json

@@ -1,32 +1,32 @@
-[
-    {
-        "id": "1",
-        "text": "Діючий",
-        "color": "#1ab394"
-    },
-    {
-        "id": "2",
-        "text": "Зупинений",
-        "color": "#ed82c8"
-    },
-    {
-        "id": "3",
-        "text": "Скасований",
-        "color": "#4a4a4a"
-    },
-    {
-        "id": "5",
-        "text": "Проектний",
-        "color": "#6495ed"
-    },
-    {
-        "id": "7",
-        "text": "Очікує розгляду",
-        "color": "#92b8ef"
-    },
-    {
-        "id": "10",
-        "text": "Архівний",
-        "color": "#d48428"
-    }
+[
+    {
+        "id": "1",
+        "text": "Діючий",
+        "color": "#1ab394"
+    },
+    {
+        "id": "2",
+        "text": "Зупинений",
+        "color": "#ed82c8"
+    },
+    {
+        "id": "3",
+        "text": "Скасований",
+        "color": "#4a4a4a"
+    },
+    {
+        "id": "5",
+        "text": "Проектний",
+        "color": "#6495ed"
+    },
+    {
+        "id": "7",
+        "text": "Очікує розгляду",
+        "color": "#92b8ef"
+    },
+    {
+        "id": "10",
+        "text": "Архівний",
+        "color": "#d48428"
+    }
 ]

package/module/test/select/core.user_uid.sql

@@ -1 +1 @@
-select uid, coalesce(coalesce(sur_name,'')||coalesce(' '||user_name,'') ||coalesce(' '||father_name,''),login) as text from admin.users
+select uid, coalesce(coalesce(sur_name,'')||coalesce(' '||user_name,'') ||coalesce(' '||father_name,''),login) as text from admin.users

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengis/gis",
-  "version": "0.2.8",
+  "version": "0.2.10",
   "type": "module",
   "author": "Softpro",
   "main": "./dist/index.js",

package/plugin.js

@@ -26,6 +26,7 @@ const columnType = {
 async function plugin(app, opts = config) {
   // API
   app.register(import('./server/routes/map/index.mjs'), opts);
+  app.register(import('./server/plugins/crons.js'), opts);
   app.register(import('./server/routes/gis/index.mjs'), opts);
 
   addHook('afterData', async ({

package/server/plugins/crons.js

@@ -0,0 +1,21 @@
+import path from 'node:path';
+import { existsSync } from 'node:fs';
+import { rm } from 'node:fs/promises';
+
+import { addCron } from '@opengis/fastify-table/utils.js';
+import rootFolder from './mapnik/funcs/rootFolder.mjs';
+
+async function clearOldVectorTiles() {
+    if (existsSync(path.join(rootFolder, 'vtile'))) {
+        await rm(path.join(rootFolder, 'vtile'), { recursive: true, force: true });
+        console.log('clear tiles...', rootFolder, 'vtile');
+        return 'vector tiles cleared';
+    }
+    return 'nothing to clear';
+}
+
+async function plugin() {
+    addCron(clearOldVectorTiles, 60 * 60 * 24);
+}
+
+export default plugin;

package/server/routes/map/controllers/vtile.js

@@ -1,11 +1,15 @@
 import path from 'node:path';
 import { existsSync } from 'node:fs';
 import { createHash } from 'node:crypto';
-import { readFile, stat, mkdir, writeFile } from 'node:fs/promises';
+import {
+    readFile, stat, mkdir, writeFile,
+} from 'node:fs/promises';
 
 import Sphericalmercator from '@mapbox/sphericalmercator';
 
-import { getTemplate, pgClients, getFilterSQL, getMeta, yml2json, getColumnCLS, getSelect } from '@opengis/fastify-table/utils.js';
+import {
+    getTemplate, pgClients, getFilterSQL, getMeta, yml2json, getColumnCLS, getSelect,
+} from '@opengis/fastify-table/utils.js';
 
 import rootFolder from '../../../plugins/mapnik/funcs/rootFolder.mjs';
 
@@ -14,9 +18,13 @@ const mercator = new Sphericalmercator({ size: 256 });
 const types = { point: 'ST_Point', polygon: 'ST_Polygon,ST_MultiPolygon' };
 const systemColumns = ['uid', 'cdate', 'editor_id', 'editor_date', 'created_by', 'created_at', 'updated_by', 'updated_at'];
 
-export default async function vtile({ pg = pgClients.client, params = {}, query = {}, unittest }, reply) {
+export default async function vtile({
+    pg = pgClients.client, params = {}, query = {}, unittest,
+}, reply) {
     const { id, y, z } = params;
-    const { sql, nocache, filter, pointZoom, type, layers } = query;
+    const {
+        sql, nocache, filter, pointZoom, type, layers,
+    } = query;
     const x = params.x?.split?.('.')?.[0] - 0;
 
     if (!id) {
@@ -39,7 +47,7 @@ export default async function vtile({ pg = pgClients.client, params = {}, query
 
     const filepath = path.join(
         rootFolder,
-        `/map/vtile/${id}/${hash ? `${createHash('md5').update(hash).digest('hex')}/` : ''}${z}/${x}/${y}.vmt`
+        `/vtile/${id}/${hash ? `${createHash('md5').update(hash).digest('hex')}/` : ''}${z}/${x}/${y}.vmt`,
     );
 
     if (existsSync(filepath) && z < 13 && !nocache && !sql && !unittest) {
@@ -57,7 +65,7 @@ export default async function vtile({ pg = pgClients.client, params = {}, query
 
     const data = await pg.query(
         'select source_path as source, style, geom_type as gt, geometry_column as gc from gis.services where service_id=$1 and is_active and is_public',
-        [id]
+        [id],
     ).then(el => el.rows?.[0]);
 
     if (!data) {
@@ -96,26 +104,24 @@ export default async function vtile({ pg = pgClients.client, params = {}, query
     const columnsFiltered = columns
         .filter(el => !['json', 'geometry'].includes(pg.pgType?.[el.dataTypeID]) && !systemColumns.includes(el.name));
 
-    const props = columnsFiltered.map(el => {
-        return el.type?.[0] === '_'
-            ? `${el.name}[1] as ${el.name}`
-            : `${el.name}${['int4'].includes(el.type) ? '::text' : ''}`;
-    });
+    const props = columnsFiltered.map(el => (el.type?.[0] === '_'
+        ? `${el.name}[1] as ${el.name}`
+        : `${el.name}${['int4'].includes(el.type) ? '::text' : ''}`));
 
     const propsColumns = props.length ? await Promise.all(props.map(async colname => {
         if (!cls?.[colname]) return `"${colname}"`;
-        const { type, data } = await pg.queryCache('select type, data from admin.cls where parent is null and name=$1 limit 1', { args: [cls[colname]], table: 'admin.cls' })
+        const { type: type2, data: data2 } = await pg.queryCache('select type, data from admin.cls where parent is null and name=$1 limit 1', { args: [cls[colname]], table: 'admin.cls' })
             .then(el => el.rows?.[0] || {}) || {};
 
-        if (!type || type === 'sql' && !data) {
+        if (!type2 || (type2 === 'sql' && !data2)) {
             return `"${colname}"`;
         }
 
-        if (type === 'json') {
+        if (type2 === 'json') {
             return `(select name from admin.cls where parent='${cls[colname]}' and code="${colname}"::text) as "${colname}"`;
         }
 
-        return `(with c(id,text) as (select * from (${data})q limit 1) select text from c where id::text="${colname}"::text) as "${colname}"`;
+        return `(with c(id,text) as (select * from (${data2})q limit 1) select text from c where id::text="${colname}"::text) as "${colname}"`;
     })) : '';
 
     const q = `SELECT ST_AsMVT(q, '${id.replace(/'/g, "''")}', 4096, 'geom','row') as tile
@@ -163,4 +169,4 @@ export default async function vtile({ pg = pgClients.client, params = {}, query
     }
 
     return reply.status(200).headers(headers).send(buffer);
-}
+}

package/server/routes/map/vtile1.js

@@ -1,14 +1,27 @@
 const headers = {
   'Content-Type': 'application/x-protobuf',
-  'Cache-Control': 'no-cache',
+  'Cache-Control': '86400',
 };
+
+import path from 'node:path';
+import { existsSync } from 'node:fs';
+import { createHash } from 'node:crypto';
+import {
+  readFile, stat, mkdir, writeFile,
+} from 'node:fs/promises';
+
 import Sphericalmercator from '@mapbox/sphericalmercator';
 
 const mercator = new Sphericalmercator({ size: 256 });
+
+// import { SphericalMercator } from '@mapbox/sphericalmercator';
+// const mercator = new SphericalMercator({ size: 256 });
+
 import {
   getColumnCLS, getTemplate, getMeta, getFilterSQL, getSelect,
 } from '@opengis/fastify-table/utils.js';
 import yaml from 'js-yaml';
+import rootFolder from '../../plugins/mapnik/funcs/rootFolder.mjs';
 
 const propsCache = {};
 
@@ -17,6 +30,25 @@ export default async function vtile({
 }, reply) {
   const { y, z } = params;
   const x = params.x.split('.')[0] - 0;
+  const {
+    type, nocache, sql, clusterZoom, filter,
+  } = query;
+
+  const hash = [type, clusterZoom, filter].join();
+
+  const filepath = path.join(
+    rootFolder,
+    `/vtile/${params.layer}/${hash ? `${createHash('md5').update(hash).digest('hex')}/` : ''}${z}/${x}/${y}.vmt`,
+  );
+
+  if (existsSync(filepath) && +z < 13 && !nocache && !sql && process.env.NODE_ENV !== 'test') {
+    const { birthtimeMs = Date.now() } = await stat(filepath);
+    const ageInMs = Date.now() - birthtimeMs;
+    if (ageInMs < 86400000) {
+      const buffer = await readFile(filepath);
+      return reply.status(200).headers(headers).send(buffer);
+    }
+  }
 
   // layer
   const [map, layer] = params.layer.includes(':') ? params.layer.split(':') : [null, params.layer];
@@ -41,7 +73,7 @@ export default async function vtile({
   // meta
   const metaTable = await getMeta({ pg, table });
   if (metaTable.error) { return metaTable; }
-  const { pk: pkey, columns } = metaTable;
+  const { pk: pkey = data.key, columns } = metaTable; // if view - specify via template
 
   // props
 
@@ -54,49 +86,51 @@ export default async function vtile({
     // console.log(select);
     const clsData = select?.data || col.data;
     const cls = clsData ? await getSelect(clsData).then(el => ({ name: clsData, type: el.arr ? 'cls' : 'select', sql: el.sql })) : null;
-    const { name, type, sql } = cls || getColumnCLS(table, col.data || key) || {};
+    const { name, type: type1, sql: sql1 } = cls || getColumnCLS(table, col.data || key) || {};
     // console.log({ name, type, sql });
-    if (name && type === 'cls') {
+    if (name && type1 === 'cls') {
       return `(select name from admin.cls where parent='${name}' and code::text=${key}::text limit 1) as "${key}_text","${key}"::text`;
     }
-    if (type === 'select' && sql) {
-      const { fields = [] } = await pg.query(`${sql} limit 0`);
-      return `(select "${fields[1].name}" from (${sql})s${idx} where "${fields[0].name}"::text="${key}"::text limit 1) as "${key}_text","${key}"::text`;
+    if (type1 === 'select' && sql1) {
+      const { fields = [] } = await pg.query(`${sql1} limit 0`);
+      return `(select "${fields[1].name}" from (${sql1})s${idx} where "${fields[0].name}"::text="${key}"::text limit 1) as "${key}_text","${key}"::text`;
     }
     return `"${key}"::text`;
   }));
   propsCache[cache] = propsCache[cache] || propsWithCls;
 
   const geomCol = (style?.type === 'point' ? `ST_Centroid(${geom})` : null)
-        || (parseInt(z, 10) < parseInt(query.pointZoom || style?.pointZoom || style?.iconZoom || '0', 10) ? `ST_Centroid(${geom})` : geom);
+    || (parseInt(z, 10) < parseInt(query.pointZoom || style?.pointZoom || style?.iconZoom || '0', 10) ? `ST_Centroid(${geom})` : geom);
 
   const koef = {
     10: 0.1, 11: 0.05, 12: 0.005, 13: 0.001, 14: 0.001, 15: 0.0005,
   }[z] || 0.0005;
 
-  const { type } = query;
   const types = { point: 'ST_Point' /* ,ST_MultiPoint */, polygon: 'ST_Polygon,ST_MultiPolygon' };
-  const sql = ((query.clusterZoom - z) > 2
-    ? `SELECT ST_AsMVT(q, '${layer}', 4096, 'geom','row') as tile
+  const queryMode = ((clusterZoom - z) > 2 ? 1 : null) || ((clusterZoom - z) > 0 ? 2 : null) || 3;
+  // console.log(z, clusterZoom, queryMode);
+  const props1 = props.filter((el, idx, arr) => idx === arr.findIndex(item => item.name === el.name));
+  const q = {
+    1: `SELECT ST_AsMVT(q, '${layer}', 4096, 'geom','row') as tile
             FROM (
                 SELECT floor(random() * 100000 + 1)::int + row_number() over() as row, point_count, ST_AsMVTGeom(st_transform(geom,3857),ST_TileEnvelope(${z},${y},${x})::box2d,4096,256,false) geom
                 FROM (
                 SELECT st_centroid(st_union(geom)) as geom, count(*) as point_count
                 FROM ${table} where ${layerQuery || 'true'} and ${filterData?.q || 'true'} and ${geom} && ${bbox2d}  ) j            
-                )q` : null
-  ) || ((query.clusterZoom - z) > 0
-    ? `SELECT ST_AsMVT(q, '${layer}', 4096, 'geom','row') as tile
+                )q`,
+    2: `SELECT ST_AsMVT(q, '${layer}', 4096, 'geom','row') as tile
             FROM (
-                SELECT floor(random() * 100000 + 1)::int +row_number() over() as row, count(*) as point_count, ST_AsMVTGeom(st_transform(st_centroid(ST_Union(geom)),3857),ST_TileEnvelope(${z},${y},${x})::box2d,4096,256,false) geom
+                SELECT floor(random() * 100000 + 1)::int +row_number() over() as row, count(*) as point_count,
+                ${props?.length ? `${props1.map(prop => `case when count(*)=1 then min(${prop.name.replace(/'/g, "''")}) else null end as "${prop.name.replace(/'/g, "''")}"`).join(',')},` : ''}
+                ST_AsMVTGeom(st_transform(st_centroid(ST_Union(geom)),3857),ST_TileEnvelope(${z},${y},${x})::box2d,4096,256,false) as geom
                 FROM (
-                SELECT geom, ST_ClusterDBSCAN(geom,${koef},1) OVER () AS cluster 
+                SELECT ${props?.length ? `${props1.map(prop => prop.name).join(',')},` : ''} geom, ST_ClusterDBSCAN(geom,${koef},1) OVER () AS cluster 
                 FROM ${table} where ${layerQuery || 'true'} and ${filterData?.q || 'true'} and ${geom} && ${bbox2d}  ) j
                 WHERE cluster IS NOT NULL
                 GROUP BY cluster
                 ORDER BY 1 DESC
-                )q`
-
-    : `SELECT ST_AsMVT(q, '${layer}', 4096, 'geom','row') as tile
+                )q`,
+    3: `SELECT ST_AsMVT(q, '${layer}', 4096, 'geom','row') as tile
                 FROM (
                     SELECT
 
@@ -120,10 +154,17 @@ export default async function vtile({
                     ${(style?.type || data?.geometry_type) === 'polygon' ? `order by st_area(st_transform(${geom},3857)) desc` : ''}
 
                     limit 3000)q
-                ) q`);
+                ) q`,
+  }[queryMode];
 
-  if (query.sql === '1') { return sql; }
-  const tile = await pg.query(sql);
+  if (query.sql === '1') { return q; }
+  const tile = await pg.query(q);
   const buffer = Buffer.concat(tile.rows.map((el) => Buffer.from(el.tile)));
-  return reply.headers(headers).send(buffer);
+
+  if (!nocache) {
+    await mkdir(path.dirname(filepath), { recursive: true });
+    await writeFile(filepath, buffer, 'binary');
+  }
+
+  return reply.headers({ ...headers, 'Cache-Control': nocache ? 'no-cache' : headers['Cache-Control'] }).send(buffer);
 }

package/utils.js

@@ -1,9 +1,11 @@
 import gisRegistry from './server/routes/gis/registers/gis.registry.js';
 import gisRegistryList from './server/routes/gis/registers/gis.registry.list.js';
 import mapRegistry from './server/routes/gis/registers/map.registry.js';
+import rootFolder from './server/plugins/mapnik/funcs/rootFolder.mjs';
 
 export {
   gisRegistry,
   gisRegistryList,
-  mapRegistry
+  mapRegistry,
+  rootFolder,
 };