@opengis/gis 0.2.50 → 0.2.52

package/module/gis/form/gis.rasters.form.json

@@ -20,10 +20,7 @@
     "raster_key": {
       "type": "Text",
       "ua": "Унікальний ключ",
-      "col": 4,
-      "validators": [
-        "required"
-      ]
+      "col": 4
     },
     "group_id": {
       "type": "Autocomplete",
@@ -57,7 +54,8 @@
       "ua": "Просторова референтна система",
       "type": "Autocomplete",
       "data": "gis.srid_from_setting",
-      "col": 6,
+      "info": "EPSG",
+      "col": 12,
       "validators": [
         "required"
       ]
@@ -65,39 +63,14 @@
     "proj4": {
       "type": "Text",
       "ua": "proj4",
-      "col": 6
-    },
-    "source_type": {
-      "type": "Text",
-      "ua": "Тип джерела, де зберігається растр",
       "col": 12
     },
     "raster_zoom": {
       "type": "Number",
       "min": 0,
-      "ua": "Зум для відображення оригіналу",
+      "ua": "previewZoom",
       "placeholder": "15 за замовченням",
-      "info": "15 за замовченням",
-      "col": 6
-    },
-    "resolution_x": {
-      "type": "Number",
-      "ua": "Координати x центру",
-      "col": 6
-    },
-    "resolution_y": {
-      "type": "Number",
-      "ua": "Координати y центру",
-      "col": 6
-    },
-    "format": {
-      "type": "Text",
-      "ua": "Формат (розширення) растру",
-      "col": 12
-    },
-    "raster_date": {
-      "type": "DatePicker",
-      "ua": "Дата створення растру",
+      "info": "Зум для відображення оригіналу",
       "col": 12
     }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengis/gis",
-  "version": "0.2.50",
+  "version": "0.2.52",
   "type": "module",
   "author": "Softpro",
   "main": "./dist/index.js",
@@ -17,7 +17,7 @@
   "scripts": {
     "dump": "bun ./node_modules/@opengis/fastify-table/dist/script/dump.js",
     "migrate": "MIGRATE=true bun ./node_modules/@opengis/fastify-table/dist/script/migrate.js",
-    "start1": "bun server",
+    "start": "bun server",
     "start:kyiv-demo": "bun --env-file=.env.kyiv-demo server",
     "start:kamyanske": "bun --env-file=.env.kamyanske server",
     "start:kr": "bun --env-file=.env.kr server",

package/server/migrations/cartocss.sql

@@ -0,0 +1,20 @@
+create schema if not exists gis;
+
+create table if not exists gis.cartocss(
+    cartocss_id  text      default next_id() not null primary key,
+    cartocss_key text                              not null constraint cartocss_key_unique unique,
+    name         text                              not null,
+    description  text,
+    config       jsonb,
+    style        text,
+    created_at   timestamp default now(),
+    created_by   text,
+    updated_by   text,
+    updated_at   timestamp,
+    group_id     text constraint services_group_id_fkey references group_list,
+    is_public    boolean   default false           not null,
+    enabled      boolean   default true            not null,
+    geom public.geometry
+);
+
+alter table gis.cartocss add column if not exists geom public.geometry;

package/server/plugins/mapnik/funcs/mapnik.js

@@ -61,6 +61,24 @@ mapClient.waitForReady(Date.now() + 5000, (err) => {
   'GetRasterStatus', // for resumable upload support
 ]
 */
+
+function formatObject(obj1) {
+  return Object.keys(obj1 || {}).reduce((acc1, key) => {
+    const { kind } = obj1[key];
+    const value = obj1[key][kind];
+    if (key === 'timestamp') {
+      Object.assign(acc1, { [key]: new Date(value * 1000).toISOString() });
+    }
+    else if (kind === 'structValue') {
+      Object.assign(acc1, { [key]: formatObject(value.fields) });
+    }
+    else {
+      Object.assign(acc1, { [key]: value });
+    }
+    return acc1;
+  }, {});
+}
+
 const grpcMethods = Object.getOwnPropertyNames(
   Object.getPrototypeOf(mapClient),
 )
@@ -89,6 +107,22 @@ const grpcMethods = Object.getOwnPropertyNames(
         });
         stream.on("end", () => res('finish'));
       }
+      else if (curr === 'GetStat') {
+        mapClient[curr].call(
+          mapClient,
+          inputData,
+          (err, data) => {
+            if (err) {
+              return rej(err || 'invalid response');
+            }
+
+            const { fields } = data?.result || {};
+            // google.protobuf.Struct to js object
+            const obj = formatObject(fields);
+            return res(obj);
+          },
+        );
+      }
       else {
         mapClient[curr].call(
           mapClient,
@@ -97,7 +131,7 @@ const grpcMethods = Object.getOwnPropertyNames(
             if (err) {
               return rej(err);
             }
-            return res(data?.fields || data);
+            return res(data);
           },
         );
       }

package/server/plugins/mapnik/map.proto

@@ -18,6 +18,7 @@ service MapService {
     rpc GetStat(GetStatRequest) returns (GetStatOut);
     rpc GetRender(GetRenderRequest) returns (GetRenderOut);
     rpc GetLogs(GetLogsRequest) returns (GetLogsOut);
+    rpc GetRasterInfo(GetRasterInfoRequest) returns (GetRasterInfoOut);
 }
 
 // ----------------------------
@@ -114,6 +115,7 @@ message GetStatusOut {
 
 message GetStatRequest {
     string period = 1;  // "5m", "1h", "1d"
+    string ttl = 2; // "0"
 }
 
 message GetStatOut {
@@ -138,6 +140,37 @@ message GetLogsOut {
     repeated string result = 1;
 }
 
+message GetRasterInfoRequest {
+    string path = 1;
+    string ttl = 2;
+}
+
+message FileItem {
+  string name = 1;
+  string size = 2;
+}
+
+message GetRasterInfoData {
+    string total_size = 1;
+    string extension = 2;
+    int64 srid = 3;
+    string proj4 = 4;
+    repeated float extent = 5;
+    repeated FileItem files = 6;
+    int64 width = 7;
+    int64 height = 8;
+    int64 bands_count = 9;
+    string resolution = 10;
+    repeated string bands = 11;
+    int64 color_depth = 12;
+    string compression = 13;
+}
+
+message GetRasterInfoOut {
+    GetRasterInfoData data = 1;
+    bool cache = 2;
+}
+
 // ----------------------------
 // Job Service Messages
 // ----------------------------

package/server/routes/gis/cartocss/add.cartocss.js

@@ -1,129 +1,6 @@
-import carto from 'carto';
-
 import { dataUpdate } from '@opengis/fastify-table/utils.js';
-import mapnik from '../../../plugins/mapnik/funcs/mapnik.js';
-
-const { UploadXML } = mapnik();
-
-const srsList = {};
-
-const getGeomColumn = (geoms, customColumn) => {
-  if (customColumn && geoms.includes(customColumn)) {
-    return customColumn;
-  }
-  if (geoms.includes('geom')) {
-    return 'geom';
-  }
-  return geoms[0];
-};
-
-async function cartoBounds({ id, dataset }, pg) {
-  const sqlBounds = dataset.map(el => ({ tbl: el.table.split('.'), geom: el.gcol || 'geom' })).map(({ tbl, geom }) => `select ST_EstimatedExtent('${tbl[0]}','${tbl[1]}', '${geom}')`);
-
-  const geom = await pg.query(
-    `update gis.style set             
-      geom=(select st_extent(st_estimatedextent) from ( ${sqlBounds.join(' union all ')})q )
-      where style_id=$1 returning geom::box2d`,
-    [id],
-  ).then(e => e.rows?.[0]?.geom);
 
-  const bounds = geom
-    ? geom.replace(/[A-Z\)\(]+/g, '').split(/[ ,]+/).map(el => el - 0)
-    : null;
-
-  if (!bounds) {
-    throw new Error('empty bounds');
-  }
-
-  return bounds;
-}
-
-function cartoData(el, extent, pg) {
-  const minzoom = el.zoom ? (+el.zoom || 5) : 5;
-  const maxzoom = el.maxzoom ? (+el.maxzoom || 22) : 22;
-  const view = `(select ${el.columns || '1'}, ${el.gcol} from ${el.table} where ${el.query || 'true'} ) as data`;
-  const layerBound = extent || [-180, -85, 180, 85];
-
-  return {
-    id: el.caption,
-    name: el.caption,
-    properties: { minzoom, maxzoom },
-    srs: el.proj4text || '+proj=longlat +datum=WGS84 +no_defs',
-    Datasource: {
-      simplify_geometries: true,
-      type: 'postgis',
-      extent: layerBound.join(','),
-      table: view,
-      geometry_field: el.gcol,
-      host: pg.options?.host || 'localhost',
-      dbname: pg.options?.database || 'postgres',
-      port: pg.options?.port || 5432,
-      user: pg.options?.user || 'postgres',
-      password: pg.options?.password || 'postgres',
-    },
-  };
-}
-
-async function uploadXML({ id, dataset, style }, pg) {
-  if (!id || !dataset?.length) {
-    return null;
-  }
-  await Promise.all(dataset.filter((el) => el.active !== false).map(async el => {
-    const { table } = el;
-
-    const geoms = await pg.query(`SELECT json_agg(attname) as geoms FROM pg_attribute
-      where attrelid = to_regclass($1::text)
-      and attnum > 0
-      AND NOT attisdropped
-      and atttypid::regtype::text ='geometry'`, [table]).then(e => e.rows?.[0]?.geoms || []);
-
-    const gcol = getGeomColumn(geoms, el.gcol);
-    Object.assign(el, { gcol });
-
-    const srid = el.srid ? el.srid
-      : await pg.query(`select st_srid(${gcol}) as srid from ${table} where ${gcol} is not null limit 1`).then(e => e.rows[0]?.srid);
-
-    if (srid && !srsList[srid] && (srid - 0 > 0)) {
-      srsList[srid] = await pg.query(`SELECT proj4text from spatial_ref_sys where srid=$1`, [srid])
-        .then(e => e.rows[0]?.proj4text);
-    }
-    Object.assign(el, { gcol, proj4text: srsList[srid] });
-  }));
-
-  const bounds = await cartoBounds({ id, dataset }, pg);
-
-  const arrayLayers = dataset.filter((el) => el.active !== false).map((el) => cartoData(el, bounds, pg));
-
-  const configJson = {
-    format: 'png',
-    Layer: arrayLayers.reverse(),
-    srs: '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over',
-    bounds,
-    'maximum-extent': '-20037508.34,-20037508.34,20037508.34,20037508.34',
-    center: [0, 0, 2],
-    minzoom: 1,
-    maxzoom: 25,
-    Stylesheet: [{
-      id: 'carto_css_style',
-      data: style || '',
-    }],
-  };
-
-  // generate
-  const res = new carto.Renderer().render(configJson);
-  const xmlFileText = typeof res === 'string' ? res : res?.data;
-  if (!xmlFileText) {
-    throw new Error('xml generation error');
-  }
-  const { status } = await UploadXML({
-    name: id,
-    xml: xmlFileText,
-  });
-  if (status !== 'success') {
-    throw new Error('xml upload error');
-  }
-  return xmlFileText;
-}
+import uploadXML from '../../mapnik/functions/uploadXML.js';
 
 export default async function addCartocss(req, reply) {
   const { pg, body, params } = req;

package/server/routes/map/controllers/mapFormat.js

@@ -28,7 +28,7 @@ export default async function mapFormat(req, reply) {
   const filteredColumns = columnNames.filter(name => !excluded.includes(name));
 
   const rows = await pg.query(
-    `SELECT ${pk} as "id", geom::json ${filteredColumns?.length > 0 ? ',' : ''}  ${filteredColumns.map((n) => `"${n}"`).join(', ')} FROM ${tname} WHERE ${pk} = $1`,
+    `SELECT ${pk} as "id", geom::json, geom::box2d as box2d ${filteredColumns?.length > 0 ? ',' : ''}  ${filteredColumns.map((n) => `"${n}"`).join(', ')} FROM ${tname} WHERE ${pk} = $1`,
     [id],
   ).then(el => el.rows || []);
 
@@ -38,6 +38,8 @@ export default async function mapFormat(req, reply) {
     .filter(col => col.data && ['select', 'badge', 'tags'].includes(col.format))
     .reduce((acc, curr) => ({ ...acc, [curr.name]: curr.data }), {});
 
+  rows.forEach(row => Object.assign(row, { box2d: undefined, bounds: row.box2d ? row.box2d.replace(/[A-Z\)\(]+/g, '').split(/[ ,]+/).map(el => el - 0) : null }));
+
   await metaFormat({
     rows, table: tname, cls: classifiers, sufix: false,
   });

package/server/routes/mapnik/controllers/checkCarto.js

@@ -0,0 +1,40 @@
+import { config, pgClients } from '@opengis/fastify-table/utils.js';
+
+import mapnik from '../../../plugins/mapnik/funcs/mapnik.js';
+import cartoBounds from '../functions/cartoBounds.js';
+
+const { RenderTile } = mapnik();
+
+const { prefix = '/api' } = config;
+
+export default async function checkCarto({
+    pg = pgClients.client, params,
+}, reply) {
+    if (!config.mapServerAddress) {
+        return reply.status(400).send({ error: 'mapnik server address needed', code: 400 });
+    }
+
+    const cartocss = await pg.query('select config, geom::box2d from gis.cartocss where cartocss_id=$1', [params.id]).then(el => el.rows?.[0]);
+
+    if (!cartocss) {
+        return reply.status(404).send({ error: `cartocss not found: ${params.id}`, code: 404 });
+    }
+
+    const rtile = await RenderTile({
+        name: params.id,
+        width: 256,
+        height: 256,
+        bbox: [3713463.7081504324, 6088362.176970857, 3713616.5822070027, 6088515.051027427],
+        ttl: '0',
+        // debug: query.debug,
+    }).catch(err => ({ err: err.toString() }));
+
+    const bounds = cartocss.geom || await cartoBounds({ id: params.id, dataset: cartocss.config }, pg);
+
+    return {
+        url: `${prefix}/gis-rtile/${params.id}/{z}/{x}/{y}.png`,
+        bounds,
+        render: !!rtile.base64,
+        renderError: !!rtile.err,
+    };
+}

package/server/routes/mapnik/controllers/mapnikLogger.js

@@ -0,0 +1,23 @@
+import { config } from '@opengis/fastify-table/utils.js';
+
+import mapnik from '../../../plugins/mapnik/funcs/mapnik.js';
+
+const { GetLogs } = mapnik();
+
+export default async function mapnikLogger({
+    params, query,
+}, reply) {
+    if (!config.mapServerAddress) {
+        return reply.status(400).send({ error: 'mapnik server address needed', code: 400 });
+    }
+
+    if (!['render', 'error', 'info', 'critical'].includes(params.type)) {
+        return reply.status(400).send({ error: 'invalid params: type', code: 400 });
+    }
+
+    const { result } = await GetLogs({
+        period: query.period,
+        type: params.type,
+    });
+    return reply.status(200).send(result.join('\n'));
+}

package/server/routes/mapnik/controllers/mapnikStat.js

@@ -0,0 +1,20 @@
+import { config } from '@opengis/fastify-table/utils.js';
+
+import mapnik from '../../../plugins/mapnik/funcs/mapnik.js';
+
+const { GetStat } = mapnik();
+
+export default async function mapnikStat({
+    params, query,
+}, reply) {
+    if (!config.mapServerAddress) {
+        return reply.status(400).send({ error: 'mapnik server address needed', code: 400 });
+    }
+
+    const stat = await GetStat({
+        period: params.period,
+        ttl: query.nocache ? '0' : undefined,
+    });
+
+    return reply.status(200).send(stat);
+}

package/server/routes/mapnik/controllers/rasterInfo.js

@@ -0,0 +1,36 @@
+import { config, pgClients } from '@opengis/fastify-table/utils.js';
+
+import mapnik from '../../../plugins/mapnik/funcs/mapnik.js';
+
+const { GetRasterInfo } = mapnik();
+
+const { prefix = '/api' } = config;
+
+export default async function rtile({
+    pg = pgClients.client, params, query,
+}, reply) {
+    if (!config.mapServerAddress) {
+        return reply.status(400).send({ error: 'mapnik server address needed', code: 400 });
+    }
+
+    const raster = pg.pk?.['gis.rasters']
+        ? await pg.query('select raster_id, source_path from gis.rasters where raster_id=$1::text', [params.id])
+            .then(el => el.rows?.[0] || {})
+        : {};
+
+    if (raster.raster_id && !raster.source_path) {
+        return reply.status(400).send({ error: 'empty source_path', code: 400 });
+    }
+
+    if (!raster.raster_id) {
+        const decodedPath = Buffer.from(params.id, 'base64url').toString('utf-8');
+        Object.assign(raster, { source_path: decodedPath });
+    }
+
+    const data = await GetRasterInfo({
+        path: raster.source_path,
+        ttl: query.nocache ? '0' : '1h',
+    });
+
+    return { ...data, url: `${prefix}/gis-rtile/${Buffer.from(raster.source_path).toString('base64url')}/{z}/{x}/{y}.png` };
+}

package/server/routes/mapnik/controllers/rtile.js

@@ -1,11 +1,10 @@
-import path from 'node:path';
 import Sphericalmercator from '@mapbox/sphericalmercator';
 
 import { config, logger, pgClients } from '@opengis/fastify-table/utils.js';
 
 import mapnik from '../../../plugins/mapnik/funcs/mapnik.js';
 
-const { RenderTile, GetRasterStatus } = mapnik();
+const { RenderTile } = mapnik();
 
 const mercator = new Sphericalmercator({ size: 256 });
 
@@ -27,7 +26,7 @@ const mercator = new Sphericalmercator({ size: 256 });
 export default async function rtile({
     pg = pgClients.client, params, query,
 }, reply) {
-    if (!RenderTile || !GetRasterStatus) {
+    if (!RenderTile) {
         return reply.status(400).send({ error: 'mapnik server address needed', code: 400 });
     }
 
@@ -57,19 +56,10 @@ export default async function rtile({
 
     if (!raster.raster_id && !cartoExists) {
         const decodedPath = Buffer.from(id, 'base64url').toString('utf-8');
-        const uploadStatus = path.extname(decodedPath) ? await GetRasterStatus({ path: decodedPath }) : {};
-
-        if (uploadStatus.exists === false) {
-            return reply.status(400).send({ error: 'raster not uploaded', code: 400 });
-        }
-
-        if (uploadStatus.finished === false) {
-            return reply.status(400).send({ error: 'upload not finished', code: 400 });
-        }
         Object.assign(raster, { source_path: decodedPath });
     }
 
-    if (!raster && !cartoExists) {
+    if (!raster.source_path && !cartoExists) {
         return reply.status(404).send({ error: 'raster / cartocss not found', code: 404 });
     }
 
@@ -97,7 +87,7 @@ export default async function rtile({
 
         const buffer = Buffer.from(data.base64, 'base64');
 
-        return reply.headers({ 'Content-Type': 'image/png' }).send(buffer);
+        return reply.headers({ 'Content-Type': 'image/png', 'Cache-Control': query.nocache ? 'no-cache' : '30d' }).send(buffer);
     }
     catch (err) {
         logger.file('rtile/error', { error: err.toString(), stack: err.stack });

package/server/routes/mapnik/functions/cartoBounds.js

@@ -0,0 +1,22 @@
+import { pgClients } from "@opengis/fastify-table/utils.js";
+
+export default async function cartoBounds({ id, dataset }, pg = pgClients.client) {
+    const sqlBounds = dataset.map(el => ({ tbl: el.table.split('.'), geom: el.gcol || 'geom' })).map(({ tbl, geom }) => `select ST_EstimatedExtent('${tbl[0]}','${tbl[1]}', '${geom}')`);
+
+    const geom = await pg.query(
+        `update gis.cartocss set             
+        geom=(select st_extent(st_estimatedextent) from ( ${sqlBounds.join(' union all ')})q )
+        where cartocss_id=$1 returning geom::box2d`,
+        [id],
+    ).then(e => e.rows?.[0]?.geom);
+
+    const bounds = geom
+        ? geom.replace(/[A-Z\)\(]+/g, '').split(/[ ,]+/).map(el => el - 0)
+        : null;
+
+    if (!bounds) {
+        // throw new Error('empty bounds');
+    }
+
+    return bounds || [-180, -85.05112877980659, 180, 85.05112877980659];
+}

package/server/routes/mapnik/functions/uploadXML.js

@@ -0,0 +1,106 @@
+import carto from 'carto';
+
+import mapnik from '../../../plugins/mapnik/funcs/mapnik.js';
+
+const { UploadXML } = mapnik();
+
+const srsList = {};
+
+const getGeomColumn = (geoms, customColumn) => {
+    if (customColumn && geoms.includes(customColumn)) {
+        return customColumn;
+    }
+    if (geoms.includes('geom')) {
+        return 'geom';
+    }
+    return geoms[0];
+};
+
+import cartoBounds from './cartoBounds.js';
+
+function cartoData(el, extent, pg) {
+    const minzoom = el.zoom ? (+el.zoom || 5) : 5;
+    const maxzoom = el.maxzoom ? (+el.maxzoom || 22) : 22;
+    const view = `(select ${el.columns || '1'}, ${el.gcol} from ${el.table} where ${el.query || 'true'} ) as data`;
+    const layerBound = extent || [-180, -85, 180, 85];
+
+    return {
+        id: el.caption,
+        name: el.caption,
+        properties: { minzoom, maxzoom },
+        srs: el.proj4text || '+proj=longlat +datum=WGS84 +no_defs',
+        Datasource: {
+            simplify_geometries: true,
+            type: 'postgis',
+            extent: layerBound.join(','),
+            table: view,
+            geometry_field: el.gcol,
+            host: pg.options?.host || 'localhost',
+            dbname: pg.options?.database || 'postgres',
+            port: pg.options?.port || 5432,
+            user: pg.options?.user || 'postgres',
+            password: pg.options?.password || 'postgres',
+        },
+    };
+}
+
+export default async function uploadXML({ id, dataset, style }, pg) {
+    if (!id || !dataset?.length || Array.isArray(dataset) === false) {
+        return null;
+    }
+    await Promise.all(dataset.filter((el) => el.active !== false).map(async el => {
+        const { table } = el;
+
+        const geoms = await pg.query(`SELECT json_agg(attname) as geoms FROM pg_attribute
+      where attrelid = to_regclass($1::text)
+      and attnum > 0
+      AND NOT attisdropped
+      and atttypid::regtype::text ='geometry'`, [table]).then(e => e.rows?.[0]?.geoms || []);
+
+        const gcol = getGeomColumn(geoms, el.gcol);
+        Object.assign(el, { gcol });
+
+        const srid = el.srid ? el.srid
+            : await pg.query(`select st_srid(${gcol}) as srid from ${table} where ${gcol} is not null limit 1`).then(e => e.rows[0]?.srid);
+
+        if (srid && !srsList[srid] && (srid - 0 > 0)) {
+            srsList[srid] = await pg.query(`SELECT proj4text from spatial_ref_sys where srid=$1`, [srid])
+                .then(e => e.rows[0]?.proj4text);
+        }
+        Object.assign(el, { gcol, proj4text: srsList[srid] });
+    }));
+
+    const bounds = await cartoBounds({ id, dataset }, pg);
+
+    const arrayLayers = dataset.filter((el) => el.active !== false).map((el) => cartoData(el, bounds, pg));
+
+    const configJson = {
+        format: 'png',
+        Layer: arrayLayers.reverse(),
+        srs: '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over',
+        bounds,
+        'maximum-extent': '-20037508.34,-20037508.34,20037508.34,20037508.34',
+        center: [0, 0, 2],
+        minzoom: 1,
+        maxzoom: 25,
+        Stylesheet: [{
+            id: 'carto_css_style',
+            data: style || '',
+        }],
+    };
+
+    // generate
+    const res = new carto.Renderer().render(configJson);
+    const xmlFileText = typeof res === 'string' ? res : res?.data;
+    if (!xmlFileText) {
+        throw new Error('xml generation error');
+    }
+    const { status } = await UploadXML({
+        name: id,
+        xml: xmlFileText,
+    });
+    if (status !== 'success') {
+        throw new Error('xml upload error');
+    }
+    return xmlFileText;
+}