@opengis/gis 0.0.17 → 0.0.19

package/server/plugins/mapnik/funcs/rasterInfo.js

@@ -0,0 +1,109 @@
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+import { existsSync } from 'node:fs';
+import { writeFile, mkdir, readFile } from 'node:fs/promises';
+
+import { getRedis, logger, getPG } from '@opengis/fastify-table/utils.js';
+
+import mapnik from './mapnik.js';
+import rasterVrt from './rasterVrt.js';
+import rasterExists from './rasterExists.js';
+import rootFolder from './rootFolder.mjs';
+
+const { gdal } = mapnik();
+
+const pg = getPG();
+const rclient = getRedis();
+
+export default async function rasterInfo(fullPath, send = () => { }, proj4text, nocache = 0) {
+  const time = [Date.now()];
+
+  const key = `raster-stat:${fullPath}`;
+  const cache = await rclient.get(key).then(el => (el ? JSON.parse(el) : null));
+
+  if (cache?.fullPath && cache.size && cache.extent && !nocache) {
+    return { type: 'redis', ...cache };
+  }
+
+  // console.log(fullPath)
+  if (fullPath.includes('.vrt')) {
+    await rasterVrt(fullPath, send)
+  }
+
+  // file cache
+  const hash = createHash('md5').update(fullPath).digest("hex");
+
+  const statPath = path.join(rootFolder, `stat/${hash}.json`);
+  if (existsSync(statPath) && !nocache) {
+    send('stat:' + statPath);
+    const fileData = await readFile(statPath, 'utf-8');
+
+    await rclient.set(key, fileData);
+    const parse = JSON.parse(fileData)
+
+    if (parse.fullPath && parse.extent) return parse;
+  }
+  // console.log(fullPath);
+
+  // size
+  const exists = fullPath.includes('.vrt') ? 1 : await rasterExists(fullPath)
+  //console.log(fullPath, exists)
+  send('exists:' + fullPath + ' - ' + (exists ? 'yes' : 'no'));
+  if (!exists) return { error: 'file not exists info ' + fullPath }
+
+  const { result: stat, error: err1 = '' } = await gdal({ name: 'ls -lah  ', path: fullPath })
+    .then(el => (el.result ? { result: el.result.replace(/\n/g, '') } : el));
+  time.push(Date.now());
+
+  const size = stat.split(' ')[4]
+
+  // srs
+  // send('srs:' + (proj4text || '') + fullPath)
+  const { result: srs, err: error1 } = proj4text ? { result: proj4text } : await gdal({ name: 'gdalsrsinfo', path: fullPath, params: '-o proj4 ' })
+    .then(el => (el.result ? { result: el.result.replace(/\n/g, '') } : el));
+
+  if (error1) return { error: error1 }
+  time.push(Date.now());
+
+  // gdalinfo
+  // send('info:' + fullPath)
+  const { result, err: error } = await gdal({ name: 'gdalinfo', path: fullPath, params: '-json' });
+
+  if (error) return { error }
+  const gdalInfo = JSON.parse(result);
+
+  time.push(Date.now());
+  send('wgs84:' + JSON.stringify(gdalInfo?.wgs84Extent) + ' coors:' + JSON.stringify(gdalInfo?.cornerCoordinates));
+
+  console.log(proj4text, gdalInfo?.cornerCoordinates)
+  if (proj4text && gdalInfo?.cornerCoordinates) {
+    const { lowerLeft, upperRight } = gdalInfo.cornerCoordinates;
+    //send(gdalInfo.cornerCoordinates);
+    const { geom, extent } = await pg.query(`select st_asgeojson(geom)::json as geom, geom::box2d as extent from 
+      (select st_transform('box(${lowerLeft[0]} ${lowerLeft[1]}, ${upperRight[0]} ${upperRight[1]})'::box2d,'${proj4text}','+proj=longlat +datum=WGS84 +no_defs') as geom )q`)
+      .then(el => el.rows[0]);
+    //send(extent)
+    gdalInfo.wgs84Extent = geom;
+    gdalInfo.extent = extent.substring(4, extent.length - 1).replace(/ /g, ',').split(',')
+  }
+
+  // extent
+  const geom = gdalInfo?.wgs84Extent;
+  const extent = gdalInfo.extent || (geom?.coordinates?.[0]?.[0] ? geom.coordinates[0][0].concat(geom.coordinates[0][2]) : undefined);
+  //  send(extent);
+
+  const data = {
+    time: { total: Date.now() - time[0], size: time[1] - time[0], srs: time[2] - time[1], info: time[3] - time[2] }, srs, gdalInfo, fullPath, size, extent, error
+  };
+
+  await rclient.setex(key, 8000, JSON.stringify(data));
+
+  // file cache
+  await mkdir(path.dirname(statPath), { recursive: true });
+
+  logger.file('editor', { time: Date.now() - time[0], srs, statPath, })
+
+  await writeFile(statPath, JSON.stringify(data, null, 2));
+
+  return data;
+}

package/server/plugins/mapnik/funcs/rasterVrt.js

@@ -0,0 +1,56 @@
+import path from 'node:path';
+
+import mapnik from './mapnik.js';
+import rasterConfig from './rasterConfig.js';
+
+const { gdal, uploadFile, downloadFile } = mapnik();
+
+export default async function rasterVrt(rasterPath, send = () => { }) {
+  if (rasterPath.includes('.tif')) {
+    return null;
+  }
+
+  const folderPath = path.posix.dirname(rasterPath);
+
+  // vrt config 
+  const configFilePath = rasterPath.replace('/preview1', '').replace('mosaic.vrt', 'config.yml')
+  const configData = await rasterConfig(configFilePath);
+
+  // -b 1 -b 2 -b 3 -overwrite
+  // -b 1 -b 2 -b 3 -hidenodata -srcnodata "0 0 0" -addalpha -overwrite 
+  const vrt = configData?.vrt || ' -b 1 -b 2 -b 3 -hidenodata -srcnodata "0 0 0" -vrtnodata "256 256 256" -addalpha -overwrite';
+
+
+  const paramsVrt = `${vrt}  -input_file_list ${folderPath}/files.txt `; // ${folderPath}/mosaic.vrt ${folderPath}/*.tif
+  send(paramsVrt)
+  const obj = {
+    name: 'gdalbuildvrt', out: ` -o ${folderPath}/mosaic.vrt `, params: paramsVrt,
+  };
+
+  // check mosaic.vrt exists
+  const time1 = Date.now()
+  const fullPath = rasterPath;
+
+  send('vrt:' + rasterPath)
+  const { result: resStat } = await gdal({ name: `ls "${fullPath}"` });
+
+  if (resStat) return { obj, resStat };
+
+  const { result: files } = await gdal({ name: `ls "${folderPath}"` });
+  const fileList = files.split('\n').filter(el => el.toLowerCase().endsWith('.tif') && !el.includes('mosaic')).map(el => path.posix.join(folderPath, el) || `"${path.posix.join(folderPath, el)}"`);
+  send({ folderPath, files: fileList.length, time: Date.now() - time1 })
+
+  if (!fileList.length) return { err: 'fileList is empty' };
+
+  await gdal({ name: `rm -rf "${folderPath}/files.txt"` });
+
+  await uploadFile({ fileBytes: Buffer.from(fileList.join('\n')), relativeFilepath: path.posix.join(folderPath, 'files.txt') });
+
+  const time = Date.now()
+  send(obj);
+  const { result, err } = await gdal(obj);
+  send(err || result);
+  await downloadFile({ filePath: fullPath });
+  send({ rasterPath, time: Date.now() - time })
+  return { result, err, obj };
+}

package/server/plugins/mapnik/funcs/rasterXML.js

@@ -0,0 +1,65 @@
+import path from 'node:path';
+import { mkdir, writeFile } from 'node:fs/promises';
+
+import { config, pgClients } from '@opengis/fastify-table/utils.js';
+
+import rasterInfo from './rasterInfo.js';
+import createXML from './createXML.js';
+import mapnik from './mapnik.js';
+import rootFolder from './rootFolder.mjs';
+
+const { loadXML } = mapnik();
+
+export default async function rasterXML({ id, pg = pgClients.client, send = () => { } }) {
+  if (!id) {
+    send('not enough params: id', true);
+    return { error: 'not enough params: id' };
+  }
+
+  const db = pg.options?.database;
+
+  const { source_path: rasterPath, srid, proj4 } = await pg.query(
+    'SELECT source_path, srid, proj4 from gis.rasters where raster_id=$1',
+    [id]
+  ).then(el => el.rows?.[0] || {});
+
+  const proj4text = srid ? await pg.query('select proj4text from spatial_ref_sys where srid::text=$1', [srid]).then(el => el.rows?.[0]?.proj4text) : null;
+  const fullPath = rasterPath ? path.posix.join(config.mapServerRoot || '', config.folder, '/map/raster/', rasterPath, (rasterPath?.toLowerCase().includes('.tif') ? '' : 'mosaic.vrt')) : null;
+
+  if (!fullPath) {
+    send('file not found ' + fullPath, true);
+    return { error: 'file not found ' + fullPath };
+  }
+
+  const { error, srs = proj4, extent, size, gdalInfo } = await rasterInfo(fullPath);
+
+  send(`xml srs: ${proj4text || proj4 || '-'} file:${fullPath} size:${size}`);
+
+  if (error) {
+    send(error.toString(), true);
+    return { error: error.toString() };
+  }
+
+  const xmlFileText = await createXML({
+    fullPath,
+    srs: proj4 || proj4text || srs,
+    extent,
+    send,
+    size,
+  });
+
+  if (srs && proj4 && gdalInfo?.wgs84Extent && id) {
+    const wgs = '+proj=longlat +datum=WGS84 +no_defs';
+    await pg.query(
+      'update gis.rasters set bbox=st_transform(st_transform($2,$5,$3),$4,$5) where raster_id=$1 returning bbox::box2d',
+      [id, gdalInfo.wgs84Extent, srs, proj4, wgs],
+    ).then(el => el.rows?.[0]) || {};
+  }
+
+  send(xmlFileText);
+  await loadXML({ xml: xmlFileText, name: id, reload: true });
+  const localPath = path.join(rootFolder, `mapnik/${db}_${id}.xml`);
+  await mkdir(path.dirname(localPath), { recursive: true });
+  await writeFile(localPath, xmlFileText);
+  return localPath;
+}

package/server/plugins/mapnik/funcs/rootFolder.mjs

@@ -0,0 +1,8 @@
+import path from 'node:path';
+import { existsSync } from 'node:fs';
+
+import { config } from '@opengis/fastify-table/utils.js';
+
+const root = config.temp || (existsSync('/data/local') ? '/data/local/kamianske' : process.cwd());
+
+export default path.join(root, 'log/map').replace(/\\/g, '/');

package/server/plugins/mapnik/index.mjs

@@ -0,0 +1,12 @@
+import fp from 'fastify-plugin';
+// import config from '../config.js';
+
+import mapnik from './funcs/mapnik.js';
+import gdalWrapper from './funcs/gdalWrapper.js';
+
+async function plugin(fastify, opts) {
+  // fastify.decorate('mapnik', mapnik);
+  // fastify.decorate('gdalWrapper', gdalWrapper);
+}
+
+export default fp(plugin);

package/server/plugins/mapnik/utils/map.proto

@@ -0,0 +1,241 @@
+syntax = "proto3";
+
+
+message render_in {
+	string path = 1;
+	string name = 2;
+	repeated double bbox = 3 [packed=true];
+	string tile = 4;
+	string xml = 5;
+	int32 width = 6;
+	int32 height = 7;
+}
+
+
+message render_out {
+	string err = 1;
+	string base64 = 2;
+	string tile = 3;
+}
+
+message xml_in {
+	string path = 1;
+	string name = 2;
+	string xml = 3;
+	bool reload = 4;
+}
+
+message xml_out {
+	string err = 1;
+	bool is_ok = 2;
+}
+
+message cmd_in {
+	string cmd = 1;
+	string svg = 2;
+	string filename = 3;
+}
+
+message cmd_out {
+	string err = 1;
+	string base64 = 2;
+}
+
+message gdal_in {
+	string name = 1;
+	string path = 2;
+	string out = 3;
+	string params = 4;
+}
+message gdal_out {
+	string result = 1;
+	string err = 2;
+}
+
+message clear_tiles_in {
+	string path = 1;
+}
+
+message clear_tiles_out {
+	oneof clear_message {
+		bool ok = 1;
+		string err = 2;
+	}
+}
+
+message empty {}
+
+message statusRenderInfo {
+	string name = 1;
+	bool isActive = 2;
+	float workTime = 3;
+}
+
+message statusInfo {
+	repeated statusRenderInfo info = 1;
+}
+
+message printMapOut {
+	string map = 1;
+	string err = 2;
+}
+
+message xyz {
+	float x = 1;
+	float y = 2;
+	float z = 3;
+}
+
+message printMapIn {
+	string baseurl = 1;
+	string baseImg = 2;
+	repeated string xml = 3;
+	string overlayXml = 4;
+	string overlay = 5;
+	int32 width = 6;
+	int32 height = 7;
+	repeated double bbox = 8;
+	xyz pos = 9;
+	string geojsonSettings = 10;
+}
+
+message log_in {
+	uint32 rows = 1;
+	string level = 2;
+}
+
+message log_out {
+	repeated string logs = 1;
+}
+
+message sqlToShpIn {
+  string db = 1;
+  string host = 2;
+  string port = 3;
+  string sql = 4;
+  string savePath = 5;
+	string shpName = 6;
+}
+
+message shapeOut {
+  string zipPath = 1;
+}
+
+message fileIn {
+	bytes fileBytes = 1;
+	string relativeFilepath = 2;
+}
+
+message deleteFileIn {
+	string dir = 1;
+	repeated string objectList = 2;
+}
+
+message fileManagerOut {
+	bool status = 1;
+	string operation = 2;
+}
+
+message checkFilesDir {
+	string dir = 1;
+	repeated string filesSearch = 2;
+}
+
+enum status {
+	LOADING = 0;
+	OK = 1;
+}
+
+message filesStatus {
+	status fileLoadingStatus = 1;
+}
+
+message fileListIn {
+	string dir = 1;
+}
+
+message fileInfo {
+	repeated fileInfoL fileInfoList = 1;
+}
+
+message fileInfoL {
+	oneof fileInfo {
+		fileInfoObj file = 1;
+		dirInfoObj dir = 2;
+	}
+}
+
+message fileInfoObj {
+	string type = 1;
+	string birthtime = 2;
+	string name = 3;
+	uint64 size = 4;
+	string ext = 5;
+	string server = 6;
+}
+
+message dirInfoObj {
+	string type = 1;
+	string birthtime = 2;
+	string name = 3;
+	uint64 size = 4;
+	uint64 count = 5;
+	string server = 6;
+}
+
+message deleteOut {
+	repeated deleteObjSuccess success = 1;
+	repeated deleteObjError error = 2;
+
+}
+
+message deleteObjSuccess {
+	string object = 1;
+	string type = 2;
+	string status = 3;
+}
+
+message deleteObjError {
+	string object = 1;
+	string fullPath = 2;
+	string result = 3;
+}
+
+message readDirIn {
+	string dir = 1;
+	repeated string filter = 2;
+	string fullPath = 3;
+}
+
+message readDirOut {
+	repeated string res = 1;
+
+}
+
+message downloadFileIn {
+	string filePath = 1;
+}
+
+message downloadFileOut {
+	bytes chunk_data = 1;
+}
+
+
+service Map {
+	rpc Render(render_in) returns (render_out) {}	
+	rpc RenderXML(render_in) returns (render_out) {}	
+	rpc LoadXML(xml_in) returns (xml_out) {}	
+	rpc MarkerIcon(cmd_in) returns (cmd_out) {}
+	rpc Gdal(gdal_in) returns (gdal_out) {}
+	rpc ClearTiles(clear_tiles_in) returns (clear_tiles_out) {}
+	rpc RenderStatus(empty) returns (statusInfo) {}
+	rpc PrintMap(printMapIn) returns (printMapOut) {}
+	rpc Log(log_in) returns (log_out) {}
+	rpc SqlToShp(sqlToShpIn) returns (shapeOut) {}
+	rpc uploadFile(fileIn) returns (fileManagerOut) {}
+	rpc deleteFiles(deleteFileIn) returns (deleteOut) {}
+	rpc checkFiles(checkFilesDir) returns (filesStatus) {}
+	rpc fileList(fileListIn) returns (fileInfo) {}
+	rpc readDir(readDirIn) returns (readDirOut) {}
+	rpc downloadFile(downloadFileIn) returns (stream downloadFileOut) {}
+}

package/server/routes/gis/index.mjs

@@ -0,0 +1,19 @@
+import insertColumns from './registers/insert.columns.js';
+import insertFilters from './registers/insert.filters.js';
+import metadataXML from './metadata/metadataXML.js';
+import getLayerGeom from './services/get.layer.geom.js';
+import gisRegistry from './registers/gis.registry.js';
+import gisRegistryList from './registers/gis.registry.list.js';
+import mapRegistry from './registers/map.registry.js';
+
+export default async function route(app) {
+  app.put('/insert-columns/:token', insertColumns);
+  app.put('/insert-filters/:token', insertFilters);
+  app.get('/gis-registry/:slug', { config: { policy: ['public'] } }, gisRegistry);
+  app.get('/gis-registry-list', { config: { policy: ['public'] } }, gisRegistryList);
+  app.get('/map-registry/:slug/:id', { config: { policy: ['public'] } }, mapRegistry);
+
+  app.get('/xml/:id', { config: { policy: ['public'] } }, metadataXML);
+
+  app.get('/get-layer-geom/:id', { config: { policy: ['public'] } }, getLayerGeom);
+}

package/server/routes/gis/metadata/metadataXML.js

@@ -0,0 +1,13 @@
+import {  handlebars, getTemplate } from "@opengis/fastify-table/utils.js";
+
+export default async function insertColumns(req,reply) {
+  const { params} = req;
+  const { id } = params;
+
+  const pt = await getTemplate('templates','ISO19136_2017_gml_template');
+  const data = await handlebars.compile(pt)({ id: id });
+
+  return reply
+    .header('Content-Type', 'text/xml; charset=utf-8')
+    .send(data);
+}

package/server/routes/gis/registers/funcs/classifiers.js

@@ -0,0 +1,26 @@
+import { getSelectVal } from '@opengis/fastify-table/utils.js';
+
+export async function attachClassifiers(rowOrRows, classifiers) {
+    const rows = Array.isArray(rowOrRows) ? rowOrRows : [rowOrRows];
+
+    for (const row of rows) {
+        for (const { name, classifier } of classifiers) {
+            const value = row[name];
+            if (value != null && classifier) {
+
+                const match = await getSelectVal({ name: classifier, values: [value] });
+                const matchVal = Object.values(match)?.[0]
+
+                if (typeof matchVal === 'object' && matchVal !== null) {
+                    row[`${name}_data`] = matchVal;
+                } else if (
+                    typeof matchVal === 'string' && matchVal !== null
+                ) {
+                    row[`${name}_text`] = matchVal;
+                };
+            }
+        }
+    }
+
+    return rowOrRows;
+}

package/server/routes/gis/registers/funcs/columns.js

@@ -0,0 +1,5 @@
+export function extractVisibleColumns(columns, mode = "table") {
+    return columns.filter((col) =>
+        mode === "card" ? col.hidden_card !== true : col.hidden_register !== true
+    );
+}

package/server/routes/gis/registers/funcs/handleRegistryRequest.js

@@ -0,0 +1,100 @@
+import { attachClassifiers } from './classifiers.js';
+import { extractVisibleColumns } from './columns.js';
+import { getMeta, getFilterSQL } from "@opengis/fastify-table/utils.js";
+
+export async function handleRegistryRequest({ settings, query, object_id, offset, limit, page, pg }) {
+    const { name, table_name, columns, filters, query: whereQuery, order, pk, register_id, is_files } = settings;
+    const parsedColumns = Array.isArray(columns) ? columns : JSON.parse(columns);
+    const filtersParsed = Array.isArray(filters) ? filters : JSON.parse(filters);
+    const activeFilters = filtersParsed.filter((f) => f.disabled !== true);
+    const { geom } = await getMeta({ pg, table: table_name });
+
+    const mode = object_id ? "card" : "table";
+    const visibleColumns = extractVisibleColumns(parsedColumns, mode);
+    const selectColumns = visibleColumns.map((col) => col.name);
+    const columnsInfo = visibleColumns.map((col) => ({
+        name: col.name,
+        ua: col.ua,
+        format: col.format,
+        data: col.data,
+        view_type: col.view_type
+    }));
+
+    const classifiers = visibleColumns
+        .filter((col) => col.data && ["select", "badge", "tags"].includes(col.format))
+        .map((col) => ({ name: col.name, classifier: col.data }));
+
+    if (object_id) {
+        const sql = `
+            SELECT "${pk}" as id, ${selectColumns.join(", ")} ${geom ? `, st_asgeojson(${geom})::json as geom` : ''}
+            FROM ${table_name}
+            WHERE ${pk} = $1
+        `;
+        const {
+            rows: [row],
+        } = await pg.query(sql, [object_id]);
+
+        //if (!row) return reply.code(404).send({ message: 'Object not found', status: 404 });
+        if (!row) throw new Error('Object not found');
+
+        await attachClassifiers(row, classifiers);
+
+        return {
+            row,
+            columns: columnsInfo,
+            register_id,
+            name,
+            is_files,
+        };
+    }
+
+    const { search, filter } = query || {};
+    const { q: sqlFilter } = (filter || search)
+        ? await getFilterSQL({
+            table: table_name,
+            search: search,
+            filter: filter,
+            filterList: filters,
+            json: 1
+        })
+        : { q: '1=1' };
+
+    const whereConditions = [whereQuery, sqlFilter].filter(Boolean).join(' AND ');
+    const whereClause = whereConditions ? `WHERE ${whereConditions}` : '';
+
+    const sqlBase = `FROM ${table_name} ${whereClause}`;
+    const sqlOrder = order ? `ORDER BY ${order}` : "";
+    const sqlLimit = `LIMIT $1 OFFSET $2`;
+
+    const sqlSelect = `SELECT "${pk}" as id, ${selectColumns.join(", ")} ${geom ? `, st_asgeojson(${geom})::json as geom` : ''} ${sqlBase}`;
+    const dataQuery = `${sqlSelect} ${sqlOrder} ${sqlLimit}`;
+    const totalQuery = `SELECT COUNT(*) ${sqlBase}`;
+
+    const [dataRes, countRes] = await Promise.all([
+        pg.query(dataQuery, [limit, offset]),
+        pg.query(totalQuery),
+    ]);
+
+    const rows = await attachClassifiers(dataRes.rows, classifiers);
+    const total = parseInt(countRes.rows[0]?.count || 0, 10);
+
+    const listConfig = {};
+    for (const col of parsedColumns) {
+        if (col.view_type === 'title') listConfig.title = col.name;
+        if (col.view_type === 'subtitle') listConfig.subtitle = col.name;
+    }
+
+    return {
+        register_id,
+        name,
+        rows,
+        total,
+        columns: columnsInfo,
+        filters: activeFilters,
+        limit,
+        page,
+        config: {
+            listConfig
+        }
+    };
+}

package/server/routes/gis/registers/gis.registry.js

@@ -0,0 +1,32 @@
+import { pgClients, getTemplate } from "@opengis/fastify-table/utils.js";
+import { handleRegistryRequest } from './funcs/handleRegistryRequest.js';
+const pg = pgClients.client;
+
+export default async function gisRegistry(req = {}) {
+    const { params = {}, query = {} } = req;
+    const { slug } = params;
+    const { page = 1, limit = 9, object_id } = query;
+    const offset = limit * (page - 1);
+
+    //if (!slug) return reply.code(404).send({ message: 'Params slug is required', status: 404 });
+    if (!slug) throw new Error('Params slug is required');
+
+    const {
+        rows: [registry],
+    } = await pg.query(
+        `SELECT register_id, name, table_name, columns, filters, query, "order", pk, register_key, is_files
+        FROM gis.registers
+        WHERE register_key = $1`,
+        [slug]
+    );
+    
+    if (registry) {
+        return await handleRegistryRequest({ settings: registry, query, object_id, offset, limit, page, pg });
+    }
+
+    const registryTemplate = await getTemplate('registry', slug);
+    //if (!registryTemplate) return reply.code(404).send({ message: 'Template not found', status: 404 });
+    if (!registryTemplate) throw new Error('Template not found');
+
+    return await handleRegistryRequest({ settings: registryTemplate, query, object_id, offset, limit, page, pg });
+}