@opengis/gis 0.2.177 → 0.2.179

package/server/routes/registers/functions/getFilterSQLParametrized/getRangeQuery.js

@@ -0,0 +1,353 @@
+/* eslint-disable no-restricted-globals */
+const dateTypeList = [
+  "date",
+  "timestamp",
+  "timestamp without time zone",
+  "timestamp with time zone",
+];
+const numberTypeList = [
+  "float8",
+  "int4",
+  "int8",
+  "numeric",
+  "double precision",
+  "integer",
+  "bigint",
+];
+
+const isValidDate = (dateStr) => {
+  if (!dateStr) {
+    return false;
+  }
+
+  if (["min", "max", "cd", "cw", "cm", "cq", "cy"].includes(dateStr)) {
+    return true;
+  }
+
+  // iso date: 2024-01-01
+  if (dateStr.includes("-")) {
+    return new Date(dateStr).toString() !== "Invalid Date";
+  }
+
+  // locale date: 01.01.2024
+  const [dd, mm, yyyy] = dateStr.split(".");
+  return new Date(`${yyyy}-${mm}-${dd}`).toString() !== "Invalid Date";
+};
+
+const isValidNumber = (numStr) =>
+  ["min", "max"].includes(numStr) ? true : !isNaN(numStr);
+
+function dt(y, m, d) {
+  return new Date(Date.UTC(y, m, d)).toISOString().slice(0, 10);
+}
+
+const dp = {
+  d: new Date().getDate(),
+  w: new Date().getDate() - (new Date().getDay() || 7) + 1,
+  m: new Date().getMonth(),
+  q: parseFloat((new Date().getMonth() / 4).toFixed()) * 3,
+  y: new Date().getFullYear(),
+};
+function formatDateISOString(date) {
+  if (!date?.includes(".")) return date;
+  const [day, month, year] = date.split(".");
+  return `${year}-${month}-${day}`;
+}
+
+function checkValid(
+  value,
+  fieldType,
+  filterType,
+  sql,
+) {
+  const sep =
+    (value.includes(",") ? "," : null) ||
+    (value.includes("_") ? "_" : null) ||
+    (value.includes("-") ? "-" : null);
+
+  // number range w/o valid separator => skip invalid filter
+  if (filterType === "range" && !sep) {
+    return { isvalid: false };
+  }
+
+  // date range, specific options: current day, week, month, year etc.
+  if (["date", "datepicker"].includes(filterType) && value === "cd") {
+    return {
+      min: dt(dp.y, dp.m, dp.d),
+      max: dt(dp.y, dp.m, dp.d),
+      isvalid: true,
+    };
+  }
+
+  if (["date", "datepicker"].includes(filterType) && value === "cw") {
+    return {
+      min: dt(dp.y, dp.m, dp.w),
+      max: dt(dp.y, dp.m, dp.w + 6),
+      isvalid: true,
+    };
+  }
+
+  if (["date", "datepicker"].includes(filterType) && value === "cm") {
+    return {
+      min: dt(dp.y, dp.m, 1),
+      max: dt(dp.y, dp.m + 1, 0),
+      isvalid: true,
+    };
+  }
+
+  if (["date", "datepicker"].includes(filterType) && value === "cq") {
+    return {
+      min: dt(dp.y, dp.q, 1),
+      max: dt(dp.y, dp.q + 3, 0),
+      isvalid: true,
+    };
+  }
+
+  if (["date", "datepicker"].includes(filterType) && value === "cy") {
+    return {
+      min: dt(dp.y, 0, 1),
+      max: dt(dp.y, 11, 31),
+      isvalid: true,
+    };
+  }
+
+  // specific period from now - days, months, years
+  if (
+    ["date", "datepicker"].includes(filterType) &&
+    value.match(/^(?<val>[1-9]\d*)(?<unit>d|m|y)$/)?.groups?.unit
+  ) {
+    const { val, unit } =
+      value.match(/^(?<val>[1-9]\d*)(?<unit>d|m|y)$/)?.groups || {};
+    const min =
+      (unit === "d" ? dt(dp.y, dp.m, dp.d - +val) : null) ||
+      (unit === "m" ? dt(dp.y, dp.m - +val, dp.d) : null) ||
+      (unit === "y" ? dt(dp.y - +val, dp.m, dp.d) : null);
+    return {
+      min,
+      max: dt(dp.y, dp.m, dp.d),
+      isvalid: isValidDate(min),
+    };
+  }
+
+  // specific quarter - before skip date value validation
+  if (
+    ["date", "datepicker"].includes(filterType) &&
+    value.match(/^(?<year>\d{4})-q(?<quarter>[1-4])$/)?.groups?.quarter
+  ) {
+    const { year, quarter } =
+      value.match(/^(?<year>\d{4})-q(?<quarter>[1-4])$/)?.groups || {};
+    const startMonth = +quarter * 3;
+    return {
+      min: dt(year, startMonth - 3, 1),
+      max: dt(year, startMonth, 0),
+      isvalid: true,
+    };
+  }
+
+  // date value validation
+  if (
+    ["date", "datepicker"].includes(filterType) &&
+    !sep &&
+    !isValidDate(value)
+  ) {
+    return { isvalid: false };
+  }
+
+  // specific date / day
+  if (
+    ["date", "datepicker"].includes(filterType) &&
+    value.match(/^\d{4}-\d{2}-\d{2}$/)?.[0]
+  ) {
+    return {
+      min: new Date(value).toISOString().split("T")[0],
+      max: new Date(value).toISOString().split("T")[0],
+      isvalid: true,
+    };
+  }
+
+  // specific month
+  if (
+    ["date", "datepicker"].includes(filterType) &&
+    value.match(/^\d{4}-\d{2}$/)?.[0]
+  ) {
+    return {
+      min: new Date(value).toISOString().split("T")[0],
+      max: dt(new Date(value).getFullYear(), new Date(value).getMonth() + 1, 0),
+      isvalid: true,
+    };
+  }
+
+  // specific year
+  if (
+    ["date", "datepicker"].includes(filterType) &&
+    value.match(/^\d{4}$/)?.[0]
+  ) {
+    return {
+      min: dt(value, 0, 1),
+      max: dt(value, 11, 31),
+      isvalid: true,
+    };
+  }
+
+  if (sep) {
+    const [minValue = "min", maxValue = "max"] = value.split(sep);
+    // range for numbers
+    if (filterType === "range" && fieldType) {
+      const isvalid =
+        (numberTypeList.includes(fieldType) || sql) &&
+        isValidNumber(minValue) &&
+        isValidNumber(maxValue);
+      return { min: minValue, max: maxValue, isvalid };
+    }
+
+    // date range, example: 01.01.2024-31.12.2024
+    if (["date", "datepicker"].includes(filterType)) {
+      const min = minValue === "min" ? minValue : formatDateISOString(minValue);
+      const max = maxValue === "max" ? maxValue : formatDateISOString(maxValue);
+      const isvalid =
+        fieldType &&
+        (dateTypeList.includes(fieldType) || sql) &&
+        isValidDate(min) &&
+        isValidDate(max);
+      return { min, max, isvalid };
+    }
+  }
+
+  return { isvalid: false };
+}
+export default function getRangeQuery({
+  value,
+  name,
+  fieldType,
+  filterType,
+  sql,
+  args: argsParams,
+}) {
+  const { min, max, isvalid } = checkValid(
+    value,
+    fieldType,
+    filterType,
+    sql,
+  );
+  if (!isvalid) {
+    return { query: "false" };
+  }
+
+  const args = Array.from(argsParams || []);
+
+  const checkBoth =
+    sql && typeof sql === "string" && sql.includes("$1") && sql.includes("$2");
+
+  // with sql subquery
+  if (
+    filterType &&
+    ["date", "datepicker"].includes(filterType) &&
+    typeof sql === "string"
+  ) {
+    if (checkBoth) {
+      args.push(min);
+      args.push(max);
+    } else {
+      args.push(min || max);
+    }
+
+    return {
+      query: sql,
+      values: checkBoth ? [min, max] : [min || max],
+    };
+  }
+  // with sql subquery
+  if (filterType && ["range"].includes(filterType) && sql) {
+    if (checkBoth) {
+      args.push(min);
+      args.push(max);
+    } else {
+      args.push(min || max);
+    }
+
+    return {
+      query: sql,
+      values: checkBoth ? [min, max] : [min || max],
+    };
+  }
+
+  // preset date values
+  if (
+    filterType &&
+    ["date", "datepicker"].includes(filterType) &&
+    ["cd", "cw", "cm", "cq", "cy"].includes(value)
+  ) {
+    if (value === "cd") {
+      args.push(min);
+    } else {
+      args.push(min);
+      args.push(max);
+    }
+
+    const query =
+      value === "cd"
+        ? `${name}::date = $${args.length}::date`
+        : `${name}::date between $${args.length - 1}::date and $${args.length
+        }::date`;
+    return { query, values: value === "cd" ? [min] : [min, max] };
+  }
+
+  // same date
+  if (
+    filterType &&
+    ["date", "datepicker"].includes(filterType) &&
+    max &&
+    min &&
+    min === max
+  ) {
+    args.push(min);
+    return { query: `${name}::date = $${args.length}::date`, values: [min] };
+  }
+
+  // skip, same as 1=1
+  if (
+    filterType &&
+    ["range"].includes(filterType) &&
+    min === "min" &&
+    max === "max"
+  ) {
+    return { query: "true" };
+  }
+
+  // default
+  if (filterType && max === "max") {
+    args.push(min);
+    const query = ["date", "datepicker"].includes(filterType)
+      ? `${name} >= $${args.length}::date`
+      : `${name} >= $${args.length}`;
+    return {
+      query,
+      values: [min],
+    };
+  }
+
+  if (filterType && min === "min") {
+    args.push(max);
+    const query = ["date", "datepicker"].includes(filterType)
+      ? `${name} <= $${args.length}::date`
+      : `${name} <= $${args.length}`;
+    return { query, values: [max] };
+  }
+
+  if (filterType && ["date", "datepicker"].includes(filterType)) {
+    args.push(min);
+    args.push(max);
+    return {
+      query: `${name}::date between $${args.length - 1}::date and $${args.length
+        }::date`,
+      values: [min, max],
+    };
+  }
+
+  args.push(min);
+  args.push(max);
+  return {
+    query: `${name} between $${args.length - 1} and $${args.length}`,
+    values: [min, max],
+  };
+}

package/server/routes/registers/functions/getFilterSQLParametrized/index.js

@@ -0,0 +1,81 @@
+import { BadRequestError } from "@opengis/fastify-table/errors.js";
+import { pgClients } from "@opengis/fastify-table/utils.js";
+
+// filter util
+import getFilterQuery from "./getFilterQuery.js";
+
+export default function getFilterSQLParametrized({
+  table,
+  fields = [],
+  filter,
+  search,
+  searchColumn: searchColumn1,
+  filterList,
+  query,
+  uid,
+}, pg = pgClients.client) {
+  if (!pg) {
+    throw new Error("empty pg");
+  }
+
+  if (!table) {
+    throw BadRequestError("param table is required");
+  }
+
+  if (!pg.pk[table]) {
+    throw BadRequestError("param table is invalid");
+  }
+
+  const args = Array.from([]);
+
+  const autoSearchColumn = fields
+    ?.filter((el) => pg.pgType?.[el.dataTypeID] === "text")
+    ?.map((el) => `"${el.name}"`)
+    .join(",");
+
+  const searchColumn =
+    searchColumn1 ||
+    autoSearchColumn;
+
+  const where = [];
+
+
+  if (search && searchColumn && typeof search === "string") {
+    args.push(
+      `%${decodeURIComponent(search.replace(/%/g, "%25")).replace(
+        /%/g,
+        "\\%"
+      )}%`
+    );
+    where.push(
+      `(${searchColumn
+        .split(",")
+        ?.map((name) => `${name}::text ilike $${args.length}`)
+        ?.join(" or ")})`
+    );
+  }
+
+  const filters = getFilterQuery(
+    {
+      filter,
+      table,
+      fields,
+      filterList,
+      uid,
+      args,
+    },
+    pg
+  );
+
+  const filterQuery = filters
+    .filter((el) => el.query)
+    .map((el) => `${el.query}`)
+    .join(" and ");
+
+  const q = where
+    .concat([query, filterQuery])
+    .filter(Boolean)
+    .join(" and ");
+
+  return { q, args };
+}

package/server/routes/registers/functions/getTableColumnMeta.js

@@ -101,20 +101,14 @@ export default async function getTableColumnMeta(
     }
 
     const original = filtered
-        ? `with c(id,text) as (select "${queryColumn.replace(
-            /"/g,
-            ""
-        )}" as id, "${queryColumn.replace(
+        ? `with c(id,text) as (select rtrim( replace( replace( replace( encode("${queryColumn.replace(/"/g, '')}"::bytea, 'base64'), '+', '' ), '-', '' ), '/', '' ), '=' ) as id, "${queryColumn.replace(
             /"/g,
             ""
         )}" as text from ${tableName} t ${sqlTable} group by "${queryColumn.replace(
             /"/g,
             ""
         )}") select id, text from c`
-        : `with c(id,text) as (select "${queryColumn.replace(
-            /"/g,
-            ""
-        )}"  as id, "${queryColumn.replace(
+        : `with c(id,text) as (select rtrim( replace( replace( replace( encode("${queryColumn.replace(/"/g, '')}"::bytea, 'base64'), '+', '' ), '-', '' ), '/', '' ), '=' ) as id, "${queryColumn.replace(
             /"/g,
             ""
         )}" as text, count(*) from ${tableName} t ${sqlTable} group by "${queryColumn.replace(

package/server/routes/registers/functions/gis.suggest.js

@@ -8,7 +8,7 @@ const rclient = getRedis();
 const count = false;
 const maxLimit = 100;
 
-export default async function gisSuggest({ token, key, val, limit: limit1 = 50 }, pg) {
+export default async function gisSuggest({ token, key, val, uid, lang = 'ua', limit: limit1 = 50 }, pg) {
     const time = Date.now();
 
     const redisKey = `${pg?.options?.database}:registry-filter:${token}`;
@@ -163,10 +163,11 @@ export default async function gisSuggest({ token, key, val, limit: limit1 = 50 }
         };
     }
 
+    // table:column as select query
     const args = [];
     const where = [];
 
-    // select, with search by text support
+    // search by text support
     if (key && typeof key === "string") {
         args.push(key.toLowerCase());
         where.push(searchQuery);
@@ -183,16 +184,6 @@ export default async function gisSuggest({ token, key, val, limit: limit1 = 50 }
         where.push(`${clsMeta.pk}=any($${args.length})`);
     }
 
-    // filter
-    if (pg.pk?.[registry.table_name] && !cls && dataTypeID) {
-        where.push(
-            `id in (select ${pg.pgType[dataTypeID]?.includes("[]")
-                ? `unnest("${name}")`
-                : `"${name}"`
-            } from ${(registry.table_name).replace(/'/g, "''")})`
-        );
-    }
-
     // order by value if not specified explicitly
     const order = clsMeta.original.toLowerCase().includes("order by")
         ? ""
@@ -209,8 +200,8 @@ export default async function gisSuggest({ token, key, val, limit: limit1 = 50 }
             "''"
         )} o 
             WHERE ${pg.pgType[dataTypeID]?.includes("[]")
-            ? `c.id = ANY(o."${name}")`
-            : `o."${name}" IS NOT DISTINCT FROM c.id`
+            ? `c.text = ANY(o."${name}")`
+            : `o."${name}" IS NOT DISTINCT FROM c.text`
         } ) and ${whereQuery || "true"}
           ${order} LIMIT $${args.length}::bigint`
         : `with c(id,text) as ( ${clsMeta.original} where ${whereQuery} ${order}) select * from c LIMIT $${args.length}::bigint`;

package/server/routes/registers/functions/handleRegistryRequest.js

@@ -1,8 +1,9 @@
 import { createHash } from 'node:crypto';
 
-import { getMeta, getRedis, metaFormat, getFilterSQLParametrized } from "@opengis/fastify-table/utils.js";
+import { getMeta, getRedis, metaFormat } from "@opengis/fastify-table/utils.js";
 import { populateFilterOptions } from './classifiers.js';
 import { extractVisibleColumns } from './columns.js';
+import getFilterSQLParametrized from './getFilterSQLParametrized/index.js';
 
 const columnType = {
   text: 'text',
@@ -72,7 +73,10 @@ export async function handleRegistryRequest({
             WHERE ${pk} = $1
         `;
     const rows = await pg.query(sql, [object_id]).then(el => el.rows || []);
-    if (!rows.length) throw new Error('Object not found');
+
+    if (!rows.length) {
+      throw new Error('Object not found');
+    }
 
     await metaFormat({
       rows, table: table_name, cls: classifiers, sufix: true,
@@ -95,13 +99,15 @@ export async function handleRegistryRequest({
   }
 
   const { search, filter } = query || {};
+
   const { q: sqlFilter, args = [] } = (filter || search)
-    ? await getFilterSQLParametrized({
+    ? getFilterSQLParametrized({
       table: table_name,
+      fields: fields1,
       search,
       filter,
       filterList: activeFilters,
-    })
+    }, pg)
     : { q: '1=1', args: [] };
 
   const whereConditions = [whereQuery, sqlFilter].filter(Boolean).join(' AND ');
@@ -113,7 +119,9 @@ export async function handleRegistryRequest({
   const sqlSelect = `SELECT "${pk}" as id ${selectColumns.length ? `, ${selectColumns.join(", ")}` : ''} ${geom ? `, st_asgeojson(${geom})::json as geom` : ''} ${sqlBase}`;
   const dataQuery = `${sqlSelect} ${sqlOrder} ${sqlLimit}`;
 
-  if (sql) return dataQuery;
+  if (sql && process.platform === 'win32') {
+    return dataQuery;
+  }
 
   const rows = await pg.query(dataQuery, args.concat([limit, offset])).then(el => el.rows || []);
   const total = await pg.query(`SELECT COUNT(*) ${sqlBase}`, args).then(el => +(el.rows[0]?.count || 0));

package/server/routes/registers/registers.route.test.js

@@ -1,15 +1,16 @@
 import { afterAll, beforeAll, describe, expect, test } from 'vitest';
 
-import { getPGAsync } from '@opengis/fastify-table/utils.js';
+import { getPGAsync, getRedis } from '@opengis/fastify-table/utils.js';
 
 import { build, injectWithHeaders } from '../../../test/helper.js';
 
-let pg;
+let pg, rclient;
 
 beforeAll(async () => {
     await build();
 
     pg = await getPGAsync();
+    rclient = getRedis();
 
     await cleanUp('before');
 });
@@ -80,7 +81,8 @@ describe('GIS routes', () => {
         const payload = {
             register_key: 'unit-test',
             name: 'unit-test',
-            table_name: table
+            table_name: table,
+            filters: [{ name: pg.pk[table], ua: 'test filter', type: 'select' }]
         };
 
         const res = await injectWithHeaders({
@@ -130,6 +132,33 @@ describe('GIS routes', () => {
         expect(res.body.length).toBeGreaterThan(0);
     });
 
+    test('GET /gis-suggest/:token returns 200', async () => {
+        const { table, filter } = await pg.query('select table_name as table, filters->0->>\'name\' as filter from gis.registers where register_id=$1', [testId]).then((r) => r.rows[0]);
+        expect(filter).toBeTypeOf('string');
+        const hash = 'unit-test';
+        const redisKey = `${pg.options.database}:registry-filter:${hash}`;
+        await rclient.set(redisKey, JSON.stringify({ register: testId, filter }), 'EX', 1 * 60);
+        const res = await injectWithHeaders({
+            method: 'GET',
+            url: `/api/gis-suggest/${hash}`,
+        });
+        await rclient.del(redisKey);
+        expect(res.statusCode).toBe(200);
+        expect(res.body).toHaveProperty('time');
+        expect(res.body).toHaveProperty('limit');
+        expect(res.body).toHaveProperty('count');
+        expect(res.body).toHaveProperty('total');
+        expect(res.body).toHaveProperty('data');
+        expect(res.body.data).toBeInstanceOf(Array);
+        expect(res.body.data.length).toBeGreaterThan(0);
+        expect(res.body.data[0]).toHaveProperty('count');
+        expect(res.body.data[0]).toHaveProperty('id');
+        expect(res.body.data[0]).toHaveProperty('text');
+
+        const base64url = await pg.query(`select rtrim( replace( replace( replace( encode("${filter.replace(/"/g, '')}"::bytea, 'base64'), '+', '' ), '-', '' ), '/', '' ), '=' ) as id from ${table} where ${filter}::text=$1`, [res.body.data[0].text]).then((r) => r.rows[0].id);
+        expect(res.body.data[0].id).toBe(base64url);
+    });
+
     test('DELETE /gis-registry/:slug returns 200', async () => {
         const res = await injectWithHeaders({
             method: 'DELETE',

package/server/routes/registers/registers.schema.js

@@ -1,4 +1,8 @@
-const queryFilterSchema = { type: 'string', pattern: '^([\\w\\d_-]+)=([А-Яа-яҐґЄєІіЇї\\d\\w\\s\\/\\[\\]\\(\\)\\{\\}\\|,.!?;:—_=-@%#$&^*+=`~]+)$' };
+const queryFilterSchema = {
+    type: ["string", "null"],
+    pattern:
+        '^([\\w-]+)=([А-Яа-яҐґЄєІіЇї\\w\\s\\\\/\\[\\]\\(\\)\\{\\}\\|,.!?;:—_=@%#$&^*+`~]*)$',
+};
 const querySearchSchema = { type: 'string', pattern: '^([А-Яа-яҐґЄєІіЇї\\d\\w\\s\\/\\[\\]\\(\\)\\{\\}\\|,.!?;:—_=-@%#$&^*+=`~]+)$' };
 
 const keySchema = {
@@ -261,7 +265,8 @@ export const gisSuggestSchema = {
         additionalProperties: false,
         properties: {
             key: keySchema,
-            val: { type: "string", pattern: "([\\d\\w]+)$" },
+            json: { type: 'integer', enum: [1] },
+            val: querySearchSchema,
             limit: { type: 'integer', minimum: 1, maximum: 100 },
         },
     },