@opengis/gis 0.2.176 → 0.2.178

package/server/routes/registers/functions/gis.suggest.js

@@ -0,0 +1,275 @@
+import { BadRequestError, NotFoundError } from "@opengis/fastify-table/errors.js";
+import { getColumnCLS, getMeta, getRedis, metaFormat } from "@opengis/fastify-table/utils.js";
+
+import getTableColumnMeta from "./getTableColumnMeta.js";
+
+const rclient = getRedis();
+
+const count = false;
+const maxLimit = 100;
+
+export default async function gisSuggest({ token, key, val, limit: limit1 = 50 }, pg) {
+    const time = Date.now();
+
+    const redisKey = `${pg?.options?.database}:registry-filter:${token}`;
+
+    const cache = await rclient.get(redisKey);
+
+    if (!cache) {
+        throw BadRequestError('invalid params: token');
+    }
+
+    const { register, filter } = JSON.parse(cache);
+
+    const {
+        rows: [registry],
+    } = await pg.query(
+        `SELECT register_id, table_name, filters FROM gis.registers WHERE register_id=$1`,
+        [register],
+    );
+
+    if (!registry) {
+        throw NotFoundError('registry not found');
+    }
+
+    if (!registry.table_name) {
+        throw new Error('registry table not set');
+    }
+
+    const filterData = (registry.filters).find(el => el.name === filter);
+
+    if (!filterData) {
+        throw new Error('registry filter not found');
+    }
+
+    const { name, data: cls } = filterData;
+
+    const metaColumns = await getMeta({ pg, table: registry.table_name }).then((r) => r.columns || []);
+
+    const { dataTypeID } = metaColumns.find((col) => col.name === name);
+
+    if (!dataTypeID) {
+        throw NotFoundError('registry column not found: ' + name);
+    }
+
+    const clsMeta = await getTableColumnMeta({
+        table: registry.table_name,
+        column: name,
+        filtered: !!(key || val),
+        key,
+    });
+
+    const { arr, searchQuery } = clsMeta;
+
+    const limit = Math.min(limit1 || maxLimit, maxLimit);
+
+    const [schemaName, tableName] = registry.table_name.split('.');
+
+    const table = [`"${schemaName}"`, `"${tableName}"`].join('.').replace(
+        /'/g,
+        "''"
+    );
+
+    // table:column + cls
+    if (arr && !cls) {
+        const sqlCls = count
+            ? `select value, count(*) from (select ${pg.pgType?.[dataTypeID]?.includes("[]")
+                ? `unnest("${name}")`
+                : `"${name}"`
+            } as value from ${table} where 1=1)q group by value`
+            : `select array_agg(distinct value)::text[] from (select ${pg.pgType?.[dataTypeID]?.includes("[]")
+                ? `unnest("${name}")`
+                : `"${name}"`
+            } as value from ${table} where 1=1)q`;
+
+        if (pg?.pk?.[registry.table_name] && !cls) {
+            const qRes = await pg.queryCache(sqlCls, {
+                table: registry.table_name,
+            });
+            const vals =
+                (count
+                    ? qRes.rows?.map?.((el) => el.value?.toString?.() || null)
+                    : qRes.rows?.[0]?.array_agg) || [];
+
+            const lower = key?.toLowerCase?.();
+            const data1 =
+                key || val
+                    ? arr
+                        ?.filter(
+                            (el) =>
+                                !lower ||
+                                (el[lang] || el.text)?.toLowerCase()?.indexOf(lower) !== -1
+                        )
+                        ?.filter(
+                            (el) => !val || val?.split?.(",")?.includes?.(el.id)
+                        )
+                    : arr;
+
+            const data2 = data1.filter((el) =>
+                el.id === "null"
+                    ? vals.includes(null)
+                    : el.id && vals.includes(el.id.toString())
+            );
+
+            const data = data2.slice(0, limit);
+
+            if (data.length && count) {
+                const counts = qRes.rows?.reduce?.(
+                    (acc, curr) => ({
+                        ...acc,
+                        [curr.value]: curr.count,
+                    }),
+                    {}
+                );
+                data.forEach((el) =>
+                    Object.assign(el, { count: counts?.[el.id] || 0 })
+                );
+            }
+
+            return {
+                time: Date.now() - time,
+                limit,
+                count: data.length,
+                total: arr.length,
+                mode: "array",
+                sql: process.platform === 'win32' ? sqlCls : undefined,
+                data,
+            };
+        }
+    }
+
+    // cls
+    if (arr) {
+        const lower = key?.toLowerCase();
+        const data1 =
+            key || val
+                ? arr
+                    ?.filter(
+                        (el) =>
+                            !lower ||
+                            (el[lang] || el.text)?.toLowerCase()?.indexOf(lower) !== -1
+                    )
+                    ?.filter((el) => !val || val?.split?.(",")?.includes?.(el.id))
+                : arr;
+        const data = data1.slice(0, Math.min(limit1 || limit, limit));
+
+        return {
+            time: Date.now() - time,
+            limit,
+            count: data.length,
+            total: arr.length,
+            mode: "array",
+            data,
+        };
+    }
+
+    // table:column as select query
+    const args = [];
+    const where = [];
+
+    // search by text support
+    if (key && typeof key === "string") {
+        args.push(key.toLowerCase());
+        where.push(searchQuery);
+    }
+
+    // pk
+    if (clsMeta.pk) {
+        where.push(`${clsMeta.pk} is not null`);
+    }
+
+    // filter by value
+    if (clsMeta.pk && typeof val === "string") {
+        args.push(val.split(","));
+        where.push(`${clsMeta.pk}=any($${args.length})`);
+    }
+
+    // order by value if not specified explicitly
+    const order = clsMeta.original.toLowerCase().includes("order by")
+        ? ""
+        : "order by 2";
+
+    const whereQuery = where.filter(Boolean).join(" and ") || "true";
+
+    args.push(limit);
+
+    let sqlSuggest = !cls
+        ? `${clsMeta.original}
+          WHERE EXISTS ( SELECT 1 FROM ${(registry.table_name).replace(
+            /'/g,
+            "''"
+        )} o 
+            WHERE ${pg.pgType[dataTypeID]?.includes("[]")
+            ? `c.text = ANY(o."${name}")`
+            : `o."${name}" IS NOT DISTINCT FROM c.text`
+        } ) and ${whereQuery || "true"}
+          ${order} LIMIT $${args.length}::bigint`
+        : `with c(id,text) as ( ${clsMeta.original} where ${whereQuery} ${order}) select * from c LIMIT $${args.length}::bigint`;
+
+    if (sqlSuggest.includes("{{parent}}")) {
+        args.push(parent || "");
+        sqlSuggest = sqlSuggest.replace(/(['"]?){{parent}}\1/gi, `$${args.length}`);
+    }
+
+    if (sqlSuggest.includes("{{uid}}")) {
+        args.push(uid || "0");
+        sqlSuggest = sqlSuggest.replace(/(['"]?){{uid}}\1/gi, `$${args.length}`);
+    }
+
+    const dataNew = await pg
+        .query(sqlSuggest, args)
+        .then((r) => r.rows || []);
+
+    // in case id / text = Boolean
+    const data = dataNew.filter(
+        (el) => Object.hasOwn(el, "id") && Object.hasOwn(el, "text")
+    );
+
+    data
+        .filter((el) => el.id === null)
+        .forEach((el) =>
+            Object.assign(el, { id: "null", text: "Показати порожні значення" })
+        );
+
+    // table:column + select
+    if (!cls) {
+        const { name: clsName = name, type = "select" } =
+            getColumnCLS(registry.table_name, name) || {};
+
+        await metaFormat(
+            {
+                rows: data,
+                table: registry.table_name,
+                cls: { text: clsName },
+                sufix: true,
+            },
+            pg
+        );
+
+        return {
+            time: Date.now() - time,
+            limit,
+            count: data.length,
+            total: clsMeta.count - 0,
+            mode: type === "cls" ? "array" : "sql",
+            db: clsMeta.db,
+            sql: process.platform === 'win32' ? sqlSuggest : undefined,
+            data: data.map((el) => ({
+                count: el.count,
+                id: el.id,
+                text: el.text_text || el.text,
+                ...(el.text_data || {}),
+            })),
+        };
+    }
+
+    return {
+        time: Date.now() - time,
+        limit,
+        count: data.length,
+        total: clsMeta.count - 0,
+        mode: "sql",
+        sql: process.platform === 'win32' ? sqlSuggest : undefined,
+        data,
+    };
+}

package/server/routes/registers/functions/handleRegistryRequest.js

@@ -1,147 +1,175 @@
-import { getMeta, getFilterSQL, metaFormat, getFilterSQLParametrized } from "@opengis/fastify-table/utils.js";
-import { populateFilterOptions } from './classifiers.js';
-import { extractVisibleColumns } from './columns.js';
-
-const columnType = {
-  text: 'text',
-  date: 'date',
-  bool: 'yes/no',
-  numeric: 'number',
-  integer: 'number',
-  'timestamp without time zone': 'date',
-  'timestamp with time zone': 'date',
-};
-
-// eslint-disable-next-line import/prefer-default-export
-export async function handleRegistryRequest({
-  settings, query, object_id, offset = 0, limit = 16, page = 1, pg, sql,
-}) {
-  const {
-    name, table_name, description, card, columns, filters, query: whereQuery, pk,
-    register_id, view, order, is_downloadable, register_key, is_map, is_register, service_id, is_files,
-  } = settings;
-  const parsedColumns = Array.isArray(columns) ? columns : JSON.parse(columns);
-  const parsedCardColumns = Array.isArray(card) ? card : JSON.parse(card);
-  const filtersParsed = Array.isArray(filters) ? filters : JSON.parse(filters);
-  const activeFilters = (filtersParsed || []).filter((f) => f.disabled !== true).map(el => ({
-    ...el, id: el.name || el.id, type: el.type, api: `/api/suggest/${table_name}:${el.name}${el.data ? `?sel=${el.data}` : ''}`,
-  }));
-  const { columns: fields1, geom } = await getMeta({ pg, table: table_name });
-
-  const fields = fields1?.map?.(({ name, dataTypeID, title }) => ({ name, type: columnType[pg.pgType?.[dataTypeID] || 'text'] || 'text', label: title || name }));
-
-  const mode = object_id ? "card" : "table";
-  const visibleColumns = (mode === 'card' && parsedCardColumns?.length) ? parsedCardColumns : extractVisibleColumns(parsedColumns, mode);
-  const selectColumns = visibleColumns.map((col) => col.name);
-  /* const columnsInfo = visibleColumns.map((col) => ({
-        name: col.name,
-        ua: col.ua,
-        format: col.format,
-        data: col.data,
-        view_type: col.view_type
-    })); */
-
-  const classifiers = visibleColumns
-    .filter((col) => col.data && ["select", "badge", "tags"].includes(col.format))
-    .reduce((acc, curr) => ({ ...acc, [curr.name]: curr.data }), {});
-
-  if (object_id) {
-    const sql = `
-            SELECT "${pk}" as id,* ${selectColumns.length ? `, ${selectColumns.join(", ")}` : ''} ${geom ? `, st_asgeojson(${geom})::json as geom` : ''}
-            FROM ${table_name}
-            WHERE ${pk} = $1
-        `;
-    const rows = await pg.query(sql, [object_id]).then(el => el.rows || []);
-    if (!rows.length) throw new Error('Object not found');
-
-    await metaFormat({
-      rows, table: table_name, cls: classifiers, sufix: true,
-    });
-
-    return {
-      row: rows[0],
-      card,
-      d: 1,
-      columns: visibleColumns,
-      is_downloadable,
-      register_key,
-      register_id,
-      service_id,
-      name,
-      is_files,
-      is_map,
-      is_register,
-    };
-  }
-
-  const { search, filter } = query || {};
-  const { q: sqlFilter, args = [] } = (filter || search)
-    ? await getFilterSQLParametrized({
-      table: table_name,
-      search,
-      filter,
-      filterList: activeFilters,
-    })
-    : { q: '1=1', args: [] };
-
-  const whereConditions = [whereQuery, sqlFilter].filter(Boolean).join(' AND ');
-
-  const sqlBase = `FROM ${table_name} WHERE ${whereConditions || 'true'}`;
-  const sqlOrder = order ? `ORDER BY ${order}` : "";
-  const sqlLimit = `LIMIT $${args.length + 1}::bigint OFFSET $${args.length + 2}::bigint`;
-
-  const sqlSelect = `SELECT "${pk}" as id ${selectColumns.length ? `, ${selectColumns.join(", ")}` : ''} ${geom ? `, st_asgeojson(${geom})::json as geom` : ''} ${sqlBase}`;
-  const dataQuery = `${sqlSelect} ${sqlOrder} ${sqlLimit}`;
-
-  if (sql) return dataQuery;
-
-  const rows = await pg.query(dataQuery, args.concat([limit, offset])).then(el => el.rows || []);
-  const total = await pg.query(`SELECT COUNT(*) ${sqlBase}`, args).then(el => +(el.rows[0]?.count || 0));
-
-  await metaFormat({
-    rows, table: table_name, cls: classifiers, sufix: true,
-  });
-
-  const listConfig = {};
-
-  if (parsedColumns?.length) {
-    for (const col of parsedColumns) {
-      if (col.view_type === 'title') listConfig.title = col.name;
-      if (col.view_type === 'subtitle') listConfig.subtitle = col.name;
-    }
-  }
-
-  await populateFilterOptions({
-    pg,
-    table: table_name,
-    filters: activeFilters,
-    columns,
-    whereConditions,
-    args
-  });
-
-  return {
-    total,
-    limit,
-    page,
-    register_id,
-    name,
-    card,
-    table_name,
-    register_key,
-    description,
-    rows,
-    service_id,
-    columns: visibleColumns,
-    filters: activeFilters,
-    fields,
-    view,
-    is_downloadable,
-    is_map,
-    is_register,
-    is_files,
-    config: {
-      listConfig,
-    },
-  };
-}
+import { createHash } from 'node:crypto';
+
+import { getMeta, getRedis, metaFormat } from "@opengis/fastify-table/utils.js";
+import { populateFilterOptions } from './classifiers.js';
+import { extractVisibleColumns } from './columns.js';
+import getFilterSQLParametrized from './getFilterSQLParametrized/index.js';
+
+const columnType = {
+  text: 'text',
+  date: 'date',
+  bool: 'yes/no',
+  numeric: 'number',
+  integer: 'number',
+  'timestamp without time zone': 'date',
+  'timestamp with time zone': 'date',
+};
+
+const rclient = getRedis();
+
+// eslint-disable-next-line import/prefer-default-export
+export async function handleRegistryRequest({
+  settings, query, object_id, offset = 0, limit = 16, page = 1, pg, sql,
+}) {
+  const {
+    name, table_name, description, card, columns, filters, query: whereQuery, pk,
+    register_id, view, order, is_downloadable, register_key, is_map, is_register, service_id, is_files,
+  } = settings;
+  const parsedColumns = Array.isArray(columns) ? columns : JSON.parse(columns);
+  const parsedCardColumns = Array.isArray(card) ? card : JSON.parse(card);
+  const filtersParsed = Array.isArray(filters) ? filters : JSON.parse(filters);
+  const activeFilters = await Promise.all((filtersParsed || []).filter((f) => f.disabled !== true).map(async (el) => {
+    const hash = createHash('md5').update([register_id, el.name, el.data].filter(Boolean).join('-')).digest('hex');
+    const redisKey = `${pg?.options?.database}:registry-filter:${hash}`;
+
+    const r = {
+      ...el, id: el.name || el.id, type: el.type, api: `/api/gis-suggest/${hash}`
+    };
+
+    const unique = await rclient.setnx(redisKey, 1);
+    const ttl = await rclient.ttl(redisKey);
+
+    // skip subsequent calls
+    if (!unique && ttl !== -1) {
+      return r;
+    }
+
+    await rclient.set(redisKey, JSON.stringify({ register: register_id, filter: el.name }), 'EX', 15 * 60);
+    return r;
+  }));
+  const { columns: fields1, geom } = await getMeta({ pg, table: table_name });
+
+  const fields = fields1?.map?.(({ name, dataTypeID, title }) => ({ name, type: columnType[pg.pgType?.[dataTypeID] || 'text'] || 'text', label: title || name }));
+
+  const mode = object_id ? "card" : "table";
+  const visibleColumns = (mode === 'card' && parsedCardColumns?.length) ? parsedCardColumns : extractVisibleColumns(parsedColumns, mode);
+  const selectColumns = visibleColumns.map((col) => col.name);
+  /* const columnsInfo = visibleColumns.map((col) => ({
+        name: col.name,
+        ua: col.ua,
+        format: col.format,
+        data: col.data,
+        view_type: col.view_type
+    })); */
+
+  const classifiers = visibleColumns
+    .filter((col) => col.data && ["select", "badge", "tags"].includes(col.format))
+    .reduce((acc, curr) => ({ ...acc, [curr.name]: curr.data }), {});
+
+  if (object_id) {
+    const sql = `
+            SELECT "${pk}" as id,* ${selectColumns.length ? `, ${selectColumns.join(", ")}` : ''} ${geom ? `, st_asgeojson(${geom})::json as geom` : ''}
+            FROM ${table_name}
+            WHERE ${pk} = $1
+        `;
+    const rows = await pg.query(sql, [object_id]).then(el => el.rows || []);
+
+    if (!rows.length) {
+      throw new Error('Object not found');
+    }
+
+    await metaFormat({
+      rows, table: table_name, cls: classifiers, sufix: true,
+    });
+
+    return {
+      row: rows[0],
+      card,
+      d: 1,
+      columns: visibleColumns,
+      is_downloadable,
+      register_key,
+      register_id,
+      service_id,
+      name,
+      is_files,
+      is_map,
+      is_register,
+    };
+  }
+
+  const { search, filter } = query || {};
+
+  const { q: sqlFilter, args = [] } = (filter || search)
+    ? getFilterSQLParametrized({
+      table: table_name,
+      fields: fields1,
+      search,
+      filter,
+      filterList: activeFilters,
+    }, pg)
+    : { q: '1=1', args: [] };
+
+  const whereConditions = [whereQuery, sqlFilter].filter(Boolean).join(' AND ');
+
+  const sqlBase = `FROM ${table_name} WHERE ${whereConditions || 'true'}`;
+  const sqlOrder = order ? `ORDER BY ${order}` : "";
+  const sqlLimit = `LIMIT $${args.length + 1}::bigint OFFSET $${args.length + 2}::bigint`;
+
+  const sqlSelect = `SELECT "${pk}" as id ${selectColumns.length ? `, ${selectColumns.join(", ")}` : ''} ${geom ? `, st_asgeojson(${geom})::json as geom` : ''} ${sqlBase}`;
+  const dataQuery = `${sqlSelect} ${sqlOrder} ${sqlLimit}`;
+
+  if (sql && process.platform === 'win32') {
+    return dataQuery;
+  }
+
+  const rows = await pg.query(dataQuery, args.concat([limit, offset])).then(el => el.rows || []);
+  const total = await pg.query(`SELECT COUNT(*) ${sqlBase}`, args).then(el => +(el.rows[0]?.count || 0));
+
+  await metaFormat({
+    rows, table: table_name, cls: classifiers, sufix: true,
+  });
+
+  const listConfig = {};
+
+  if (parsedColumns?.length) {
+    for (const col of parsedColumns) {
+      if (col.view_type === 'title') listConfig.title = col.name;
+      if (col.view_type === 'subtitle') listConfig.subtitle = col.name;
+    }
+  }
+
+  await populateFilterOptions({
+    pg,
+    table: table_name,
+    filters: activeFilters,
+    columns,
+    whereConditions,
+    args
+  });
+
+  return {
+    total,
+    limit,
+    page,
+    register_id,
+    name,
+    card,
+    table_name,
+    register_key,
+    description,
+    rows,
+    service_id,
+    columns: visibleColumns,
+    filters: activeFilters,
+    fields,
+    view,
+    is_downloadable,
+    is_map,
+    is_register,
+    is_files,
+    config: {
+      listConfig,
+    },
+  };
+}