@opengis/fastify-table 1.4.87 → 1.5.0

package/server/routes/file/controllers/download.js

@@ -1,66 +0,0 @@
-import path from 'node:path';
-
-import providers from '../../../plugins/file/providers/index.js';
-import getMimeType from '../../../plugins/file/providers/mime/index.js';
-
-/* const allowedPublicDirs = [
-  'upload', 'page', 'site', 'maps', 'tmp', 'uploads', 'site_slider', 'module', 'product', 'image', 'geo_works_file',
-]; */
-
-/**
- * Апі використовується для скачування файлів за допомогою fs або s3
- *
- * @method GET
- * @summary Апі для скачування файлів за провайдером
- * @priority 4
- * @alias download
- * @type api
- * @tag file
- * @requires getMimeType
- * @param {Object} params Параметри URL
- * @errors 400,404,403,500
- * @returns {Number} status Номер помилки
- * @returns {String|Object} error Опис помилки
- * @returns {Object} headers Заголовки HTTP
- * @returns {String} pipe Шлях до файла для скачування або відображення
- */
-
-async function download({
-  params,
-}, reply) {
-  if (!params?.['*']) return { message: 'not enough params', status: 400 };
-
-  const filename = params['*'].startsWith('/') ? params['*'].slice(1) : params['*'];
-  if (!filename) return { message: "required param 'filename'", status: 400 };
-
-  if (filename?.includes?.('..')) return { message: 'wrong params', status: 400 };
-
-  /* const { security } = getSettings();
-  const { enabled: externalAccess } = await pg.one('select enabled from admin.data_api where account_name = $1 and api_key = $2', { args: [account, key] });
-  if ((sid === 35 && !isUser && !externalAccess)
-    || (sid === 1
-      && !allowedPublicDirs.some((dir) => filename?.includes?.(`${dir}/`))
-      && !security?.public_dirs?.split(',')?.some((dir) => filename?.includes?.(`${dir}/`)))
-  ) {
-    return { error: 'Немає доступу', status: 403 };
-  } */
-
-  const filepath = filename.startsWith('files/')
-    ? filename
-    : path.join('files', filename);
-
-  // download
-  const fp = providers({});
-
-  const fileStream = await fp.downloadFile(filepath);
-  if (!fileStream) return { error: `Файл не знайдено - ${filename}`, status: 404 };
-
-  const headers = {
-    'Content-Disposition': `attachment; filename=${path.basename(filename)}`,
-    'Content-Type': getMimeType(filepath),
-  };
-  reply.headers(headers);
-  return fileStream;
-}
-
-export default download;

package/server/routes/file/controllers/export.js

@@ -1,290 +0,0 @@
-/* eslint-disable no-console */
-/* eslint-disable no-param-reassign */
-/* eslint-disable no-plusplus */
-/* eslint-disable no-await-in-loop */
-/* eslint-disable no-nested-ternary */
-import path from 'node:path';
-import { createHash } from 'node:crypto';
-import { existsSync } from 'node:fs';
-import {
-  mkdir, readFile, rm, writeFile,
-} from 'node:fs/promises';
-
-import config from '../../../../config.js';
-import logger from '../../../plugins/logger/getLogger.js';
-import getTemplate from '../../../plugins/table/funcs/getTemplate.js';
-import getMeta from '../../../plugins/pg/funcs/getMeta.js';
-import pgClients from '../../../plugins/pg/pgClients.js';
-import eventStream from '../../../plugins/util/funcs/eventStream.js';
-import getData from '../../../plugins/table/funcs/getData.js';
-import getFolder from '../../../plugins/crud/funcs/utils/getFolder.js';
-import metaFormat from '../../../plugins/table/funcs/metaFormat/index.js';
-
-import jsonToXls from './utils/jsonToXls.js';
-import jsonToCsv from './utils/jsonToCsv.js';
-import formatResult from './utils/formatResult.js';
-
-const startStreamWithTotal = 10000;
-const rootDir = getFolder(config, 'local');
-
-/**
- * Експорт даних з таблиці
- *
- * @method GET
- * @alias exportTable
- * @type api
- * @tag export
- * @summary Експорт даних у таблицю(xlsx, csv, json, geojson)
- * @priority 1
- * @example
- * /api/export?table=com_property.subjects.table&format=csv&cols=economy_type,name_ua
- * @param {String} format Формат документу на виході
- * @param {Boolean} nocache Чи використовувати кеш
- * @param {String} table Таблиця в БД
- * @param {String|Number} filter Параметр фільтру для застосування до експортованих даних
- * @errors 400, 500
- * @returns {Number} status Номер помилки
- * @returns {String} error Опис помилки
- * @returns {String|Object} message Повертає SQL запит або opt або рядки SQL запиту
- * @returns {String} file Шлях до файла для скачування або відображення
- */
-
-export default async function exportTable({
-  pg = pgClients.client, user, unittest, columns: columns1, cls, query = {}, host = '127.0.0.1', tableSql, sourceName,
-}, reply) {
-  const {
-    id, cols, search, format = 'json',
-    table, filter = 'empty', nocache,
-    formatAnswer = 'file', sql, stream,
-  } = query;
-
-  if (!table && !tableSql) {
-    return reply.status(400).send('not enough params: table');
-  }
-
-  if (!['csv', 'xlsx', 'json', 'geojson'].includes(format)) {
-    return reply.status(400).send('param format is invalid');
-  }
-
-  const date = new Date();
-  const sufixName = `${filter}-${cols || 'all'}-${search}-${query.limit || 'unlimited'}`;
-  const sufixDate = [date.getFullYear(), date.getMonth(), date.getDate(), date.getHours()].join('-');
-  const objInfo = createHash('md5').update([sufixName, sufixDate].join('-')).digest('hex');
-  const fileName = (table || (tableSql ? createHash('md5').update(tableSql).digest('hex') : '')).concat('_').concat(objInfo).concat('.').concat(format);
-
-  const filePath = path.join(rootDir, '/files/temp', fileName);
-  const ext = path.extname(filePath);
-  const cacheFile = existsSync(filePath);
-
-  const filePathJSON = ['csv', 'xlsx', 'geojson'].includes(format) ? filePath.replace(ext, '.json') : filePath;
-  const cacheFileJSON = existsSync(filePathJSON);
-
-  // return from cache
-  if (cacheFile && !sql && !nocache && !config.disableCache) {
-    return formatResult({
-      filePath, formatAnswer, folder: rootDir, reply,
-    });
-  }
-
-  // delete old file, prevent append
-  if (nocache || config.disableCache) {
-    if (cacheFile) await rm(filePath);
-    if (cacheFileJSON && format !== 'json') await rm(filePathJSON);
-  }
-
-  const loadTable = await getTemplate('table', table);
-
-  const meta = await getMeta({ pg, table: loadTable?.table || table });
-
-  if (!meta?.pk && !meta?.view && !tableSql) {
-    return reply.status(404).send('table not found');
-  }
-
-  if (format === 'geojson' && !meta?.geom) {
-    return reply.status(400).send('Ця форма не містить полів геометрії. Виберіть тип, який не потребує геометрії для вивантаження');
-  }
-
-  const options = {
-    id,
-    table,
-    pg,
-    filter,
-    search,
-    user,
-    sql,
-    sufix: false,
-  };
-
-  // check total count, debug sql etc.
-  const result = tableSql
-    ? await pg.query(`select count(*) as total, json_agg(row_to_json(q)) as rows from (${tableSql})q`).then(el => el.rows?.[0] || {})
-    : await getData(options, reply, true);
-
-  if (sql) return result;
-
-  if (!result?.rows?.length) {
-    return reply.status(200).send('Немає даних, які можна експортувати');
-  }
-
-  const { total, filtered = result.total || 0 } = result;
-
-  const limit = startStreamWithTotal > filtered
-    ? filtered
-    : (Math.min(query.limit || 1000, startStreamWithTotal) || startStreamWithTotal);
-
-  Object.assign(options, { limit });
-
-  const colmodel = (columns1 || loadTable?.columns || meta?.columns || [])?.map((el) => ({
-    name: el.name,
-    data: el.data || el.option,
-    title: el.title || el.ua,
-    type: el.type || el.format || 'text',
-    html: el.html,
-  })); // skip html to avoid errors
-
-  // get present columns
-  const columns = cols === 'all' || !cols ? colmodel : colmodel
-    ?.filter((el) => (el.type || /\./.test(el.name))
-      && !el?.hidden && (cols?.split(',')?.length ? cols.split(',').includes(el.name) : true))
-    ?.filter(el => (Object.hasOwn(el, 'export') ? !el.export : true));
-
-  const htmls = columns.filter(el => el.html).reduce((acc, curr) => ({ ...acc, [curr.name]: curr.html }), {});
-
-  const columnList = columns?.map((el) => (/\./.test(el.name)
-    ? `${el.name.split('.')[0]}->>'${el.name.split('.')[1]}' as ${el.name.split('.').pop()}` // check for json data
-    : el.name));
-
-  const send = (+filtered > startStreamWithTotal || stream) && !unittest
-    ? eventStream(reply)
-    : (unittest ? console.log : () => { });
-
-  // export xlsx / csv / json
-  const source = loadTable?.title || loadTable?.ua || table || sourceName;
-
-  const interval = setInterval(async () => {
-    send('process query...');
-  }, 5000);
-
-  // start stream only if total exceed limit, but use while anyway
-  const res = {};
-  let offset = 0;
-  let page = 1;
-  let seq = 0;
-
-  send(`Всього в реєстрі: ${result.total} (${filtered} з урахуванням фільтрів)`);
-
-  if (!cacheFileJSON || nocache || config.disableCache) {
-    while ((+filtered - offset > 0) && !res?.error) {
-      try {
-        send(`Оброблено: ${offset}/${filtered}`);
-
-        const { rows = [] } = tableSql
-          ? await pg.query(`select * from (${tableSql})q limit ${options.limit} offset ${offset}`)
-          : await getData({ ...options, page }, reply, true);
-        send(`seq: ${++seq}`);
-        send(`Обробка ${rows.length} об'єктів...`);
-
-        if (!rows.length) {
-          send('Обробка даних успішно завершена');
-          break;
-        }
-
-        await metaFormat({
-          rows, cls, htmls, sufix: false,
-        }, pg);
-        // skip non present after metaFormat
-        if (!tableSql) {
-          rows.forEach((row) => {
-            Object.keys(row).filter((el) => !columnList.includes(el)).forEach((key) => delete row[key]);
-          });
-        }
-
-        const jsonFileExists = existsSync(filePathJSON);
-
-        // convert from json to format
-        if (!jsonFileExists) { // if json not exists
-          await mkdir(path.dirname(filePath), { recursive: true });
-          await writeFile(filePathJSON, JSON.stringify(rows));
-        }
-        else { // if json exists
-          const jsonData = JSON.parse(await readFile(filePathJSON) || '{}');
-          const moreData = jsonData.concat(rows); // rewrite to appendFile?
-          await writeFile(filePathJSON, JSON.stringify(moreData));
-        }
-
-        offset += rows.length;
-        page++;
-      }
-      catch (err) {
-        send(`error: ${err.toString()}`);
-        logger.file('export/table/error', {
-          filePath: filePathJSON,
-          total,
-          filtered,
-          offset,
-          result: res,
-          error: err.toString(),
-          stack: err.stack,
-        });
-        Object.assign(res, { error: err.toString() });
-      }
-    }
-  }
-
-  clearInterval(interval);
-
-  if (res.error) {
-    send(res.error, 1);
-    return reply.status(500).send(res.error);
-  }
-
-  logger.file('export/table', {
-    table, format, total, filtered, time: Date.now() - date.getTime(),
-  });
-
-  if (format !== 'json') {
-    const txt = nocache || config.disableCache || !cacheFileJSON
-      ? `Сформовано файл формату json. Початок конвертації в ${format}...`
-      : `Знайдено файл формату json. Початок конвертації в ${format}...`;
-    send(txt);
-  }
-
-  if (format === 'geojson') {
-    const rows = JSON.parse(await readFile(filePathJSON) || '[]');
-
-    const geojson = {
-      type: 'FeatureCollection',
-      features: rows.map((row) => ({
-        type: 'Feature',
-        name: 'export',
-        geometry: row.geom,
-        properties: Object.fromEntries(Object.entries(row).filter(([key]) => key !== 'geom')),
-      })),
-    };
-
-    await mkdir(path.dirname(filePath), { recursive: true });
-    await writeFile(filePath, JSON.stringify(geojson));
-  }
-
-  const resp = {};
-
-  if (format === 'csv') {
-    await jsonToCsv({
-      filePath: filePathJSON, send, colmodel, domain: host, source, columnList,
-    });
-  }
-  if (format === 'xlsx') {
-    await jsonToXls({
-      filePath: filePathJSON, send, colmodel, domain: host, source, resp,
-    });
-  }
-
-  if (resp.error) {
-    return reply.status(resp.status || 500).send(resp.error);
-  }
-
-  send('Файл успішно сформовано. Натистіть кнопку ще раз для завантаження даних', 1);
-  return formatResult({
-    filePath, formatAnswer, folder: rootDir, reply,
-  });
-}

package/server/routes/file/controllers/files.js

@@ -1,72 +0,0 @@
-import path from 'node:path';
-import mime from '../../../plugins/file/providers/mime/index.js';
-
-import isFileExists from '../../../plugins/file/isFileExists.js';
-import downloadFile from '../../../plugins/file/downloadFile.js';
-
-import { applyHook } from '../../../../utils.js';
-
-/**
- * Апі використовується для отримання різних файлів і можливістю змінювати їх
- *
- * @method GET
- * @summary Отримання файлів з директорії системи
- * @priority 5
- * @alias files
- * @type api
- * @tag file
- * @requires getFile
- * @requires mime
- * @param {Object} params Параметри URL
- * @errors 500
- * @returns {Number} status Номер помилки
- * @returns {String} error Опис помилки
- * @returns {Object} headers Заголовки HTTP
- * @returns {String} pipe Шлях до файла для скачування або відображення
- */
-
-export default async function getFile({
-  params = {},
-}, reply) {
-  if (!params?.['*']) {
-    return { message: 'not enough params', status: 400 };
-  }
-
-  if (params['*']?.includes('../')) {
-    return { message: 'wrong params', status: 403 };
-  }
-
-  const relpath = (params['*'].startsWith('/') ? params['*'].slice(1) : params['*']).replace(/files\//g, '') + (params['*'].endsWith('/') ? 'index.html' : '');
-
-  if (!relpath) {
-    return { message: 'No required param \'filename\'', status: 400 };
-  }
-
-  if (relpath.includes('..')) {
-    return { message: 'wrong params', status: 400 };
-  }
-
-  const hookData = await applyHook('preFile', { relpath, reply });
-  if (hookData) return hookData;
-
-  const filepath = path.join('files', relpath);
-
-  const exists = await isFileExists(filepath);
-  if (!exists) {
-    return { message: 'file not found', status: 404 };
-  }
-
-  const fileStream = await downloadFile(filepath);
-
-  if (!fileStream) {
-    return { message: 'file not found', status: 404 };
-  }
-
-  const headers = {
-    'Content-Disposition': `inline; filename=${path.basename(filepath)}`,
-    'Cache-Control': 'max-age=86400',
-    'Content-Type': `${mime(filepath)}`, // ; charset=utf-8 --- untested
-  };
-  reply.headers(headers);
-  return fileStream;
-}

package/server/routes/file/controllers/resize.js

@@ -1,96 +0,0 @@
-/* eslint-disable no-console */
-import path from 'node:path';
-import { imageSize } from 'image-size';
-
-import {
-  config, downloadFile, uploadFile, grpc, isFileExists,
-} from '../../../../utils.js';
-
-import getMimeType from '../../../plugins/file/providers/mime/index.js';
-
-const defaultWidth = 400;
-const defaultHeight = 240;
-const maxWidth = 2000;
-
-const getHeight = (width, size, ratio = 1) => {
-  if (size && size.toLowerCase().split('x')[1]) return size.toLowerCase().split('x')[1];
-  if (width) return width / ratio;
-  return defaultHeight;
-};
-
-const { resizeImage } = grpc();
-
-/**
- * Апі використовується для зміни розміру фото за шляхом
- */
-
-export default async function resize({ query = {}, unittest }, reply) {
-  const {
-    filepath, quality, size, w, h, nocache,
-  } = query;
-
-  if (!filepath) {
-    return reply.status(400).send('not enough query params: filepath');
-  }
-
-  const basename = path.basename(filepath);
-  const mimeType = getMimeType(filepath);
-  const resizePath1 = size
-    ? filepath.replace(basename, `${size}_resized_${basename}`)
-    : filepath.replace(basename, `${w || defaultWidth}_${h || (w ? '' : defaultHeight)}_resized_${basename}`);
-
-  // get Resize Data
-  const fileExists = await isFileExists(resizePath1);
-  const originalFileExists = await isFileExists(resizePath1.replace('files/', 'files/original/')); // resize-all API compatibility
-
-  const resizePath = originalFileExists ? resizePath1.replace('files/', 'files/original/') : resizePath1;
-
-  const resizeData = fileExists ? await downloadFile(resizePath, { buffer: true }) : null;
-
-  if (resizeData && !config.disableCache && !nocache && !unittest) {
-    return reply.headers({ 'Content-Type': mimeType, 'Cache-control': 'max-age=604800' }).send(resizeData);
-  }
-
-  // get File Data
-  const fileData = await downloadFile(filepath, { buffer: true });
-
-  if (!fileData?.length) {
-    return reply.status(404).send(`Файл не знайдено - ${filepath}`);
-  }
-
-  const resizeQuality = Math.min(quality || 75, 100);
-
-  const { width = defaultWidth, height = defaultHeight } = imageSize(fileData) || {};
-
-  const ratio = width / height;
-
-  const check = (size?.toLowerCase?.()?.split?.('x') || []).concat(w, h).filter(el => el && +el > maxWidth);
-
-  if (check.length) {
-    return reply.status(400).send({ message: 'resize image size too big' });
-  }
-
-  const resizeWidth = (h && !w ? h * ratio : null)
-    || (size?.toLowerCase?.()?.split?.('x')?.[1] && !size?.toLowerCase?.()?.split?.('x')?.[0] ? size.toLowerCase().split('x')[1] * ratio : null)
-    || w
-    || size?.toLowerCase?.()?.split?.('x')?.[0]
-    || Math.min(width, maxWidth);
-
-  const resizeHeight = h || getHeight(resizeWidth, size, ratio);
-
-  if (config.trace) {
-    console.log('original width/height/ratio: ', width, height, ratio);
-    console.log('resize width/height/ratio/quality: ', resizeWidth, resizeHeight, resizeWidth / resizeHeight, resizeQuality);
-  }
-
-  const { result } = await resizeImage({
-    base64: Buffer.from(fileData).toString('base64'),
-    width: resizeWidth,
-    height: resizeHeight,
-    quality: resizeQuality,
-  });
-
-  await uploadFile(resizePath, Buffer.from(result, 'base64'));
-
-  return reply.headers({ 'Content-Type': mimeType }).send(Buffer.from(result, 'base64'));
-}

package/server/routes/file/controllers/resizeAll.js

@@ -1,165 +0,0 @@
-/* eslint-disable no-console */
-import { imageSize } from 'image-size';
-
-import {
-  pgClients, eventStream, logger, isFileExists, downloadFile, uploadFile, grpc,
-} from '../../../../utils.js';
-
-import { images } from '../../../plugins/file/utils/allowedExtensions.js';
-
-const { resizeImage } = grpc();
-
-async function resizeOne({
-  relpath: relpath1, resizeQuality = 80, maxWidth = 2048, errors = [], success = [], notfound = [], equalwidth = [], send = () => { }, unittest,
-}) {
-  const exists = await isFileExists(relpath1);
-  const existsOriginal = await isFileExists(relpath1.replace('files/', 'files/original/'));
-
-  const relpath = existsOriginal ? relpath1.replace('files/', 'files/original/') : relpath1;
-
-  const buffer = (exists || existsOriginal) ? await downloadFile(relpath, { buffer: true }) : null;
-
-  if (!buffer?.length) {
-    notfound.push(relpath1);
-    send(`file not found: ${relpath1}`);
-    return null;
-  }
-
-  if (!existsOriginal) {
-    await uploadFile(relpath1.replace('files/', 'files/original/'), buffer);
-    send(`original image moved to: ${relpath1.replace('files/', 'files/original/')}`);
-  }
-  else {
-    send(`original image already saved to: ${relpath}`);
-  }
-
-  const dimensions = {};
-  try {
-    const { width, height } = imageSize(buffer) || {};
-    Object.assign(dimensions, { width, height });
-  }
-  catch (err) {
-    // handle broken image (callback w/ buffer input not supported)
-    errors.push(relpath1);
-    send(`${err.toString()}: ${relpath1} (${buffer?.length} - ${exists.ContentType})`);
-    return null;
-  }
-
-  const { width = maxWidth, height = 240 } = dimensions;
-
-  const ratio = width / height;
-
-  const resizeWidth = Math.min(width, maxWidth);
-  const resizeHeight = resizeWidth / ratio;
-
-  send(`original ${relpath} width/height/ratio: ${width} / ${height} / ${ratio}`);
-  send(`resize ${relpath} width/height/ratio/quality: ${resizeWidth} / ${resizeHeight} / ${resizeWidth / resizeHeight} / ${resizeQuality}`);
-
-  // skip for resize with small unit test image
-  if (resizeWidth === width && !unittest) {
-    send(`resize ${relpath} skip: resize width equals to original`);
-    equalwidth.push(relpath1);
-    return null;
-  }
-
-  const { result } = await resizeImage({
-    base64: buffer.toString('base64'),
-    width: resizeWidth,
-    height: resizeHeight,
-    quality: resizeQuality,
-  });
-
-  await uploadFile(relpath1, Buffer.from(result, 'base64'));
-  send(`resize ${relpath1} success`);
-  success.push(relpath1);
-  return null;
-}
-
-export default async function resizeAll(req, reply) {
-  const { pg = pgClients.client, query = {}, unittest } = req;
-
-  const {
-    nocache, filepath, quality = 80, limit, sql, w = 2048,
-  } = query;
-
-  if (nocache) {
-    const clearQ = `update crm.files set resized=null 
-    where ext=any($1) 
-    and ${filepath ? 'file_path=$2' : '1=1'} 
-    and resized is not null`;
-
-    if (sql) return clearQ;
-
-    const { rowCount = 0 } = await pg.query(clearQ, [images, filepath].filter(Boolean));
-    return reply.status(200).send({ msg: 'clear cache success', rowCount });
-  }
-
-  const resizeQuality = Math.min(quality, 100);
-
-  const q = `select file_id as id, file_path as relpath from crm.files
-  where ext=any($1)
-  and ${filepath ? 'file_path=$2' : '1=1'} 
-  and resized is null ${limit ? `limit ${limit}` : ''}`;
-
-  if (sql) return q;
-
-  const { rows = [], rowCount = 0 } = await pg.query(q, [images, filepath].filter(Boolean));
-
-  if (rowCount === 0) {
-    return reply.status(200).send('nothing to resize');
-  }
-
-  const success = [];
-  const errors = [];
-  const notfound = [];
-  const equalwidth = [];
-
-  const send = unittest ? console.log : eventStream(reply);
-
-  try {
-    send(`file max resize width: ${w}`);
-    send(`file max resize quality: ${resizeQuality}`);
-
-    send(`rows found: ${rowCount}`);
-
-    await rows
-      .filter((el, idx, array) => array.findIndex(item => item.relpath === el.relpath) === idx)
-      .reduce((promise, { relpath, id }) => promise.then(() => resizeOne({
-        relpath, id, resizeQuality, maxWidth: w, errors, success, notfound, equalwidth, send, unittest,
-      })), Promise.resolve());
-
-    if (success.length) {
-      await pg.query('update crm.files set resized=true where file_path=any($1::text[])', [success]);
-    }
-
-    const skip = errors.concat(notfound).concat(equalwidth);
-
-    if (skip.length) {
-      await pg.query('update crm.files set resized=false where file_path=any($1::text[])', [skip]);
-    }
-
-    logger.file('file/resize', {
-      success, errors, notfound, equalwidth,
-    });
-
-    send('resize all success');
-  }
-  catch (err) {
-    send(err.toString());
-    logger.file('file/resize', { error: err.toString(), stack: err.stack });
-  }
-  finally {
-    send('event stream finish', 1);
-  }
-
-  return {
-    success,
-    errors,
-    notfound,
-    equalwidth,
-    options: {
-      maxWidth: w,
-      quality: resizeQuality,
-    },
-  };
-}