@opengis/fastify-table 1.4.29 → 1.4.30

package/config.js

@@ -1,5 +1,4 @@
 import dotenv from 'dotenv';
-import path from 'node:path';
 
 import { existsSync, readFileSync } from 'node:fs';
 
@@ -8,46 +7,21 @@ import unflattenObject from './server/plugins/util/funcs/unflattenObject.js';
 const fileName = ['config.json', '/data/local/config.json'].find(el => (existsSync(el) ? el : null));
 const config = fileName ? JSON.parse(readFileSync(fileName)) : {};
 
+const { skipKeys = ['windir'] } = config;
+
 // npm run dev === cross-env NODE_ENV=development
 // alt: node --env=development
 Object.assign(config, {
   storageList: {},
   allTemplates: config?.allTemplates || {},
   skipCheckPolicyRoutes: [],
-  env: process.env?.NODE_ENV || process.argv[2]?.split?.('=')?.pop?.(),
+  env: process.env?.NODE_ENV,
 });
 
 function loadEnvConfig() {
-  if (config.env && existsSync(`.env.${config.env}`)) {
-    const { parsed } = dotenv.config({ path: `.env.${config.env}` });
-    if (parsed) {
-      console.log('start with env:', config.env);
-
-      const obj = unflattenObject(parsed);
-
-      Object.keys(obj || {})
-        .filter(key => typeof obj[key] === 'string'
-          && (obj[key].startsWith('[') || ['true', 'false'].includes(obj[key]))) // json array / boolean
-        .forEach(key => {
-          try {
-            obj[key] = JSON.parse(obj[key]);
-          }
-          catch (err) {
-            console.warn(`Invalid JSON for key "${key}": ${obj[key]}`);
-          }
-        });
-      if (obj) {
-        Object.assign(config, obj);
-        console.log('env init success', config.env, config.pg?.database);
-      }
-      else {
-        console.log('env init error', config.env, config.pg?.database);
-      }
-    }
-    else {
-      console.error('env init error: malformed file', config.env);
-    }
-  }
+  // node --env-file-if-exists=.env.dev --env-file-if-exists=.env server
+  const configKeys = Object.keys(process.env).filter(key => !skipKeys.includes(key) && key.charAt(0) === key.charAt(0)?.toLowerCase?.() || key.includes('.')).reduce((acc, curr) => ({ ...acc, [curr]: process.env[curr] }), {});
+  Object.assign(config, unflattenObject(configKeys));
 }
 
 loadEnvConfig();

package/index.js

@@ -5,6 +5,8 @@ import { fileURLToPath } from 'node:url';
 
 import config from './config.js';
 
+const { maxFileSize = 512 } = config;
+
 // helpers
 // import helperPlugin from './server/helpers/index.js';
 
@@ -113,7 +115,11 @@ async function plugin(fastify, opt) {
   templatesRoutes(fastify, opt);
 
   // from fastify-file
-  await fastify.register(import('@fastify/multipart')); // content parser, await before adding upload routes
+  await fastify.register(import('@fastify/multipart'), {
+    limits: {
+      fileSize: maxFileSize * 1024 * 1024,
+    },
+  }); // content parser, await before adding upload routes
   fastify.register(import('./server/routes/file/index.mjs'), opt);
   fastify.register(import('./server/routes/grpc/index.mjs'), opt);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengis/fastify-table",
-  "version": "1.4.29",
+  "version": "1.4.30",
   "type": "module",
   "description": "core-plugins",
   "keywords": [
@@ -26,7 +26,8 @@
     "test:routes": "node --test .\\test\\routes",
     "test:functions": "node --test .\\test\\functions",
     "compress": "node compress.js",
-    "dev": "set NODE_ENV=development&& node server.js"
+    "dev1": "set NODE_ENV=dev&& node server.js",
+    "dev": "node --env-file=.env.dev --env-file=.env server"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "3.554.0",
@@ -37,7 +38,6 @@
     "dotenv": "16.5.0",
     "fastify": "5.3.3",
     "fastify-plugin": "5.0.1",
-    "formidable": "3.5.1",
     "handlebars": "4.7.8",
     "image-size": "1.2.0",
     "ioredis": "5.3.2",

package/server/plugins/cron/index.js

@@ -47,7 +47,7 @@ async function runCron({
 
 async function plugin(fastify) {
     if (config.cronList?.length) {
-        config.cronList?.filter(el => el.query && !el.disabled)?.forEach?.((el, idx) => {
+        config.cronList?.filter?.(el => el.query && !el.disabled)?.forEach?.((el, idx) => {
             const { interval, db, query } = el;
             const name = createHash('md5').update(`${config.port || 3000}:${db}:${query}`).digest('hex');
             const pg = getPG(db);

package/server/plugins/crud/funcs/dataDelete.js

@@ -32,59 +32,55 @@ export default async function dataDelete({
 
   const delQuery = `delete from ${table} WHERE ${pk}::text = $1::text returning *`;
 
-    // for transactions
-    const isClient = typeof pg.query === 'function' && typeof pg.release === 'function';
-    const client = isClient ? pg : await pg.connect();
+  // for transactions
+  const isClient = typeof pg.query === 'function' && typeof pg.release === 'function';
+  const client = isClient ? pg : await pg.connect();
+
+  if (isClient || !client.pk) {
+    client.options = pg.options;
+    client.tlist = pg.tlist;
+    client.pgType = pg.pgType;
+    client.relkinds = pg.relkinds;
+    client.pk = pg.pk;
+  }
 
+  try {
     if (!isClient) {
-      client.caller = 'dataDelete';
-    }
-
-    if (isClient || !client.pk) {
-      client.options = pg.options;
-      client.tlist = pg.tlist;
-      client.pgType = pg.pgType;
-      client.relkinds = pg.relkinds;
-      client.pk = pg.pk;
+      await client.query('begin;');
     }
 
-    try {
-      if (client.caller === 'dataDelete') {
-        await client.query('begin;');
-      }
+    const row = {};
+    await extraData({
+      table, form: tokenData?.form, id, uid, row,
+    }, client);
 
-      const row = {};
-      await extraData({
-        table, form: tokenData?.form, id, uid, row,
-      }, client);
+    const res = await client.query(delQuery, [id])
+      .then(el => (el.rows?.[0] ? { rowCount: 1, ...el.rows[0] } : {}));
 
-      const res = await client.query(delQuery, [id])
-        .then(el => (el.rows?.[0] ? { rowCount: 1, ...el.rows[0] } : {}));
+    await logChanges({
+      pg: client, table, tokenData, referer, id, uid, type: 'DELETE',
+    });
 
-      await logChanges({
-        pg: client, table, tokenData, referer, id, uid, type: 'DELETE',
-      });
+    if (config.redis) { rclient.incr(`pg:${table}:crud`); }
 
-      if (config.redis) { rclient.incr(`pg:${table}:crud`); }
-
-      if (client.caller === 'dataDelete') {
-        await client.query('commit;');
-      }
-
-      return { ...res, ...row };
+    if (!isClient) {
+      await client.query('commit;');
     }
-catch (err) {
-      logger.file('crud/delete', {
-        error: err.toString(), stack: err.stack, table, id, referer, uid, form: tokenData?.form,
-      });
-      if (client.caller === 'dataDelete') {
-        await client.query('rollback;');
-      }
-      throw err;
+
+    return { ...res, ...row };
+  }
+  catch (err) {
+    logger.file('crud/delete', {
+      error: err.toString(), stack: err.stack, table, id, referer, uid, form: tokenData?.form,
+    });
+    if (!isClient) {
+      await client.query('rollback;');
     }
- finally {
-      if (client.caller === 'dataDelete') {
-        await client.query('begin;');
-      }
+    throw err;
+  }
+  finally {
+    if (!isClient) {
+      await client.query('begin;');
     }
+  }
 }

package/server/plugins/crud/funcs/dataUpdate.js

@@ -79,10 +79,6 @@ export default async function dataUpdate({
   const isClient = typeof pg.query === 'function' && typeof pg.release === 'function';
   const client = isClient ? pg : await pg.connect();
 
-  if (!isClient) {
-    client.caller = 'dataUpdate';
-  }
-
   if (isClient || !client.pk) {
     client.options = pg.options;
     client.tlist = pg.tlist;
@@ -92,7 +88,7 @@ export default async function dataUpdate({
   }
 
   try {
-    if (client.caller === 'dataUpdate') {
+    if (!isClient) {
       await client.query('begin;');
     }
     const res = await client.query(updateQuery, [id, ...filterValue])
@@ -161,7 +157,7 @@ export default async function dataUpdate({
 
     if (config.redis) { rclient.incr(`pg:${table}:crud`); }
 
-    if (client.caller === 'dataUpdate') {
+    if (!isClient) {
       await client.query('commit;');
     }
     return res || {};
@@ -170,13 +166,13 @@ export default async function dataUpdate({
     logger.file('crud/update', {
       error: err.toString(), stack: err.stack, table, id, referer, uid, form: tokenData?.form,
     });
-    if (client.caller === 'dataUpdate') {
+    if (!isClient) {
       await client.query('rollback;');
     }
     throw err;
   }
   finally {
-    if (client.caller === 'dataUpdate') {
+    if (!isClient) {
       client.release();
     }
   }

package/server/plugins/file/uploadMultiPart.js

@@ -1,62 +1,100 @@
 import path from 'node:path';
-import { mkdir } from 'node:fs/promises';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { randomUUID } from 'node:crypto';
+import { imageSize } from 'image-size';
 
 import config from '../../../config.js';
 
 import providers from './providers/index.js';
 
-import uploadFileDisk from './utils/uploadFileDisk.js';
-
 import { all, images } from './utils/allowedExtensions.js';
 
+import grpc from '../grpc/grpc.js';
+
+const { resizeImage } = grpc();
+
+const { resizeImageMinSize = 5 } = config; // resize images >= 5 MB by default
+
+async function writeFileToDisk(file, buffer) {
+  if (!file?.filepath || !file.extension || !buffer) { return null; }
+
+  // resize big images
+  if (images.find(el => el === file.extension) && file.size >= (resizeImageMinSize * 1024 * 1024)) {
+    const { width = 320, height = 240 } = imageSize(buffer) || {};
+
+    const ratio = width / height;
+
+    const resizeWidth = Math.min(width, 2048);
+    const resizeHeight = resizeWidth / ratio;
+
+    const { result } = await resizeImage({
+      base64: buffer.toString('base64'),
+      width: resizeWidth,
+      height: resizeHeight,
+      quality: 75,
+    });
+
+    await writeFile(`${file.filepath.replace(`.${file.extension}`, `_original.${file.extension}`)}`, buffer);
+    await writeFile(file.filepath, Buffer.from(result, 'base64'));
+    return null;
+  }
+
+  await writeFile(file.filepath, buffer);
+  return null;
+}
+
 export default async function uploadMultiPart(req) {
   const allowedExtensions = {
     '/file/upload-image/*': images,
   }[req.routeOptions?.url || ''] || all;
 
-  const dir = req.params?.['*'] || 'uploads';
-  const yearMonthDay = (new Date()).toISOString().split('T')[0];
-  const dbname = req.pg?.options?.database || req.pg?.database || config.pg?.database; // request / config params / default config params
-  const folder = path.join(config.root || `/data/local/${dbname || ''}`, config.folder || '', 'files', dir, yearMonthDay);
-  await mkdir(folder, { recursive: true });
-
-  const file = await uploadFileDisk({ req, folder }) || {};
-  if (!file?.filepath) throw new Error('upload error');
+  const parts = req.parts();
+  const part = await parts.next();
 
-  // originalFilepath for auto-resized images
-  const { filepath, originalFilepath } = file;
+  if (!part?.value?.filename) {
+    throw new Error('upload error');
+  }
 
-  const extName = path.extname(filepath).slice(1).toLowerCase();
+  const ext = path.extname(part.value.filename).toLowerCase();
 
   // check extension
-  if (allowedExtensions.indexOf(extName) === -1) {
+  if (!allowedExtensions.includes(ext.substring(1))) {
     throw new Error('file extension is not allowed');
   }
 
-  Object.assign(file, {
-    relativeOriginalFilepath: originalFilepath ? path.join(
-      '/files',
-      dir,
-      yearMonthDay,
-      path.relative(path.join(folder, 'files'), originalFilepath).replace(/\\/g, '/').replace(/\.\.\//g, ''),
-    ) : undefined,
-    relativeFilepath: path.join(
-      '/files',
-      dir,
-      yearMonthDay,
-      path.relative(path.join(folder, 'files'), filepath).replace(/\\/g, '/').replace(/\.\.\//g, ''),
-    ),
-  });
-  // console.log(path.join(folder, 'files), filepath);
+  const buffer = await part.value.toBuffer();
+
+  if (!buffer?.length) {
+    throw new Error('file buffer is empty');
+  }
+
+  const dir = req.params?.['*'] || 'uploads';
+  const yearMonthDay = (new Date()).toISOString().split('T')[0];
+
+  const dbname = req.pg?.options?.database || req.pg?.database || config.pg?.database; // request / config params / default config params
+
+  const rootDir = config.root || `/data/local/${dbname || ''}`;
+  const reldirpath = path.join('/files', dir, yearMonthDay);
+  const folder = path.join(rootDir, config.folder || '', reldirpath);
+
+  const newFilename = `${randomUUID()}${ext}`;
+
+  const file = {
+    originalFilename: part.value.filename,
+    filepath: path.join(folder, newFilename).replace(/\\/g, '/'),
+    relativeFilepath: path.join(reldirpath, newFilename).replace(/\\/g, '/'),
+    size: Buffer.byteLength(buffer),
+    mimetype: part.value.mimetype,
+    extension: ext.substring(1),
+  };
+
+  await mkdir(folder, { recursive: true });
+  await writeFileToDisk(file, buffer);
 
   // move file to s3
   if (config.s3?.endpoint) {
-    const fp = providers({ provider: 'fs' });
-    const data = await fp.downloadFile(filepath, { buffer: true });
-    if (typeof data === 'function') throw new Error('data is function!');
-    if (!data) throw new Error('upload error');
     const s3 = providers();
-    await s3.uploadFile(file.relativeFilepath, data);
+    await s3.uploadFile(file.relativeFilepath, buffer);
     if (config.trace) console.log('upload to s3', file.relativeFilepath);
   }
 

package/server/plugins/util/funcs/unflattenObject.js

@@ -1,14 +1,28 @@
+/* eslint-disable no-param-reassign */
 export default function unflattenObject(flatObj) {
     return Object.keys(flatObj).reduce((acc, key) => {
         const keys = key.split('.');
         keys.reduce((nestedObj, part, index) => {
             if (index === keys.length - 1) {
-                nestedObj[part] = flatObj[key];
-            } else {
+                // json array
+                if (typeof flatObj[key] === 'string' && flatObj[key].startsWith('[')) {
+                    try {
+                        nestedObj[part] = JSON.parse(flatObj[key]);
+                    }
+                    catch (err) {
+                        console.error(`Error parsing JSON for key ${key}:`, err);
+                        nestedObj[part] = flatObj[key]; // fallback to original value if parsing fails
+                    }
+                }
+                else {
+                    nestedObj[part] = ['true', 'false'].includes(flatObj[key]) ? JSON.parse(flatObj[key]) : flatObj[key];
+                }
+            }
+            else {
                 nestedObj[part] = nestedObj[part] || {};
             }
             return nestedObj[part];
         }, acc);
         return acc;
     }, {});
-}
+}

package/server/plugins/file/utils/uploadFileDisk.js

@@ -1,87 +0,0 @@
-import path from 'node:path';
-import { rename, readFile, writeFile } from 'node:fs/promises';
-
-import { formidable } from 'formidable';
-import { imageSize } from 'image-size';
-
-import config from '../../../../config.js';
-
-import { images } from './allowedExtensions.js';
-import grpc from '../../grpc/grpc.js';
-
-const { resizeImage } = grpc();
-const { maxFileSize = 512 } = config;
-
-async function handleImages(file) {
-  const ext = path.extname(file.originalFilename).toLowerCase();
-  if (images.find(el => el === ext.substring(1)) && file.size >= 5000000) {
-    const fileData = await readFile(file.filepath, { buffer: true });
-    const { width = 320, height = 240 } = imageSize(fileData) || {};
-
-    const ratio = width / height;
-
-    const resizeWidth = Math.min(width, 2048);
-    const resizeHeight = resizeWidth / ratio;
-
-    const { result } = await resizeImage({
-      base64: Buffer.from(fileData).toString('base64'),
-      width: resizeWidth,
-      height: resizeHeight,
-      quality: 75,
-    });
-
-    await rename(file.filepath, `${file.filepath}_original${ext}`);
-    await writeFile(file.filepath + ext, Buffer.from(result, 'base64'));
-
-    file.originalFilepath = `${file.filepath}_original${ext}`;
-    file.filepath += ext;
-
-    return file;
-  }
-  return null;
-}
-
-function promisify(func) {
-  // eslint-disable-next-line func-names
-  return async function (request) {
-    return new Promise((resolve, reject) => {
-      func(request, (err, fields, files) => {
-        if (err) { reject(err); }
-        resolve({ fields, files });
-      });
-    });
-  };
-}
-/*
-[{
-    filepath,
-    mimetype,
-    originalFilename,
-    size
-}]
-*/
-async function upload({ req, folder }) {
-  const form = formidable({ uploadDir: folder, keepExtensions: false, maxFileSize: maxFileSize * 1024 * 1024 });
-  const parse = promisify(form.parse.bind(form));
-
-  const files = await parse(req.raw).then((data) => data.files);
-
-  if (!files?.file?.[0]) return {};
-  const file = files.file[0];
-  const ext = path.extname(file.originalFilename).toLowerCase();
-
-  // resize big images
-  const result = await handleImages(file);
-  if (result) {
-    return result;
-  }
-
-  // rename with ext, keepExtensions has bug  if file with .
-  await rename(file.filepath, file.filepath + ext);
-
-  file.filepath += ext;
-
-  return file;
-}
-
-export default upload;