@opengis/fastify-table 1.4.87 → 1.5.0

package/dist/server/plugins/file/providers/s3/client.js

@@ -0,0 +1,26 @@
+import { S3Client } from "@aws-sdk/client-s3";
+import { createHash } from "crypto";
+function md5(string) {
+    return createHash("md5").update(string).digest("hex");
+}
+const CLIENT_CACHE = {};
+const getS3Client = ({ accessKeyId, secretAccessKey, user, password, endpoint, host, region = "us-west-2", }) => {
+    // md5 have a hign chance of collisions consider switching on highload
+    // if (!accessKeyId || !secretAccessKey || !endpoint) throw new Error('No required params');
+    const uniqueKey = md5(`${accessKeyId}${secretAccessKey}${endpoint}`);
+    let client = CLIENT_CACHE[uniqueKey];
+    if (client)
+        return client;
+    client = new S3Client({
+        credentials: {
+            accessKeyId: accessKeyId || user,
+            secretAccessKey: secretAccessKey || password,
+        },
+        endpoint: endpoint || host,
+        forcePathStyle: true,
+        region,
+    });
+    CLIENT_CACHE[uniqueKey] = client;
+    return client;
+};
+export default { getS3Client };

package/dist/server/plugins/file/providers/s3/funcs/downloadFile.js

@@ -0,0 +1,42 @@
+import { GetObjectCommand } from "@aws-sdk/client-s3";
+import { readFile } from "node:fs/promises";
+import { createReadStream } from "node:fs";
+import client from "../client.js";
+import streamToBuffer from "../../utils/streamToBuffer.js";
+// if not found on s3 => fs
+import fsFuncs from "../../fs.js";
+import getPath from "../../../utils/getPath.js";
+import getS3FilePath from "./utils/getS3FilePath.js";
+const getFileStream = (s3Settings) => async (fp, options = {}) => {
+    const filepath = getS3FilePath(fp, s3Settings);
+    const s3Client = client.getS3Client(s3Settings);
+    const bucketName = s3Settings.containerName;
+    const bucketParams = {
+        Bucket: bucketName,
+        Key: filepath[0] === "/" ? filepath?.slice(1) : filepath,
+    };
+    try {
+        const data = await s3Client.send(new GetObjectCommand(bucketParams));
+        if (options.buffer) {
+            return streamToBuffer(data.Body);
+        }
+        return data.Body;
+    }
+    catch (err) {
+        const filepath1 = getPath(fp, options);
+        if (!filepath1) {
+            return null;
+        }
+        const { fileExists } = fsFuncs();
+        const exists = await fileExists(filepath1);
+        if (!exists) {
+            return null;
+        }
+        if (options.buffer) {
+            return readFile(filepath1);
+        }
+        const fileStream = createReadStream(filepath1);
+        return fileStream;
+    }
+};
+export default getFileStream;

package/dist/server/plugins/file/providers/s3/funcs/fileExists.js

@@ -0,0 +1,24 @@
+import { HeadObjectCommand, } from '@aws-sdk/client-s3';
+import client from '../client.js';
+import getPath from '../../../utils/getPath.js';
+import getS3FilePath from './utils/getS3FilePath.js';
+// if not found on s3 => fs
+import isFileExists from '../../../utils/isFileExists.js';
+const getFileMetadata = (s3Settings) => async (fp) => {
+    const filepath = getS3FilePath(fp, s3Settings);
+    const s3Client = client.getS3Client(s3Settings);
+    const bucketName = s3Settings.containerName;
+    const bucketParams = {
+        Bucket: bucketName,
+        Key: filepath[0] === '/' ? filepath?.slice(1) : filepath,
+    };
+    try {
+        const data = await s3Client.send(new HeadObjectCommand(bucketParams));
+        return data;
+    }
+    catch (err) {
+        const filepath1 = getPath(fp);
+        return isFileExists(filepath1);
+    }
+};
+export default getFileMetadata;

package/dist/server/plugins/file/providers/s3/funcs/uploadFile.js

@@ -0,0 +1,35 @@
+import { Upload } from "@aws-sdk/lib-storage";
+import client from "../client.js";
+import getValidData from "../../utils/getValidData.js";
+import getDataSize from "../../utils/getDataSize.js";
+import dataTypes from "../../utils/handlers/dataTypes.js";
+import getMimeType from "../../mime/index.js";
+import getS3FilePath from "./utils/getS3FilePath.js";
+const uploadFile = (s3Settings) => async (fp, data) => {
+    const filepath = getS3FilePath(fp, s3Settings);
+    const validData = await getValidData({ data, types: [dataTypes.stream] });
+    const size = await getDataSize({ data });
+    const type = getMimeType(filepath);
+    try {
+        const s3Client = client.getS3Client(s3Settings);
+        const bucketName = s3Settings.containerName;
+        const bucketParams = {
+            Bucket: bucketName,
+            Key: filepath,
+            ContentLength: size,
+            ContentType: type,
+            Body: validData,
+        };
+        const parallelUpload = new Upload({
+            client: s3Client,
+            params: bucketParams,
+        });
+        const res = await parallelUpload.done();
+        // const res = await s3Client.send(new PutObjectCommand(bucketParams));
+        return res; // For unit tests.
+    }
+    catch (err) {
+        throw new Error(`Upload error s3: ${err}. Message: ${err.message}. Keys: ${Object.keys(err)}. Filepath: ${filepath}. Data type: ${typeof validData}.`);
+    }
+};
+export default uploadFile;

package/dist/server/plugins/file/providers/s3/funcs/utils/getS3FilePath.js

@@ -0,0 +1,18 @@
+import path from "node:path";
+import config from "../../../../../../../config.js";
+function getPath(fp, s3Settings) {
+    const filepath = fp.replace(/\\/g, "/");
+    if (s3Settings.folder && !filepath.includes(s3Settings.folder)) {
+        return path.join(s3Settings.folder, fp.startsWith("files") ? "" : "files", filepath);
+    }
+    return filepath;
+}
+function getS3FilePath(fp, s3Settings) {
+    const filepath = getPath(fp, s3Settings);
+    const unixPath = filepath.replace(/\\/g, "/").split("work/").pop();
+    const prefix = !filepath.includes(config.folder) ? config.folder : "";
+    const sufix = unixPath[0] === "/" ? "" : "/";
+    const s3FilePath = prefix + sufix + unixPath;
+    return s3FilePath;
+}
+export default getS3FilePath;

package/{server → dist/server}/plugins/file/providers/s3/index.js

@@ -1,12 +1,11 @@
-import downloadFile from './funcs/downloadFile.js';
-import fileExists from './funcs/fileExists.js';
-import uploadFile from './funcs/uploadFile.js';
-
-export default function s3Storage(opt) {
-  return {
-    name: 's3',
-    downloadFile: downloadFile(opt),
-    uploadFile: uploadFile(opt),
-    fileExists: fileExists(opt),
-  };
-}
+import downloadFile from './funcs/downloadFile.js';
+import fileExists from './funcs/fileExists.js';
+import uploadFile from './funcs/uploadFile.js';
+export default function s3Storage(opt) {
+    return {
+        name: 's3',
+        downloadFile: downloadFile(opt),
+        uploadFile: uploadFile(opt),
+        fileExists: fileExists(opt),
+    };
+}

package/dist/server/plugins/file/providers/utils/getDataSize.js

@@ -0,0 +1,19 @@
+import isText from "./typeguards/isText.js";
+import isPath from "./typeguards/isPath.js";
+import isBuffer from "./typeguards/isBuffer.js";
+import isReadableStream from "./typeguards/isReadableStream.js";
+import isArray from "./typeguards/isArray.js";
+import sizeHandlers from "./handlers/sizeHandlers.js";
+const getDataSizeInBytes = async ({ data }) => {
+    const validators = [isReadableStream, isBuffer, isPath, isText, isArray];
+    const values = await Promise.all(validators.map(async (func) => ({
+        name: func.name,
+        bool: await func(data),
+    })));
+    const [value] = values.filter(({ bool }) => bool === true);
+    if (!value)
+        throw new Error("No support for this data type");
+    const handler = sizeHandlers[value.name];
+    return handler({ data });
+};
+export default getDataSizeInBytes;

package/dist/server/plugins/file/providers/utils/getValidData.js

@@ -0,0 +1,30 @@
+import isText from "./typeguards/isText.js";
+import isPath from "./typeguards/isPath.js";
+import isBuffer from "./typeguards/isBuffer.js";
+import isReadableStream from "./typeguards/isReadableStream.js";
+import isArray from "./typeguards/isArray.js";
+import convertHandlers from "./handlers/index.js";
+const getValidData = async ({ data, types }) => {
+    // chain of responsibility
+    const validators = [isReadableStream, isBuffer, isPath, isText, isArray];
+    const values = await Promise.all(validators.map(async (func) => ({
+        name: func.name,
+        bool: await func(data),
+    })));
+    const isValid = !values.reduce((acc, { name, bool }) => {
+        let valid = false;
+        types.forEach((dataType) => {
+            if (name === dataType)
+                valid = bool;
+        });
+        return acc && !valid;
+    }, true);
+    if (isValid)
+        return data;
+    const [value] = values.filter(({ bool }) => bool === true);
+    if (!value)
+        throw new Error("No support for this data type");
+    const handler = convertHandlers[value.name];
+    return handler({ data, types });
+};
+export default getValidData;

package/dist/server/plugins/file/providers/utils/handlers/dataTypes.js

@@ -0,0 +1,7 @@
+const types = {
+    buffer: 'isBuffer',
+    stream: 'isReadableStream',
+    path: 'isPath',
+    text: 'isText',
+};
+export default types;

package/dist/server/plugins/file/providers/utils/handlers/index.js

@@ -0,0 +1,50 @@
+import { readFile } from "fs/promises";
+import { createReadStream } from "fs";
+import { Readable } from "stream";
+import dataTypes from "./dataTypes.js";
+const handlers = {
+    isReadableStream: () => "Unsupported",
+    isBuffer: ({ data, types }) => {
+        if (types.includes(dataTypes.stream)) {
+            return Readable.from(data);
+        }
+        return null;
+    },
+    isArray: ({ data, types }) => {
+        if (types.includes(dataTypes.stream)) {
+            return Readable.from(Buffer.from(data));
+        }
+        if (types.includes(dataTypes.buffer)) {
+            return Buffer.from(data, "utf-8");
+        }
+        if (types.includes(dataTypes.path)) {
+            return "Unsupported";
+        }
+        return null;
+    },
+    isPath: async ({ data, types }) => {
+        if (types.includes(dataTypes.stream)) {
+            return createReadStream(data);
+        }
+        if (types.includes(dataTypes.buffer)) {
+            return readFile(data);
+        }
+        if (types.includes(dataTypes.text)) {
+            return (await readFile(data)).toString();
+        }
+        return null;
+    },
+    isText: async ({ data, types }) => {
+        if (types.includes(dataTypes.stream)) {
+            return Readable.from(Buffer.from(data));
+        }
+        if (types.includes(dataTypes.buffer)) {
+            return Buffer.from(data, "utf-8");
+        }
+        if (types.includes(dataTypes.path)) {
+            return "Unsupported";
+        }
+        return null;
+    },
+};
+export default handlers;

package/dist/server/plugins/file/providers/utils/handlers/sizeHandlers.js

@@ -0,0 +1,9 @@
+import { stat } from "fs/promises";
+const handlers = {
+    isReadableStream: () => "Unsupported",
+    isBuffer: ({ data }) => data.length,
+    isArray: ({ data }) => Buffer.byteLength(data),
+    isPath: async ({ data }) => (await stat(data)).size,
+    isText: ({ data }) => Buffer.byteLength(data),
+};
+export default handlers;

package/dist/server/plugins/file/providers/utils/streamToBuffer.js

@@ -0,0 +1,7 @@
+const streamToBuffer = (stream) => new Promise((res, rej) => {
+    const buf = [];
+    stream.on('data', (data) => buf.push(data));
+    stream.on('end', () => res(Buffer.concat(buf)));
+    stream.on('error', (err) => rej(err));
+});
+export default streamToBuffer;

package/{server → dist/server}/plugins/file/providers/utils/typeguards/isArray.js

@@ -1,3 +1,2 @@
-const isArray = (array) => Array.isArray(array);
-
-export default isArray;
+const isArray = (array) => Array.isArray(array);
+export default isArray;

package/{server → dist/server}/plugins/file/providers/utils/typeguards/isBuffer.js

@@ -1,3 +1,2 @@
-const isBuffer = (buffer) => Buffer.isBuffer(buffer);
-
-export default isBuffer;
+const isBuffer = (buffer) => Buffer.isBuffer(buffer);
+export default isBuffer;

package/dist/server/plugins/file/providers/utils/typeguards/isPath.js

@@ -0,0 +1,3 @@
+import fileExists from "../../../utils/isFileExists.js";
+const isPath = async (path) => fileExists(path, { check: true });
+export default isPath;

package/dist/server/plugins/file/providers/utils/typeguards/isReadableStream.js

@@ -0,0 +1,7 @@
+/* eslint-disable no-underscore-dangle */
+const isReadableStream = (stream) => stream !== null
+    && typeof stream === 'object'
+    && typeof stream.pipe === 'function'
+    && typeof stream._read === 'function'
+    && typeof stream._readableState === 'object';
+export default isReadableStream;

package/{server → dist/server}/plugins/file/providers/utils/typeguards/isText.js

@@ -1,3 +1,2 @@
-const isText = (text) => text !== null && typeof text === 'string';
-
-export default isText;
+const isText = (text) => text !== null && typeof text === 'string';
+export default isText;

package/dist/server/plugins/file/uploadFile.js

@@ -0,0 +1,14 @@
+import path from "path";
+import providers from "./providers/index.js";
+async function uploadFile(filePath, data, options = {}) {
+    const filename = path.basename(filePath);
+    // prefix
+    const prefix = (options.prefix === "date"
+        ? new Date().toISOString().split("T")[0]
+        : null) || (options.prefix === "3s" ? filename.substring(0, 3) : "");
+    const relativePath = path.join(path.dirname(filePath), prefix, filename);
+    const fp = providers(options);
+    await fp.uploadFile(relativePath, data, options);
+    return relativePath;
+}
+export default uploadFile;

package/dist/server/plugins/file/uploadMultiPart.js

@@ -0,0 +1,101 @@
+/* eslint-disable no-console */
+import path from "node:path";
+import { mkdir, writeFile } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import { imageSize } from "image-size";
+import { existsSync } from "node:fs";
+import config from "../../../config.js";
+import providers from "./providers/index.js";
+import { all, images } from "./utils/allowedExtensions.js";
+import grpc from "../grpc/grpc.js";
+import getFileType from "./utils/getFileType.js";
+const { resizeImage } = grpc();
+const { resizeImageMinSize = 5 } = config; // resize images >= 5 MB by default
+async function writeFileToDisk(file, buffer) {
+    if (!file?.filepath || !file.extension || !buffer) {
+        return null;
+    }
+    // resize big images
+    if (images.find((el) => el === file.extension) &&
+        file.size >= resizeImageMinSize * 1024 * 1024) {
+        const { width = 320, height = 240 } = imageSize(buffer) || {};
+        const ratio = width / height;
+        const resizeWidth = Math.min(width, 2048);
+        const resizeHeight = resizeWidth / ratio;
+        const { result } = await resizeImage({
+            base64: buffer.toString("base64"),
+            width: resizeWidth,
+            height: resizeHeight,
+            quality: 75,
+        });
+        await writeFile(`${file.filepath.replace(`.${file.extension}`, `_original.${file.extension}`)}`, buffer);
+        await writeFile(file.filepath, Buffer.from(result, "base64"));
+        return null;
+    }
+    await writeFile(file.filepath, buffer);
+    return null;
+}
+export default async function uploadMultiPart(req, { extensions, raw, subdir, originalFilename = false } = {}) {
+    const allowedExtensions = req.routeOptions?.url === "/file/upload-image/*" ? images : all;
+    const parts = req.parts();
+    // const part = await parts.next();
+    const { value: part, done } = await parts.next();
+    if (done) {
+        throw new Error("parts already processed");
+    }
+    const value = part?.fields?.file || part;
+    if (!value?.file?.on || !value?.filename) {
+        throw new Error("upload error");
+    }
+    const ext = path.extname(value.filename).toLowerCase();
+    if (Array.isArray(extensions) && !extensions.includes(ext)) {
+        throw new Error("file extension is not allowed");
+    }
+    // check extension
+    if (!allowedExtensions.includes(ext.substring(1))) {
+        throw new Error("file extension is not allowed");
+    }
+    const fileBuffer = await new Promise((res, rej) => {
+        const chunks = [];
+        value.file.on("data", (chunk) => chunks.push(chunk));
+        value.file.on("end", () => res(Buffer.concat(chunks)));
+        value.file.on("error", rej);
+    });
+    if (!fileBuffer?.length) {
+        throw new Error("file buffer is empty");
+    }
+    if (raw)
+        return { buffer: fileBuffer, filename: value.filename, ext }; // return buffer for custom upload
+    const dir = subdir != null ? subdir : req.params?.["*"] || "uploads";
+    const yearMonthDay = subdir != null ? "" : new Date().toISOString().split("T")[0];
+    const dbname = req.pg?.options?.database || req.pg?.database || config.pg?.database; // request / config params / default config params
+    const rootDir = config.root || `/data/local/${dbname || ""}`;
+    const reldirpath = path.join("/files", dir, yearMonthDay);
+    const folder = path.join(rootDir, config.folder || "", reldirpath);
+    const newFilename = originalFilename
+        ? value.filename
+        : `${randomUUID()}${ext}`;
+    if (existsSync(path.join(folder, newFilename).replace(/\\/g, "/"))) {
+        throw new Error("file with specified name already exists in directory");
+    }
+    const file = {
+        originalFilename: value.filename,
+        newFilename,
+        filepath: path.join(folder, newFilename).replace(/\\/g, "/"),
+        filetype: getFileType(newFilename),
+        relativeFilepath: path.join(reldirpath, newFilename).replace(/\\/g, "/"),
+        size: Buffer.byteLength(fileBuffer),
+        mimetype: value.mimetype,
+        extension: ext.substring(1),
+    };
+    await mkdir(folder, { recursive: true });
+    await writeFileToDisk(file, fileBuffer);
+    // move file to s3
+    if (config.s3?.endpoint) {
+        const s3 = providers();
+        await s3.uploadFile(file.relativeFilepath, fileBuffer);
+        if (config.trace)
+            console.log("upload to s3", file.relativeFilepath);
+    }
+    return file;
+}

package/dist/server/plugins/file/utils/allowedExtensions.js

@@ -0,0 +1,60 @@
+const all = [
+    "jpg",
+    "jpeg",
+    "png",
+    "json",
+    "geojson",
+    "pdf",
+    "doc",
+    "docx",
+    "xks",
+    "csv",
+    "xls",
+    "xlsx",
+    "txt",
+    "log",
+    "mp4",
+    "mov",
+    "avi",
+    "zip",
+    "rar",
+    "7z",
+    "gz",
+    "geotiff",
+    "tiff",
+    "tif",
+    "svg",
+    "p7s",
+    "asice",
+    "shp",
+    "xml",
+    "gpkg",
+    "dmf",
+    "dwg",
+    "ico",
+    "ppt",
+    "pptx",
+];
+const images = ["jpg", "jpeg", "png", "tiff", "tif", "ico", "svg"];
+const videos = ["mp4", "mov", "avi"];
+const documents = [
+    "pdf",
+    "doc",
+    "docx",
+    "xks",
+    "csv",
+    "xls",
+    "xlsx",
+    "txt",
+    "log",
+    "xml",
+    "ppt",
+    "pptx",
+];
+export { all, images, videos, documents };
+export default {
+    all,
+    images,
+    videos,
+    documents,
+};

package/{server → dist/server}/plugins/file/utils/getFileType.js

@@ -1,10 +1,12 @@
-import path from 'node:path';
-import { images, videos, documents } from './allowedExtensions.js';
-
-export default function getFileType(filename) {
-    const ext = path.extname(filename).toLocaleLowerCase().slice(1);
-    if (images.includes(ext)) return 'image';
-    if (videos.includes(ext)) return 'video';
-    if (documents.includes(ext)) return 'document';
-    return 'other';
-}
+import path from 'node:path';
+import { images, videos, documents } from './allowedExtensions.js';
+export default function getFileType(filename) {
+    const ext = path.extname(filename).toLocaleLowerCase().slice(1);
+    if (images.includes(ext))
+        return 'image';
+    if (videos.includes(ext))
+        return 'video';
+    if (documents.includes(ext))
+        return 'document';
+    return 'other';
+}

package/dist/server/plugins/file/utils/getPath.js

@@ -0,0 +1,30 @@
+import path from "node:path";
+import config from "../../../../config.js";
+function getPath(filepath, options = {}) {
+    // used at getValidData
+    if (typeof filepath !== "string" && options?.check) {
+        return null;
+    }
+    if (typeof filepath !== "string") {
+        throw new Error("invalid filepath type");
+    }
+    // full path support + windows drive
+    const relpath = filepath.replace(/\\/g, "/");
+    const isFull = relpath.substring(0, 14).includes("/data/local/") ||
+        Object.values(config.fileStorage || {}).find((el) => relpath.includes(el));
+    if (isFull)
+        return relpath;
+    const rootDir = config.root || `/data/local/${options.database || config.pg?.database}`;
+    const sufix = ["files", "/files"].find((el) => relpath.startsWith(el))
+        ? ""
+        : "files";
+    const prefix = !relpath.toLowerCase().includes(rootDir.toLowerCase())
+        ? rootDir
+        : "";
+    if (config.folder &&
+        !relpath.toLowerCase().includes(config.folder.toLowerCase())) {
+        return path.join(prefix, config.folder, sufix, relpath);
+    }
+    return path.join(prefix, relpath);
+}
+export default getPath;

package/dist/server/plugins/file/utils/isFileExists.js

@@ -0,0 +1,15 @@
+import { access } from "node:fs/promises";
+import getPath from "./getPath.js";
+const isFileExists = async (filepath, options = {}) => {
+    const fullPath = getPath(filepath, options);
+    if (!fullPath)
+        return false;
+    try {
+        await access(fullPath);
+        return true;
+    }
+    catch (err) {
+        return false;
+    }
+};
+export default isFileExists;

package/dist/server/plugins/grpc/file2json.js

@@ -0,0 +1,50 @@
+import path from "node:path";
+import downloadFile from "../file/downloadFile.js";
+import getGRPC from "./grpc.js";
+const grpc = getGRPC();
+const file2json = async ({ buffer: buffer1, ext: ext1, filepath = "", encoding: encoding1 = "UTF-8", debug, }) => {
+    if (!filepath && !(buffer1?.length && ext1))
+        return null;
+    const ext = ext1 || path.extname(filepath || "");
+    const encoding = { CP1251: "windows-1251" }[encoding1] || encoding1;
+    const buffer = buffer1 ||
+        (await downloadFile(filepath, { provider: "fs", buffer: true, debug })) ||
+        "{}";
+    if (debug)
+        return { filepath, buffer };
+    if ([".xls", ".xlsx"].includes(ext)) {
+        const { result } = await grpc.excelToJson({
+            base64: buffer.toString("base64"),
+            type: "xls",
+        });
+        const json = typeof result === "object" ? result : JSON.parse(result || {});
+        return json;
+    }
+    if ([".csv", ".txt"].includes(ext)) {
+        const { result } = await grpc.excelToJson({
+            base64: buffer.toString("base64"),
+            type: "csv",
+            encoding,
+        });
+        const json = typeof result === "object" ? result : JSON.parse(result || {});
+        return json;
+    }
+    if ([".zip"].includes(ext)) {
+        const { result } = await grpc.shpToGeojson({
+            base64: buffer.toString("base64"),
+            encoding,
+        });
+        const json = typeof result === "object" ? result : JSON.parse(result);
+        return json;
+    }
+    if ([".xml"].includes(ext)) {
+        const { result } = await grpc.xmlToJson({ xml: buffer.toString("utf-8") });
+        const json = typeof result === "object" ? result : JSON.parse(result);
+        return json;
+    }
+    if ([".geojson", ".json"].includes(ext)) {
+        return JSON.parse(buffer.toString("utf-8"));
+    }
+    return filepath.replace(/"/g, "");
+};
+export default file2json;