@opengis/fastify-table 2.3.0 → 2.3.2

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AA4FA,iBAAe,MAAM,CAAC,OAAO,EAAE,GAAG,iBAgLjC;;AACD,wBAEG"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AA6FA,iBAAe,MAAM,CAAC,OAAO,EAAE,GAAG,iBAiLjC;;AACD,wBAEG"}

package/dist/index.js

@@ -34,6 +34,7 @@ import metricPlugin from "./server/plugins/metric/index.js";
 import redisPlugin from "./server/plugins/redis/index.js";
 import loggerPlugin from "./server/plugins/logger/index.js";
 import authPlugin from "./server/plugins/auth/index.js";
+import chunkedUploadPlugin from "./server/plugins/upload/index.js";
 // utils
 import execMigrations from "./server/plugins/migration/exec.migrations.js";
 import pgClients from "./server/plugins/pg/pgClients.js";
@@ -113,6 +114,7 @@ async function plugin(fastify) {
     execMigrations(path.join(cwd, "server/migrations"), pgClients.client, true).catch((err) => console.warn(err.toString()));
     // plugins / utils / funcs
     authPlugin(fastify); // fastify-auth api + hooks integrated to core
+    chunkedUploadPlugin(fastify);
     policyPlugin(fastify);
     metricPlugin();
     redisPlugin(fastify);

package/dist/server/plugins/upload/index.d.ts

@@ -1,6 +1,9 @@
+import { FastifyInstance } from "fastify";
 export declare const fetchTimeoutMs: number;
 export declare const prefix = "file/upload2";
 export declare const uploadChunkDirectory = "/files/uploads";
 export declare const fileDir: string;
 export declare const metaDir: string;
+declare function chunkedUploadPlugin(app: FastifyInstance): void;
+export default chunkedUploadPlugin;
 //# sourceMappingURL=index.d.ts.map

package/dist/server/plugins/upload/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/index.ts"],"names":[],"mappings":"AASA,eAAO,MAAM,cAAc,QAE1B,CAAC;AAEF,eAAO,MAAM,MAAM,iBAAiB,CAAC;AAErC,eAAO,MAAM,oBAAoB,mBAAmB,CAAC;AAErD,eAAO,MAAM,OAAO,QAEE,CAAC;AAEvB,eAAO,MAAM,OAAO,QAA4B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAW1C,eAAO,MAAM,cAAc,QAE1B,CAAC;AAEF,eAAO,MAAM,MAAM,iBAAiB,CAAC;AAErC,eAAO,MAAM,oBAAoB,mBAAmB,CAAC;AAErD,eAAO,MAAM,OAAO,QAEE,CAAC;AAEvB,eAAO,MAAM,OAAO,QAA4B,CAAC;AAKjD,iBAAS,mBAAmB,CAAC,GAAG,EAAE,eAAe,QAIhD;AAED,eAAe,mBAAmB,CAAC"}

package/dist/server/plugins/upload/index.js

@@ -10,3 +10,11 @@ export const fileDir = path
     .join(rootDir, uploadChunkDirectory)
     .replace(/\\/g, "/");
 export const metaDir = path.join(fileDir, "tmp");
+import { cleanupNonRedisUploads } from "./s3.js";
+// w/ out redis - chunked uploads are non-resumable in order to avoid unclosed multiparts (non-discoverable via s3 commands with current version of minio)
+function chunkedUploadPlugin(app) {
+    app.addHook("onClose", async () => {
+        await cleanupNonRedisUploads();
+    });
+}
+export default chunkedUploadPlugin;

package/dist/server/plugins/upload/s3.d.ts

@@ -1,3 +1,10 @@
+export declare function cleanupNonRedisUploads(s3?: any): Promise<void>;
+export declare function saveUploadMeta(id: string, meta: any, ttl?: number): Promise<void>;
+export declare function refreshUploadTTL(id: string, ttl?: number, s3?: any): Promise<void>;
+export declare function deleteUploadMeta(id: string, key?: string): Promise<void>;
+export declare function getUploadMeta(id: string): Promise<any>;
+export declare function getResumableUploadId(key: string): Promise<any>;
+export declare function cleanupStaleUploads(s3?: any): Promise<void>;
 export declare function findUpload({ id }: {
     id: string;
 }, s3?: any): Promise<{
@@ -9,16 +16,13 @@ export declare function findUpload({ id }: {
     uploaded: any;
     parts: any;
 }>;
-export declare function listUploads({ Bucket }: {
-    Bucket: string;
-}, s3?: any): Promise<any>;
 export declare function startUpload({ Bucket, Key, ContentType, fileSize, }: {
     Bucket: string;
     Key: string;
     ContentType: string;
     fileSize: number;
 }, s3?: any): Promise<{
-    Key: any;
+    Key: string;
     Bucket: any;
     UploadId: any;
     fileSize: any;
@@ -50,7 +54,14 @@ export declare function uploadChunk({ id, chunk, partNumber, }: {
 export declare function finishUpload({ id, }: {
     id: string;
 }, s3?: any): Promise<{
+    finished: boolean;
     uploaded: any;
+    filepath: any;
+}>;
+export declare function abortUpload({ id, }: {
+    id: string;
+}, s3?: any): Promise<{
+    aborted: boolean;
 }>;
 export declare function getUploadStatus({ id, Key, Bucket, }: {
     id: string;

package/dist/server/plugins/upload/s3.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/s3.ts"],"names":[],"mappings":"AAiBA,wBAAsB,UAAU,CAAC,EAAE,EAAE,EAAE,EAAE;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,EAAE,EAAE,MAAW;;;;;;;;GAkCrE;AAED,wBAAsB,WAAW,CAC/B,EAAE,MAAM,EAAE,EAAE;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,EAC9B,EAAE,MAAW,gBAed;AAED,wBAAsB,WAAW,CAC/B,EACE,MAAM,EACN,GAAG,EACH,WAAW,EACX,QAAQ,GACT,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB,EACD,EAAE,MAAW;;;;;;;;;;;;;;;;GAoDd;AAED,wBAAsB,WAAW,CAC/B,EACE,EAAE,EACF,KAAK,EACL,UAAU,GACX,EAAE;IACD,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,GAAG,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;CACpB,EACD,EAAE,MAAW;;;;;;;;GAuBd;AAED,wBAAsB,YAAY,CAChC,EACE,EAAE,GACH,EAAE;IACD,EAAE,EAAE,MAAM,CAAC;CACZ,EACD,EAAE,MAAW;;GAmCd;AAED,wBAAsB,eAAe,CACnC,EACE,EAAE,EACF,GAAG,EACH,MAAM,GACP,EAAE;IACD,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,EACD,EAAE,MAAW;;;;;;;;;;;;;;GAgEd"}
+{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/s3.ts"],"names":[],"mappings":"AAiGA,wBAAsB,sBAAsB,CAAC,EAAE,MAAW,iBAwCzD;AAED,wBAAsB,cAAc,CAClC,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,GAAG,EACT,GAAG,SAAqB,iBAazB;AAED,wBAAsB,gBAAgB,CACpC,EAAE,EAAE,MAAM,EACV,GAAG,SAAqB,EACxB,EAAE,MAAW,iBAsBd;AAED,wBAAsB,gBAAgB,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,iBA0B9D;AAED,wBAAsB,aAAa,CAAC,EAAE,EAAE,MAAM,gBAQ7C;AAED,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,MAAM,gBAMrD;AAED,wBAAsB,mBAAmB,CAAC,EAAE,MAAW,iBAyEtD;AAED,wBAAsB,UAAU,CAAC,EAAE,EAAE,EAAE,EAAE;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,EAAE,EAAE,MAAW;;;;;;;;GAsCrE;AAKD,wBAAsB,WAAW,CAC/B,EACE,MAAM,EACN,GAAG,EACH,WAAW,EACX,QAAQ,GACT,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB,EACD,EAAE,MAAW;;;;;;;;;;;;;;;;GAsDd;AAED,wBAAsB,WAAW,CAC/B,EACE,EAAE,EACF,KAAK,EACL,UAAU,GACX,EAAE;IACD,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,GAAG,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;CACpB,EACD,EAAE,MAAW;;;;;;;;GA0Bd;AAED,wBAAsB,YAAY,CAChC,EACE,EAAE,GACH,EAAE;IACD,EAAE,EAAE,MAAM,CAAC;CACZ,EACD,EAAE,MAAW;;;;GAuCd;AAED,wBAAsB,WAAW,CAC/B,EACE,EAAE,GACH,EAAE;IACD,EAAE,EAAE,MAAM,CAAC;CACZ,EACD,EAAE,MAAW;;GAwBd;AAED,wBAAsB,eAAe,CACnC,EACE,EAAE,EACF,GAAG,EACH,MAAM,GACP,EAAE;IACD,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,EACD,EAAE,MAAW;;;;;;;;;;;;;;GAoEd"}

package/dist/server/plugins/upload/s3.js

@@ -1,57 +1,272 @@
 import path from "node:path";
-import { CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, ListMultipartUploadsCommand, HeadObjectCommand, ListPartsCommand, } from "@aws-sdk/client-s3";
+import { CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, HeadObjectCommand, ListPartsCommand, AbortMultipartUploadCommand, } from "@aws-sdk/client-s3";
 import config from "../../../config.js";
 import s3Client from "../file/providers/s3/client.js";
 import rclient from "../redis/client.js";
+import logger from "../logger/getLogger.js";
 const uploads = new Map();
+const resumableUploads = new Map();
+// non-redis upload timeout handlers
+const uploadTimeouts = new Map();
+const UPLOAD_TTL_SECONDS = 60 * 60;
+// expire gracefully to give cleanup worker time to abort stale uploads
+const CLEANUP_GRACE_SECONDS = 60 * 60 * 24;
+const REDIS_TTL = UPLOAD_TTL_SECONDS + CLEANUP_GRACE_SECONDS;
+const uploadMetaKey = (id) => `upload:${id}:meta`;
+const uploadExpireKey = (id) => `upload:${id}:expires`;
+const resumableKey = (key) => `resumable:${key}`;
+function clearUploadTimeout(id) {
+    const timeout = uploadTimeouts.get(id);
+    if (timeout) {
+        clearTimeout(timeout);
+        uploadTimeouts.delete(id);
+    }
+}
+async function scheduleNonRedisCleanup(id, s3 = s3Client) {
+    if (!s3)
+        throw new Error("No S3 client");
+    if (config.redis)
+        return;
+    clearUploadTimeout(id);
+    const timeout = setTimeout(async () => {
+        try {
+            const meta = uploads.get(id);
+            if (!meta)
+                return;
+            try {
+                await s3.send(new AbortMultipartUploadCommand({
+                    Bucket: meta.Bucket,
+                    Key: meta.Key,
+                    UploadId: id,
+                }));
+            }
+            catch (err) {
+                console.error("AbortMultipartUpload failed", {
+                    uploadId: id,
+                    Bucket: meta.Bucket,
+                    Key: meta.Key,
+                    error: err?.message,
+                });
+                logger.file("upload/abort/error", {
+                    uploadId: id,
+                    Bucket: meta.Bucket,
+                    Key: meta.Key,
+                    error: err?.message,
+                });
+            }
+            uploads.delete(id);
+            if (resumableUploads.get(meta.Key) === id) {
+                resumableUploads.delete(meta.Key);
+            }
+        }
+        catch (err) {
+            console.error(err);
+            logger.error(err);
+        }
+        finally {
+            clearUploadTimeout(id);
+        }
+    }, UPLOAD_TTL_SECONDS * 1000);
+    // don't keep node process alive
+    timeout.unref?.();
+    uploadTimeouts.set(id, timeout);
+}
+// used via onClose hook to avoid memory leaks
+export async function cleanupNonRedisUploads(s3 = s3Client) {
+    if (!s3)
+        throw new Error("No S3 client");
+    if (config.redis)
+        return;
+    console.log("cleanupNonRedisUploads start");
+    const entries = [...uploads.entries()];
+    for (const [id, meta] of entries) {
+        try {
+            await s3.send(new AbortMultipartUploadCommand({
+                Bucket: meta.Bucket,
+                Key: meta.Key,
+                UploadId: id,
+            }));
+        }
+        catch (err) {
+            console.error("AbortMultipartUpload failed", {
+                uploadId: id,
+                Bucket: meta.Bucket,
+                Key: meta.Key,
+                error: err?.message,
+            });
+            logger.file("upload/abort/error", {
+                uploadId: id,
+                Bucket: meta.Bucket,
+                Key: meta.Key,
+                error: err?.message,
+            });
+        }
+        clearUploadTimeout(id);
+        uploads.delete(id);
+        if (resumableUploads.get(meta.Key) === id) {
+            resumableUploads.delete(meta.Key);
+        }
+    }
+    console.log("open unfinished multipart uploads closed", entries.length);
+}
+export async function saveUploadMeta(id, meta, ttl = UPLOAD_TTL_SECONDS) {
+    if (!config.redis)
+        return;
+    const expiresAt = Date.now() + UPLOAD_TTL_SECONDS * 1000;
+    await Promise.all([
+        rclient.set(uploadMetaKey(id), JSON.stringify(meta), "EX", REDIS_TTL),
+        rclient.set(uploadExpireKey(id), String(expiresAt), "EX", ttl),
+        rclient.set(resumableKey(meta.Key), id, "EX", ttl),
+    ]);
+}
+export async function refreshUploadTTL(id, ttl = UPLOAD_TTL_SECONDS, s3 = s3Client) {
+    if (!s3)
+        throw new Error("No S3 client");
+    if (!config.redis) {
+        await scheduleNonRedisCleanup(id, s3);
+        return;
+    }
+    const meta = await getUploadMeta(id);
+    if (!meta)
+        return;
+    const expiresAt = Date.now() + ttl * 1000;
+    await Promise.all([
+        rclient.expire(uploadMetaKey(id), ttl),
+        rclient.set(uploadExpireKey(id), String(expiresAt), "EX", ttl),
+        rclient.expire(resumableKey(meta.Key), ttl),
+    ]);
+}
+export async function deleteUploadMeta(id, key) {
+    clearUploadTimeout(id);
+    if (!config.redis) {
+        const meta = uploads.get(id);
+        uploads.delete(id);
+        const redisKey = key || meta?.Key;
+        if (redisKey && resumableUploads.get(redisKey) === id) {
+            resumableUploads.delete(redisKey);
+        }
+        return;
+    }
+    const redisKey = key || (await getUploadMeta(id))?.Key;
+    const keys = [uploadMetaKey(id), uploadExpireKey(id)];
+    if (redisKey) {
+        keys.push(resumableKey(redisKey));
+    }
+    await rclient.del(keys);
+}
+export async function getUploadMeta(id) {
+    if (!config.redis) {
+        return uploads.get(id);
+    }
+    const raw = await rclient.get(uploadMetaKey(id));
+    return raw ? JSON.parse(raw) : undefined;
+}
+export async function getResumableUploadId(key) {
+    if (!config.redis) {
+        return resumableUploads.get(key);
+    }
+    return rclient.get(resumableKey(key));
+}
+export async function cleanupStaleUploads(s3 = s3Client) {
+    if (!s3)
+        throw new Error("No S3 client");
+    if (!config.redis) {
+        await cleanupNonRedisUploads(s3);
+        return;
+    }
+    let cursor = "0";
+    do {
+        const [nextCursor, keys] = await rclient.scan(cursor, "MATCH", "upload:*:expires", "COUNT", 100);
+        cursor = nextCursor;
+        for (const expireKey of keys) {
+            try {
+                const expiresAt = Number(await rclient.get(expireKey));
+                if (!expiresAt || expiresAt > Date.now()) {
+                    continue;
+                }
+                const uploadId = expireKey
+                    .replace("upload:", "")
+                    .replace(":expires", "");
+                const meta = await getUploadMeta(uploadId);
+                if (!meta) {
+                    await rclient.del(expireKey);
+                    continue;
+                }
+                const { Bucket, Key } = meta;
+                try {
+                    await s3.send(new AbortMultipartUploadCommand({
+                        Bucket,
+                        Key,
+                        UploadId: uploadId,
+                    }));
+                }
+                catch (err) {
+                    console.error("AbortMultipartUpload failed", {
+                        uploadId,
+                        Bucket,
+                        Key,
+                        error: err?.message,
+                    });
+                    logger.file("upload/abort/error", {
+                        uploadId,
+                        Bucket,
+                        Key,
+                        error: err?.message,
+                    });
+                }
+                await deleteUploadMeta(uploadId, Key);
+            }
+            catch (err) {
+                console.error(err);
+                logger.error(err);
+            }
+        }
+    } while (cursor !== "0");
+}
 export async function findUpload({ id }, s3 = s3Client) {
-    const cachedInfo = config.redis ? await rclient.get(id) : undefined;
-    const { Key, Bucket, fileSize: cacheSize, ContentType: cacheContentType, } = cachedInfo ? JSON.parse(cachedInfo) : uploads.get(id) || {};
+    if (!s3)
+        throw new Error("No S3 client");
+    const cachedInfo = await getUploadMeta(id);
+    const { Key, Bucket, fileSize: cacheSize, ContentType: cacheContentType, } = cachedInfo || {};
     const parts = await s3
         .send(new ListPartsCommand({
         Bucket,
         Key,
         UploadId: id,
     }))
-        .catch((err) => null);
+        .catch(() => null);
     return {
         UploadId: parts ? id : undefined,
         Key,
         Bucket,
         fileSize: cacheSize,
         ContentType: cacheContentType,
-        uploaded: parts.Parts?.reduce((sum, part) => sum + (part.Size || 0), 0) ||
-            0,
-        parts: parts.Parts?.map((p) => ({
+        uploaded: parts?.Parts?.reduce((sum, part) => sum + (part.Size || 0), 0) || 0,
+        parts: parts?.Parts?.map((p) => ({
             ETag: p.ETag,
             PartNumber: p.PartNumber,
         })) || [],
     };
 }
-export async function listUploads({ Bucket }, s3 = s3Client) {
-    const res = await s3.send(new ListMultipartUploadsCommand({ Bucket }));
-    if (!res.Uploads) {
-        return [];
-    }
-    return res.Uploads.map((upload) => ({
-        key: upload.Key,
-        uploadId: upload.UploadId,
-        initiated: upload.Initiated,
-        owner: upload.Owner?.DisplayName,
-        storageClass: upload.StorageClass,
-    }));
-}
+// todo: optional - keep originalFileName, else - id + extension
+//  + implement rotateFilenameS3 logic, like const filename = await rotateFilenameS3({ Name: originalFilename, Bucket, Prefix }, s3Client);
+// ! if originalFileName - check names with spaces - s3 mights throw error - encode request or at upload?
 export async function startUpload({ Bucket, Key, ContentType, fileSize, }, s3 = s3Client) {
-    const cachedInfo = config.redis ? await rclient.get(Key) : undefined;
-    const { Key: cacheKey, Bucket: cacheBucket, UploadId, fileSize: cacheSize, extension: cacheExtension, ContentType: cacheContentType, } = cachedInfo ? JSON.parse(cachedInfo) : uploads.get(Key) || {};
-    if (UploadId) {
+    if (!s3)
+        throw new Error("No S3 client");
+    const cachedUploadId = await getResumableUploadId(Key);
+    const cachedMeta = cachedUploadId
+        ? await getUploadMeta(cachedUploadId)
+        : undefined;
+    if (cachedUploadId && cachedMeta) {
+        await refreshUploadTTL(cachedUploadId, UPLOAD_TTL_SECONDS, s3);
         return {
-            Key: cacheKey,
-            Bucket: cacheBucket,
-            UploadId,
-            fileSize: cacheSize,
-            extension: cacheExtension,
-            ContentType: cacheContentType,
+            Key,
+            Bucket: cachedMeta.Bucket,
+            UploadId: cachedUploadId,
+            fileSize: cachedMeta.fileSize,
+            extension: cachedMeta.extension,
+            ContentType: cachedMeta.ContentType,
             cache: true,
         };
     }
@@ -61,22 +276,29 @@ export async function startUpload({ Bucket, Key, ContentType, fileSize, }, s3 =
         ContentType,
     }));
     const extension = path.extname(Key).substring(1);
+    const meta = {
+        Key,
+        Bucket,
+        fileSize,
+        extension,
+        ContentType,
+    };
     if (config.redis) {
-        await rclient.set(res.UploadId, JSON.stringify({ Key, Bucket, fileSize, extension, ContentType }), "EX", 60 * 60);
+        await saveUploadMeta(res.UploadId, meta);
     }
     else {
-        uploads.set(res.UploadId, {
-            Key,
-            Bucket,
-            fileSize,
-            extension,
-            ContentType,
-        });
+        uploads.set(res.UploadId, meta);
+        resumableUploads.set(Key, res.UploadId);
+        await scheduleNonRedisCleanup(res.UploadId, s3);
     }
-    return { UploadId: res.UploadId };
+    return {
+        UploadId: res.UploadId,
+    };
 }
 export async function uploadChunk({ id, chunk, partNumber, }, s3 = s3Client) {
-    let upload = await findUpload({ id }, s3);
+    if (!s3)
+        throw new Error("No S3 client");
+    const upload = await findUpload({ id }, s3);
     if (!upload?.UploadId) {
         throw new Error("upload not found");
     }
@@ -90,17 +312,20 @@ export async function uploadChunk({ id, chunk, partNumber, }, s3 = s3Client) {
         PartNumber: partNumber,
         Body: chunk,
     }));
+    await refreshUploadTTL(id, UPLOAD_TTL_SECONDS, s3);
     return upload;
 }
 export async function finishUpload({ id, }, s3 = s3Client) {
-    const redisCache = config.redis ? await rclient.get(id) : undefined;
-    const { fileSize: cachedSize, Key, Bucket, } = redisCache ? JSON.parse(redisCache) : uploads.get(id) || {};
+    if (!s3)
+        throw new Error("No S3 client");
+    const redisCache = await getUploadMeta(id);
+    const { fileSize: cachedSize, Key, Bucket } = redisCache || {};
     const size = cachedSize ? Number(cachedSize) : undefined;
     const upload = await findUpload({ id }, s3);
     if (!upload?.uploaded) {
         throw new Error("File not uploaded");
     }
-    if (size && size != upload.uploaded) {
+    if (size && size !== upload.uploaded) {
         throw new Error("File size mismatch");
     }
     await s3.send(new CompleteMultipartUploadCommand({
@@ -111,21 +336,48 @@ export async function finishUpload({ id, }, s3 = s3Client) {
             Parts: upload.parts.sort((a, b) => a.PartNumber - b.PartNumber),
         },
     }));
-    // if (config.redis) await rclient.del(Key);
-    return { uploaded: upload.uploaded };
+    await deleteUploadMeta(id, Key);
+    return {
+        finished: true,
+        uploaded: upload.uploaded,
+        filepath: Key.replace(config.folder, ""),
+    };
+}
+export async function abortUpload({ id, }, s3 = s3Client) {
+    if (!s3)
+        throw new Error("No S3 client");
+    const meta = await getUploadMeta(id);
+    if (!meta) {
+        return {
+            aborted: false,
+        };
+    }
+    await s3.send(new AbortMultipartUploadCommand({
+        Bucket: meta.Bucket,
+        Key: meta.Key,
+        UploadId: id,
+    }));
+    await deleteUploadMeta(id, meta.Key);
+    return {
+        aborted: true,
+    };
 }
 export async function getUploadStatus({ id, Key, Bucket, }, s3 = s3Client) {
-    const redisCache = config.redis ? await rclient.get(id) : undefined;
-    // const redisCache = config.redis ? await rclient.get(Key) : undefined;
-    const { fileSize: cachedSize, Key: redisKey, Bucket: redisBucket, } = redisCache ? JSON.parse(redisCache) : uploads.get(id) || {};
+    if (!s3)
+        throw new Error("No S3 client");
+    const redisCache = id ? await getUploadMeta(id) : undefined;
+    const { fileSize: cachedSize, Key: redisKey, Bucket: redisBucket, } = redisCache || {};
     if (!redisKey && Key && Bucket) {
         const res = await s3
             .send(new HeadObjectCommand({
             Bucket,
             Key,
         }))
-            .catch((err) => false);
-        return { exists: !!res, uploaded: res?.ContentLength };
+            .catch(() => false);
+        return {
+            exists: !!res,
+            uploaded: res?.ContentLength,
+        };
     }
     const parts = await s3
         .send(new ListPartsCommand({
@@ -133,7 +385,7 @@ export async function getUploadStatus({ id, Key, Bucket, }, s3 = s3Client) {
         Key: redisKey,
         UploadId: id,
     }))
-        .catch((err) => ({
+        .catch(() => ({
         Parts: [],
     }));
     const res = id && !parts.Parts?.length
@@ -142,17 +394,17 @@ export async function getUploadStatus({ id, Key, Bucket, }, s3 = s3Client) {
             Bucket: redisBucket,
             Key: redisKey,
         }))
-            .catch((err) => false)
+            .catch(() => false)
         : false;
     return {
-        exists: !!res || !!parts.Parts?.length,
+        exists: !!res || !!parts?.Parts?.length,
         finished: !!res,
         uploaded: res
             ? res.ContentLength
-            : parts.Parts?.reduce((acc, curr) => acc + (curr.Size || 0), 0) || 0,
+            : parts?.Parts?.reduce((acc, curr) => acc + (curr.Size || 0), 0) || 0,
         id,
         size: cachedSize ? Number(cachedSize) : undefined,
-        parts: parts.Parts?.map((p) => ({
+        parts: parts?.Parts?.map((p) => ({
             ETag: p.ETag,
             PartNumber: p.PartNumber,
         })) || [],

package/dist/server/plugins/upload/startUpload.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"startUpload.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/startUpload.ts"],"names":[],"mappings":"AAiBA,wBAA8B,WAAW,CAAC,EACxC,IAAI,EACJ,EAAE,EACF,QAAQ,EACR,IAAI,EACJ,MAAM,GACP,EAAE;IACD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,gBA6EA"}
+{"version":3,"file":"startUpload.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/startUpload.ts"],"names":[],"mappings":"AAiBA,wBAA8B,WAAW,CAAC,EACxC,IAAI,EACJ,EAAE,EACF,QAAQ,EACR,IAAI,EACJ,MAAM,GACP,EAAE;IACD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,gBA+EA"}

package/dist/server/plugins/upload/startUpload.js

@@ -20,18 +20,19 @@ export default async function startUpload({ host, id, fileName, size, subdir, })
             .join(uploadChunkDirectory, subdir || "")
             .replace(/\\/g, "/");
         if (config.s3) {
-            const filepath = path.posix.join(relativeDirpath, `${id1}.${extension}`);
+            const filepath = path.posix.join(config.folder || '', relativeDirpath, `${id1}.${extension}`);
             const { UploadId } = await startUploadS3({
                 Bucket: config.s3?.containerName || "work",
                 Key: filepath[0] === "/" ? filepath?.slice(1) : filepath,
                 ContentType: mimes[extension],
                 fileSize: size,
             });
-            return { UploadId, provider: "s3" };
+            return { id: UploadId, UploadId, provider: "s3" };
         }
         const key = id1.split("-").pop();
         const meta = {
             id: id1,
+            UploadId: id1,
             key,
             fileName,
             size,
@@ -56,6 +57,7 @@ export default async function startUpload({ host, id, fileName, size, subdir, })
             subdir,
             metaPath: undefined,
             relativeDirpath,
+            provider: "fs"
         };
         await writeFile(path.join(metaDir, `${id1}.json`), JSON.stringify(metaData, null, 2));
         return { ...meta, metaPath: undefined };

package/dist/server/plugins/upload/uploadChunk.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"uploadChunk.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/uploadChunk.ts"],"names":[],"mappings":"AAqBA,wBAA8B,WAAW,CAAC,EACxC,IAAI,EACJ,EAAE,EACF,IAAI,EACJ,MAAM,EACN,GAAG,EACH,IAAI,GACL,EAAE;IACD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,GAAG,CAAC;IACV,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;CACd,gBA+HA"}
+{"version":3,"file":"uploadChunk.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/uploadChunk.ts"],"names":[],"mappings":"AAqBA,wBAA8B,WAAW,CAAC,EACxC,IAAI,EACJ,EAAE,EACF,IAAI,EACJ,MAAM,EACN,GAAG,EACH,IAAI,GACL,EAAE;IACD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,GAAG,CAAC;IACV,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;CACd,gBAwIA"}

package/dist/server/plugins/upload/uploadChunk.js

@@ -39,14 +39,16 @@ export default async function uploadChunk({ host, id, body, offset, end, size, }
             return { error: "Chunk size mismatch", code: 400 };
         }
         const relpath = path.posix.join(uploadChunkDirectory, `${id}.${meta.extension}`);
-        const filepath = path.posix.join(config.folder, uploadChunkDirectory, `${id}.${meta.extension}`);
         const res = await getFileSize(relpath, { fallback: false }).catch((err) => ({
             error: err.toString(),
+            code: err.code,
         }));
-        if (res.error && res.error !== "Error: file not found") {
+        if (res.error &&
+            res.error !== "Error: file not found" &&
+            res.code !== "ENOENT") {
             throw new Error(res.error);
         }
-        if (meta.size > res) {
+        if (!res.error && res > meta.size) {
             throw new Error("uploaded size exceeds file size");
         }
         // append chunk to existing file
@@ -67,8 +69,16 @@ export default async function uploadChunk({ host, id, body, offset, end, size, }
             // save metadata
             const fileBytes = Buffer.from(JSON.stringify(metaData, null, 2));
             await writeFile(path.join(metaDir, `${id}.json`), fileBytes);
-            const payload = { uploaded: meta.uploaded, finished };
-            const hookData = await applyHook("afterChunkedUpload", { id, meta });
+            const payload = {
+                uploaded: meta.uploaded,
+                finished,
+                filepath: path.posix.join(meta.relativeDirpath, meta.fileName),
+            };
+            const hookData = await applyHook("afterChunkedUpload", {
+                id,
+                meta,
+                payload,
+            });
             return hookData || payload;
         }
         return { uploaded: meta.uploaded, finished };
@@ -78,7 +88,9 @@ export default async function uploadChunk({ host, id, body, offset, end, size, }
     }
     const url = `${host}/${prefix}/${id}`;
     const range = `bytes ${offset || 0}-${end}/${size}`;
-    console.log("PATCH request", url, range, body.length);
+    if (config.trace || process.env.NODE_ENV === "test") {
+        console.log("PATCH request", url, range, body.length);
+    }
     const res = await fetch(url, {
         method: "PATCH",
         headers: {

package/dist/server/routes/table/controllers/suggest.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"suggest.d.ts","sourceRoot":"","sources":["../../../../../server/routes/table/controllers/suggest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAuBpD,wBAAsB,kBAAkB,CAAC,EACvC,KAAK,EACL,QAAQ,EACR,MAAM,EACN,UAAU,EACV,QAAQ,EACR,UAAU,EACV,GAAG,EACH,EAAqB,GACtB,EAAE;IACD,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,GAAG,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,EAAE,CAAC,EAAE,UAAU,CAAC;CACjB;;;;;UA+EA;AAED,wBAA8B,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,gBAmXzD"}
+{"version":3,"file":"suggest.d.ts","sourceRoot":"","sources":["../../../../../server/routes/table/controllers/suggest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAuBpD,wBAAsB,kBAAkB,CAAC,EACvC,KAAK,EACL,QAAQ,EACR,MAAM,EACN,UAAU,EACV,QAAQ,EACR,UAAU,EACV,GAAG,EACH,EAAqB,GACtB,EAAE;IACD,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,GAAG,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,EAAE,CAAC,EAAE,UAAU,CAAC;CACjB;;;;;UAwFA;AAED,wBAA8B,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,gBAyXzD"}

package/dist/server/routes/table/controllers/suggest.js

@@ -27,6 +27,14 @@ export async function getTableColumnMeta({ table, template, column, selectName,
         .filter((el) => !el.disabled && el.sql && el.sql.includes(column) && false)
         .map((el, i) => ` left join lateral (${el.sql}) ${el.name || `t${i}`} on 1=1 `)
         ?.join("") || "";
+    if (arr?.length && !arr.find((el) => el.id === "null")) {
+        arr.push({
+            color: "#B8B8B8",
+            icon: null,
+            id: "null",
+            text: "None",
+        });
+    }
     // w/out select
     const searchQueryDefault = startsWith && key
         ? `(left(lower("text"::text),${key.length}) = $1 )`
@@ -176,7 +184,7 @@ export default async function suggest(req, reply) {
         if (tableName && pg?.pk?.[tableName] && columnName) {
             const qRes = await pg.queryCache(sqlCls, { table: tableName });
             const vals = (query.count
-                ? qRes.rows?.map?.((el) => el.value?.toString?.())
+                ? qRes.rows?.map?.((el) => el.value?.toString?.() || null)
                 : qRes.rows?.[0]?.array_agg) || [];
             const lower = query.key?.toLowerCase?.();
             const data1 = query.key || query.val
@@ -185,7 +193,9 @@ export default async function suggest(req, reply) {
                     (el[lang] || el.text)?.toLowerCase()?.indexOf(lower) !== -1)
                     ?.filter((el) => !query.val || query.val?.split?.(",")?.includes?.(el.id))
                 : arr;
-            const data2 = data1.filter((el) => el.id && vals.includes(el.id.toString()));
+            const data2 = data1.filter((el) => el.id === "null"
+                ? vals.includes(null)
+                : el.id && vals.includes(el.id.toString()));
             const data = data2.slice(0, Math.min(query.limit || limit, limit));
             if (data.length && query.count) {
                 const counts = qRes.rows?.reduce?.((acc, curr) => ({
@@ -245,7 +255,7 @@ export default async function suggest(req, reply) {
         };
     }
     // select, with search by text support
-    const search = query.key ? searchQuery : null;
+    const search = typeof query.key === "string" ? searchQuery : null;
     // filter by value
     const val = query.val
         ? `${meta.pk}=any('{${query.val.replace(/'/g, "''")}}')`
@@ -258,13 +268,22 @@ export default async function suggest(req, reply) {
     const where = [search, val, filter, meta.pk ? `${meta.pk} is not null` : null]
         .filter(Boolean)
         .join(" and ") || "true";
-    const sqlSuggest = `with c(id,text) as ( ${meta.original.replace(/{{parent}}/gi, parent)} where ${where} ${meta.original.toLowerCase().includes("order by") ? "" : "order by 2"}) select * from c where ${filter} limit ${Math.min(query.limit || meta.limit || limit, limit)}`.replace(/{{uid}}/g, user?.uid || "0");
+    // const sqlSuggest = `with c(id,text) as ( ${meta.original.replace(/{{parent}}/gi, parent)} where ${where} ${meta.original.toLowerCase().includes("order by") ? "" : "order by 2"}) select * from c where ${filter} limit ${Math.min(query.limit || meta.limit || limit, limit)}`.replace(/{{uid}}/g, user?.uid || "0");
+    const sqlSuggest = `${meta.original}
+  WHERE EXISTS ( SELECT 1 FROM ${tableName} o WHERE o.${column} IS NOT DISTINCT FROM c.id ) and ${val || "true"} and ${search || "true"}
+  ${meta.original.toLowerCase().includes("order by") ? "" : "order by 2"}
+  LIMIT ${Math.min(query.limit || meta.limit || limit, limit)}`
+        .replace(/{{parent}}/gi, parent)
+        .replace(/{{uid}}/g, user?.uid || "0");
     if (query.sql && debugMode) {
         return sqlSuggest;
     }
-    const { rows: dataNew } = await pg.query(sqlSuggest, query.key ? [query.key.toLowerCase()] : []);
+    const { rows: dataNew } = await pg.query(sqlSuggest, typeof query.key === "string" ? [query.key.toLowerCase()] : []);
     // in case id / text = Boolean
     const data = dataNew.filter((el) => Object.hasOwn(el, "id") && Object.hasOwn(el, "text"));
+    data
+        .filter((el) => el.id === null)
+        .forEach((el) => Object.assign(el, { id: "null", text: "None" }));
     // table:column + select
     if (tableName && column) {
         const { name = selectName || column, type = "select" } = getColumnCLS(tableName, column) || {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengis/fastify-table",
-  "version": "2.3.0",
+  "version": "2.3.2",
   "type": "module",
   "description": "core-plugins",
   "keywords": [