@opengis/fastify-table 2.2.12 → 2.3.1

package/dist/helper.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"helper.d.ts","sourceRoot":"","sources":["../helper.ts"],"names":[],"mappings":"AAGA,OAAO,MAAM,MAAM,aAAa,CAAC;AAGjC,OAAO,EAAE,SAAS,EAAkB,MAAM,YAAY,CAAC;AAEvD,QAAA,MAAQ,MAAM,KAAoB,CAAC;AAInC,wBAAsB,KAAK,kBAI1B;AAGD,wBAAsB,QAAQ,kBAW7B;AAED,wBAAsB,KAAK,iBAyB1B;AAED,OAAO,EAEL,MAAM,EACN,MAAM,EACN,SAAS,GACV,CAAC"}
+{"version":3,"file":"helper.d.ts","sourceRoot":"","sources":["../helper.ts"],"names":[],"mappings":"AAEA,OAAO,MAAM,MAAM,aAAa,CAAC;AAGjC,OAAO,EAAE,SAAS,EAAkB,MAAM,YAAY,CAAC;AAEvD,QAAA,MAAQ,MAAM,KAAoB,CAAC;AAInC,wBAAsB,KAAK,kBAI1B;AAGD,wBAAsB,QAAQ,kBAW7B;AAED,wBAAsB,KAAK,iBA0B1B;AAED,OAAO,EAEL,MAAM,EACN,MAAM,EACN,SAAS,GACV,CAAC"}

package/dist/helper.js

@@ -31,6 +31,7 @@ export async function build() {
         });
         await app.ready(); // ? ensure plugins registered, can not add fastify hooks after app is ready
     }
+    console.log("server started", process.env.PORT || 3000);
     return app;
 }
 export { 

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AA4FA,iBAAe,MAAM,CAAC,OAAO,EAAE,GAAG,iBAgLjC;;AACD,wBAEG"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AA6FA,iBAAe,MAAM,CAAC,OAAO,EAAE,GAAG,iBAiLjC;;AACD,wBAEG"}

package/dist/index.js

@@ -34,6 +34,7 @@ import metricPlugin from "./server/plugins/metric/index.js";
 import redisPlugin from "./server/plugins/redis/index.js";
 import loggerPlugin from "./server/plugins/logger/index.js";
 import authPlugin from "./server/plugins/auth/index.js";
+import chunkedUploadPlugin from "./server/plugins/upload/index.js";
 // utils
 import execMigrations from "./server/plugins/migration/exec.migrations.js";
 import pgClients from "./server/plugins/pg/pgClients.js";
@@ -113,6 +114,7 @@ async function plugin(fastify) {
     execMigrations(path.join(cwd, "server/migrations"), pgClients.client, true).catch((err) => console.warn(err.toString()));
     // plugins / utils / funcs
     authPlugin(fastify); // fastify-auth api + hooks integrated to core
+    chunkedUploadPlugin(fastify);
     policyPlugin(fastify);
     metricPlugin();
     redisPlugin(fastify);

package/dist/server/plugins/file/getFileSize.d.ts

@@ -0,0 +1,2 @@
+export default function getFileSize(filePath: string, options?: Record<string, any>): Promise<any>;
+//# sourceMappingURL=getFileSize.d.ts.map

package/dist/server/plugins/file/getFileSize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"getFileSize.d.ts","sourceRoot":"","sources":["../../../../server/plugins/file/getFileSize.ts"],"names":[],"mappings":"AAGA,wBAA8B,WAAW,CACvC,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,gBAclC"}

package/dist/server/plugins/file/getFileSize.js

@@ -0,0 +1,12 @@
+import path from "node:path";
+import providers from "./providers/index.js";
+export default async function getFileSize(filePath, options = {}) {
+    const filename = path.basename(filePath);
+    // prefix
+    const prefix = (options.prefix === "date"
+        ? new Date().toISOString().split("T")[0]
+        : null) || (options.prefix === "3s" ? filename.substring(0, 3) : "");
+    const relativePath = path.join(path.dirname(filePath), prefix, filename);
+    const fp = providers(options);
+    return fp.getFileSize(relativePath, options);
+}

package/dist/server/plugins/file/providers/s3/funcs/getFileSize.d.ts

@@ -0,0 +1,3 @@
+declare const getFileSize: (s3Settings?: any) => (fp: any, options?: Record<string, any>) => Promise<any>;
+export default getFileSize;
+//# sourceMappingURL=getFileSize.d.ts.map

package/dist/server/plugins/file/providers/s3/funcs/getFileSize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"getFileSize.d.ts","sourceRoot":"","sources":["../../../../../../../server/plugins/file/providers/s3/funcs/getFileSize.ts"],"names":[],"mappings":"AAYA,QAAA,MAAM,WAAW,GACd,aAAa,GAAG,MACV,IAAI,GAAG,EAAE,UAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,iBAuBhD,CAAC;AAEJ,eAAe,WAAW,CAAC"}

package/dist/server/plugins/file/providers/s3/funcs/getFileSize.js

@@ -0,0 +1,30 @@
+import { HeadObjectCommand } from "@aws-sdk/client-s3";
+import s3Client from "../client.js";
+import config from "../../../../../../config.js";
+import getPath from "../../../utils/getPath.js";
+import getS3FilePath from "./utils/getS3FilePath.js";
+// if not found on s3 => fs
+import fs from "../../fs.js";
+const { getFileSize: getFileSizeFs } = fs();
+const getFileSize = (s3Settings) => async (fp, options = {}) => {
+    const filepath = getS3FilePath(fp, s3Settings);
+    const bucketParams = {
+        Bucket: s3Settings?.containerName || config.s3?.containerName || "work",
+        Key: filepath[0] === "/" ? filepath?.slice(1) : filepath,
+    };
+    try {
+        const data = await s3Client.send(new HeadObjectCommand(bucketParams));
+        return data.ContentLength;
+    }
+    catch (err) {
+        if (options.fallback === false) {
+            throw new Error("file not found");
+        }
+        const filepath1 = getPath(fp);
+        if (!filepath1) {
+            throw new Error("file not found");
+        }
+        return getFileSizeFs(filepath1);
+    }
+};
+export default getFileSize;

package/dist/server/plugins/file/providers/s3/funcs/uploadFile.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"uploadFile.d.ts","sourceRoot":"","sources":["../../../../../../../server/plugins/file/providers/s3/funcs/uploadFile.ts"],"names":[],"mappings":"AAaA,QAAA,MAAM,UAAU,GAAI,aAAa,GAAG,MAAY,IAAI,GAAG,EAAE,MAAM,GAAG,+EA6BjE,CAAC;AAEF,eAAe,UAAU,CAAC"}
+{"version":3,"file":"uploadFile.d.ts","sourceRoot":"","sources":["../../../../../../../server/plugins/file/providers/s3/funcs/uploadFile.ts"],"names":[],"mappings":"AAcA,QAAA,MAAM,UAAU,GAAI,aAAa,GAAG,MAAY,IAAI,GAAG,EAAE,MAAM,GAAG,+EA6BjE,CAAC;AAEF,eAAe,UAAU,CAAC"}

package/dist/server/plugins/file/providers/s3/funcs/uploadFile.js

@@ -6,6 +6,7 @@ import getDataSize from "../../utils/getDataSize.js";
 import dataTypes from "../../utils/handlers/dataTypes.js";
 import getMimeType from "../../mime/index.js";
 import getS3FilePath from "./utils/getS3FilePath.js";
+// chunks not supported for s3
 const uploadFile = (s3Settings) => async (fp, data) => {
     const filepath = getS3FilePath(fp, s3Settings);
     const validData = await getValidData({ data, types: [dataTypes.stream] });

package/dist/server/plugins/file/providers/s3/index.d.ts

@@ -3,5 +3,6 @@ export default function s3Storage(opt: any): {
     downloadFile: (fp: string, options?: Record<string, any>) => Promise<any>;
     uploadFile: (fp: any, data: any) => Promise<import("@aws-sdk/client-s3").CompleteMultipartUploadCommandOutput>;
     fileExists: (fp: any, options?: Record<string, any>) => Promise<any>;
+    getFileSize: (fp: any, options?: Record<string, any>) => Promise<any>;
 };
 //# sourceMappingURL=index.d.ts.map

package/dist/server/plugins/file/providers/s3/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../../server/plugins/file/providers/s3/index.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,OAAO,UAAU,SAAS,CAAC,GAAG,EAAE,GAAG;;;;;EAOzC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../../server/plugins/file/providers/s3/index.ts"],"names":[],"mappings":"AAKA,MAAM,CAAC,OAAO,UAAU,SAAS,CAAC,GAAG,EAAE,GAAG;;;;;;EAQzC"}

package/dist/server/plugins/file/providers/s3/index.js

@@ -1,5 +1,6 @@
 import downloadFile from "./funcs/downloadFile.js";
 import fileExists from "./funcs/fileExists.js";
+import getFileSize from "./funcs/getFileSize.js";
 import uploadFile from "./funcs/uploadFile.js";
 export default function s3Storage(opt) {
     return {
@@ -7,5 +8,6 @@ export default function s3Storage(opt) {
         downloadFile: downloadFile(opt),
         uploadFile: uploadFile(opt),
         fileExists: fileExists(opt),
+        getFileSize: getFileSize(opt),
     };
 }

package/dist/server/plugins/file/uploadFile.d.ts

@@ -1,3 +1,3 @@
-declare function uploadFile(filePath: string, data: any, options?: Record<string, string>): Promise<string>;
+declare function uploadFile(filePath: string, data: any, options?: Record<string, any>): Promise<string>;
 export default uploadFile;
 //# sourceMappingURL=uploadFile.d.ts.map

package/dist/server/plugins/file/uploadFile.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"uploadFile.d.ts","sourceRoot":"","sources":["../../../../server/plugins/file/uploadFile.ts"],"names":[],"mappings":"AAGA,iBAAe,UAAU,CACvB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,GAAG,EACT,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAM,mBAgBrC;AAED,eAAe,UAAU,CAAC"}
+{"version":3,"file":"uploadFile.d.ts","sourceRoot":"","sources":["../../../../server/plugins/file/uploadFile.ts"],"names":[],"mappings":"AAGA,iBAAe,UAAU,CACvB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,GAAG,EACT,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,mBAgBlC;AAED,eAAe,UAAU,CAAC"}

package/dist/server/plugins/upload/getUploadStatus.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"getUploadStatus.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/getUploadStatus.ts"],"names":[],"mappings":"AAaA,wBAA8B,eAAe,CAAC,EAC5C,IAAI,EACJ,EAAE,GACH,EAAE;IACD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;CACZ,gBA+BA"}
+{"version":3,"file":"getUploadStatus.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/getUploadStatus.ts"],"names":[],"mappings":"AAeA,wBAA8B,eAAe,CAAC,EAC5C,IAAI,EACJ,EAAE,GACH,EAAE;IACD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;CACZ,gBAmCA"}

package/dist/server/plugins/upload/getUploadStatus.js

@@ -2,10 +2,15 @@ import path from "node:path";
 import { existsSync } from "node:fs";
 import { readFile } from "node:fs/promises";
 import { prefix, metaDir, uploadChunkDirectory, fetchTimeoutMs, } from "./index.js";
+import config from "../../../config.js";
 import isFileExists from "../file/isFileExists.js";
+import { getUploadStatus as getUploadStatusS3 } from "./s3.js";
 export default async function getUploadStatus({ host, id, }) {
     // return local file metadata
     if (!host) {
+        if (config.s3) {
+            return getUploadStatusS3({ id });
+        }
         const metaExists = existsSync(path.join(metaDir, `${id}.json`));
         // check file upload status: finished/inprogress
         const meta = metaExists

package/dist/server/plugins/upload/index.d.ts

@@ -1,6 +1,9 @@
+import { FastifyInstance } from "fastify";
 export declare const fetchTimeoutMs: number;
 export declare const prefix = "file/upload2";
 export declare const uploadChunkDirectory = "/files/uploads";
 export declare const fileDir: string;
 export declare const metaDir: string;
+declare function chunkedUploadPlugin(app: FastifyInstance): void;
+export default chunkedUploadPlugin;
 //# sourceMappingURL=index.d.ts.map

package/dist/server/plugins/upload/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/index.ts"],"names":[],"mappings":"AASA,eAAO,MAAM,cAAc,QAAmC,CAAC;AAE/D,eAAO,MAAM,MAAM,iBAAiB,CAAC;AAErC,eAAO,MAAM,oBAAoB,mBAAmB,CAAC;AAErD,eAAO,MAAM,OAAO,QAEE,CAAC;AAEvB,eAAO,MAAM,OAAO,QAA4B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAW1C,eAAO,MAAM,cAAc,QAE1B,CAAC;AAEF,eAAO,MAAM,MAAM,iBAAiB,CAAC;AAErC,eAAO,MAAM,oBAAoB,mBAAmB,CAAC;AAErD,eAAO,MAAM,OAAO,QAEE,CAAC;AAEvB,eAAO,MAAM,OAAO,QAA4B,CAAC;AAKjD,iBAAS,mBAAmB,CAAC,GAAG,EAAE,eAAe,QAIhD;AAED,eAAe,mBAAmB,CAAC"}

package/dist/server/plugins/upload/index.js

@@ -1,12 +1,20 @@
 import path from "node:path";
 import config from "../../../config.js";
 import getFolder from "../crud/funcs/utils/getFolder.js";
-config.chunkSize = +(config.chunkSize || 1048576); // 1 MB per chunk by default
+config.chunkSize = +(config.chunkSize || 5242880); // 5 MB per chunk by default
 const rootDir = getFolder(config, "local");
-export const fetchTimeoutMs = +(config.fetchTimeoutMs || 5000);
+export const fetchTimeoutMs = +(config.fetchTimeoutMs || (process.platform === "win32" ? 50000 : 5000));
 export const prefix = "file/upload2";
 export const uploadChunkDirectory = "/files/uploads";
 export const fileDir = path
     .join(rootDir, uploadChunkDirectory)
     .replace(/\\/g, "/");
 export const metaDir = path.join(fileDir, "tmp");
+import { cleanupNonRedisUploads } from "./s3.js";
+// w/ out redis - chunked uploads are non-resumable in order to avoid unclosed multiparts (non-discoverable via s3 commands with current version of minio)
+function chunkedUploadPlugin(app) {
+    app.addHook("onClose", async () => {
+        await cleanupNonRedisUploads();
+    });
+}
+export default chunkedUploadPlugin;

package/dist/server/plugins/upload/s3.d.ts

@@ -0,0 +1,83 @@
+export declare function cleanupNonRedisUploads(s3?: any): Promise<void>;
+export declare function saveUploadMeta(id: string, meta: any, ttl?: number): Promise<void>;
+export declare function refreshUploadTTL(id: string, ttl?: number, s3?: any): Promise<void>;
+export declare function deleteUploadMeta(id: string, key?: string): Promise<void>;
+export declare function getUploadMeta(id: string): Promise<any>;
+export declare function getResumableUploadId(key: string): Promise<any>;
+export declare function cleanupStaleUploads(s3?: any): Promise<void>;
+export declare function findUpload({ id }: {
+    id: string;
+}, s3?: any): Promise<{
+    UploadId: string | undefined;
+    Key: any;
+    Bucket: any;
+    fileSize: any;
+    ContentType: any;
+    uploaded: any;
+    parts: any;
+}>;
+export declare function startUpload({ Bucket, Key, ContentType, fileSize, }: {
+    Bucket: string;
+    Key: string;
+    ContentType: string;
+    fileSize: number;
+}, s3?: any): Promise<{
+    Key: string;
+    Bucket: any;
+    UploadId: any;
+    fileSize: any;
+    extension: any;
+    ContentType: any;
+    cache: boolean;
+} | {
+    UploadId: any;
+    Key?: undefined;
+    Bucket?: undefined;
+    fileSize?: undefined;
+    extension?: undefined;
+    ContentType?: undefined;
+    cache?: undefined;
+}>;
+export declare function uploadChunk({ id, chunk, partNumber, }: {
+    id: string;
+    chunk: any;
+    partNumber: number;
+}, s3?: any): Promise<{
+    UploadId: string | undefined;
+    Key: any;
+    Bucket: any;
+    fileSize: any;
+    ContentType: any;
+    uploaded: any;
+    parts: any;
+}>;
+export declare function finishUpload({ id, }: {
+    id: string;
+}, s3?: any): Promise<{
+    uploaded: any;
+}>;
+export declare function abortUpload({ id, }: {
+    id: string;
+}, s3?: any): Promise<{
+    aborted: boolean;
+}>;
+export declare function getUploadStatus({ id, Key, Bucket, }: {
+    id: string;
+    Key?: string;
+    Bucket?: string;
+}, s3?: any): Promise<{
+    exists: boolean;
+    uploaded: any;
+    finished?: undefined;
+    id?: undefined;
+    size?: undefined;
+    parts?: undefined;
+} | {
+    exists: boolean;
+    finished: boolean;
+    uploaded: any;
+    id: string;
+    size: number | undefined;
+    parts: any;
+}>;
+//# sourceMappingURL=s3.d.ts.map

package/dist/server/plugins/upload/s3.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/s3.ts"],"names":[],"mappings":"AAgGA,wBAAsB,sBAAsB,CAAC,EAAE,MAAW,iBAuCzD;AAED,wBAAsB,cAAc,CAClC,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,GAAG,EACT,GAAG,SAAqB,iBAazB;AAED,wBAAsB,gBAAgB,CACpC,EAAE,EAAE,MAAM,EACV,GAAG,SAAqB,EACxB,EAAE,MAAW,iBAqBd;AAED,wBAAsB,gBAAgB,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,iBA0B9D;AAED,wBAAsB,aAAa,CAAC,EAAE,EAAE,MAAM,gBAQ7C;AAED,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,MAAM,gBAMrD;AAED,wBAAsB,mBAAmB,CAAC,EAAE,MAAW,iBAwEtD;AAED,wBAAsB,UAAU,CAAC,EAAE,EAAE,EAAE,EAAE;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,EAAE,EAAE,MAAW;;;;;;;;GAqCrE;AAED,wBAAsB,WAAW,CAC/B,EACE,MAAM,EACN,GAAG,EACH,WAAW,EACX,QAAQ,GACT,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB,EACD,EAAE,MAAW;;;;;;;;;;;;;;;;GAqDd;AAED,wBAAsB,WAAW,CAC/B,EACE,EAAE,EACF,KAAK,EACL,UAAU,GACX,EAAE;IACD,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,GAAG,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;CACpB,EACD,EAAE,MAAW;;;;;;;;GAyBd;AAED,wBAAsB,YAAY,CAChC,EACE,EAAE,GACH,EAAE;IACD,EAAE,EAAE,MAAM,CAAC;CACZ,EACD,EAAE,MAAW;;GAoCd;AAED,wBAAsB,WAAW,CAC/B,EACE,EAAE,GACH,EAAE;IACD,EAAE,EAAE,MAAM,CAAC;CACZ,EACD,EAAE,MAAW;;GAuBd;AAED,wBAAsB,eAAe,CACnC,EACE,EAAE,EACF,GAAG,EACH,MAAM,GACP,EAAE;IACD,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,EACD,EAAE,MAAW;;;;;;;;;;;;;;GAmEd"}

package/dist/server/plugins/upload/s3.js

@@ -0,0 +1,387 @@
+import path from "node:path";
+import { CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, HeadObjectCommand, ListPartsCommand, AbortMultipartUploadCommand, } from "@aws-sdk/client-s3";
+import config from "../../../config.js";
+import s3Client from "../file/providers/s3/client.js";
+import rclient from "../redis/client.js";
+import logger from "../logger/getLogger.js";
+const uploads = new Map();
+const resumableUploads = new Map();
+// non-redis upload timeout handlers
+const uploadTimeouts = new Map();
+const UPLOAD_TTL_SECONDS = 60 * 60;
+// expire gracefully to give cleanup worker time to abort stale uploads
+const CLEANUP_GRACE_SECONDS = 60 * 60 * 24;
+const REDIS_TTL = UPLOAD_TTL_SECONDS + CLEANUP_GRACE_SECONDS;
+const uploadMetaKey = (id) => `upload:${id}:meta`;
+const uploadExpireKey = (id) => `upload:${id}:expires`;
+const resumableKey = (key) => `resumable:${key}`;
+function clearUploadTimeout(id) {
+    const timeout = uploadTimeouts.get(id);
+    if (timeout) {
+        clearTimeout(timeout);
+        uploadTimeouts.delete(id);
+    }
+}
+async function scheduleNonRedisCleanup(id, s3 = s3Client) {
+    if (config.redis)
+        return;
+    clearUploadTimeout(id);
+    const timeout = setTimeout(async () => {
+        try {
+            const meta = uploads.get(id);
+            if (!meta)
+                return;
+            try {
+                await s3.send(new AbortMultipartUploadCommand({
+                    Bucket: meta.Bucket,
+                    Key: meta.Key,
+                    UploadId: id,
+                }));
+            }
+            catch (err) {
+                console.error("AbortMultipartUpload failed", {
+                    uploadId: id,
+                    Bucket: meta.Bucket,
+                    Key: meta.Key,
+                    error: err?.message,
+                });
+                logger.file("upload/abort/error", {
+                    uploadId: id,
+                    Bucket: meta.Bucket,
+                    Key: meta.Key,
+                    error: err?.message,
+                });
+            }
+            uploads.delete(id);
+            if (resumableUploads.get(meta.Key) === id) {
+                resumableUploads.delete(meta.Key);
+            }
+        }
+        catch (err) {
+            console.error(err);
+            logger.error(err);
+        }
+        finally {
+            clearUploadTimeout(id);
+        }
+    }, UPLOAD_TTL_SECONDS * 1000);
+    // don't keep node process alive
+    timeout.unref?.();
+    uploadTimeouts.set(id, timeout);
+}
+// used via onClose hook to avoid memory leaks
+export async function cleanupNonRedisUploads(s3 = s3Client) {
+    if (config.redis)
+        return;
+    console.log("cleanupNonRedisUploads start");
+    const entries = [...uploads.entries()];
+    for (const [id, meta] of entries) {
+        try {
+            await s3.send(new AbortMultipartUploadCommand({
+                Bucket: meta.Bucket,
+                Key: meta.Key,
+                UploadId: id,
+            }));
+        }
+        catch (err) {
+            console.error("AbortMultipartUpload failed", {
+                uploadId: id,
+                Bucket: meta.Bucket,
+                Key: meta.Key,
+                error: err?.message,
+            });
+            logger.file("upload/abort/error", {
+                uploadId: id,
+                Bucket: meta.Bucket,
+                Key: meta.Key,
+                error: err?.message,
+            });
+        }
+        clearUploadTimeout(id);
+        uploads.delete(id);
+        if (resumableUploads.get(meta.Key) === id) {
+            resumableUploads.delete(meta.Key);
+        }
+    }
+    console.log("open unfinished multipart uploads closed", entries.length);
+}
+export async function saveUploadMeta(id, meta, ttl = UPLOAD_TTL_SECONDS) {
+    if (!config.redis)
+        return;
+    const expiresAt = Date.now() + UPLOAD_TTL_SECONDS * 1000;
+    await Promise.all([
+        rclient.set(uploadMetaKey(id), JSON.stringify(meta), "EX", REDIS_TTL),
+        rclient.set(uploadExpireKey(id), String(expiresAt), "EX", ttl),
+        rclient.set(resumableKey(meta.Key), id, "EX", ttl),
+    ]);
+}
+export async function refreshUploadTTL(id, ttl = UPLOAD_TTL_SECONDS, s3 = s3Client) {
+    if (!config.redis) {
+        await scheduleNonRedisCleanup(id, s3);
+        return;
+    }
+    const meta = await getUploadMeta(id);
+    if (!meta)
+        return;
+    const expiresAt = Date.now() + ttl * 1000;
+    await Promise.all([
+        rclient.expire(uploadMetaKey(id), ttl),
+        rclient.set(uploadExpireKey(id), String(expiresAt), "EX", ttl),
+        rclient.expire(resumableKey(meta.Key), ttl),
+    ]);
+}
+export async function deleteUploadMeta(id, key) {
+    clearUploadTimeout(id);
+    if (!config.redis) {
+        const meta = uploads.get(id);
+        uploads.delete(id);
+        const redisKey = key || meta?.Key;
+        if (redisKey && resumableUploads.get(redisKey) === id) {
+            resumableUploads.delete(redisKey);
+        }
+        return;
+    }
+    const redisKey = key || (await getUploadMeta(id))?.Key;
+    const keys = [uploadMetaKey(id), uploadExpireKey(id)];
+    if (redisKey) {
+        keys.push(resumableKey(redisKey));
+    }
+    await rclient.del(keys);
+}
+export async function getUploadMeta(id) {
+    if (!config.redis) {
+        return uploads.get(id);
+    }
+    const raw = await rclient.get(uploadMetaKey(id));
+    return raw ? JSON.parse(raw) : undefined;
+}
+export async function getResumableUploadId(key) {
+    if (!config.redis) {
+        return resumableUploads.get(key);
+    }
+    return rclient.get(resumableKey(key));
+}
+export async function cleanupStaleUploads(s3 = s3Client) {
+    if (!config.redis) {
+        await cleanupNonRedisUploads(s3);
+        return;
+    }
+    let cursor = "0";
+    do {
+        const [nextCursor, keys] = await rclient.scan(cursor, "MATCH", "upload:*:expires", "COUNT", 100);
+        cursor = nextCursor;
+        for (const expireKey of keys) {
+            try {
+                const expiresAt = Number(await rclient.get(expireKey));
+                if (!expiresAt || expiresAt > Date.now()) {
+                    continue;
+                }
+                const uploadId = expireKey
+                    .replace("upload:", "")
+                    .replace(":expires", "");
+                const meta = await getUploadMeta(uploadId);
+                if (!meta) {
+                    await rclient.del(expireKey);
+                    continue;
+                }
+                const { Bucket, Key } = meta;
+                try {
+                    await s3.send(new AbortMultipartUploadCommand({
+                        Bucket,
+                        Key,
+                        UploadId: uploadId,
+                    }));
+                }
+                catch (err) {
+                    console.error("AbortMultipartUpload failed", {
+                        uploadId,
+                        Bucket,
+                        Key,
+                        error: err?.message,
+                    });
+                    logger.file("upload/abort/error", {
+                        uploadId,
+                        Bucket,
+                        Key,
+                        error: err?.message,
+                    });
+                }
+                await deleteUploadMeta(uploadId, Key);
+            }
+            catch (err) {
+                console.error(err);
+                logger.error(err);
+            }
+        }
+    } while (cursor !== "0");
+}
+export async function findUpload({ id }, s3 = s3Client) {
+    const cachedInfo = await getUploadMeta(id);
+    const { Key, Bucket, fileSize: cacheSize, ContentType: cacheContentType, } = cachedInfo || {};
+    const parts = await s3
+        .send(new ListPartsCommand({
+        Bucket,
+        Key,
+        UploadId: id,
+    }))
+        .catch(() => null);
+    return {
+        UploadId: parts ? id : undefined,
+        Key,
+        Bucket,
+        fileSize: cacheSize,
+        ContentType: cacheContentType,
+        uploaded: parts?.Parts?.reduce((sum, part) => sum + (part.Size || 0), 0) || 0,
+        parts: parts?.Parts?.map((p) => ({
+            ETag: p.ETag,
+            PartNumber: p.PartNumber,
+        })) || [],
+    };
+}
+export async function startUpload({ Bucket, Key, ContentType, fileSize, }, s3 = s3Client) {
+    const cachedUploadId = await getResumableUploadId(Key);
+    const cachedMeta = cachedUploadId
+        ? await getUploadMeta(cachedUploadId)
+        : undefined;
+    if (cachedUploadId && cachedMeta) {
+        await refreshUploadTTL(cachedUploadId, UPLOAD_TTL_SECONDS, s3);
+        return {
+            Key,
+            Bucket: cachedMeta.Bucket,
+            UploadId: cachedUploadId,
+            fileSize: cachedMeta.fileSize,
+            extension: cachedMeta.extension,
+            ContentType: cachedMeta.ContentType,
+            cache: true,
+        };
+    }
+    const res = await s3.send(new CreateMultipartUploadCommand({
+        Bucket,
+        Key,
+        ContentType,
+    }));
+    const extension = path.extname(Key).substring(1);
+    const meta = {
+        Key,
+        Bucket,
+        fileSize,
+        extension,
+        ContentType,
+    };
+    if (config.redis) {
+        await saveUploadMeta(res.UploadId, meta);
+    }
+    else {
+        uploads.set(res.UploadId, meta);
+        resumableUploads.set(Key, res.UploadId);
+        await scheduleNonRedisCleanup(res.UploadId, s3);
+    }
+    return {
+        UploadId: res.UploadId,
+    };
+}
+export async function uploadChunk({ id, chunk, partNumber, }, s3 = s3Client) {
+    const upload = await findUpload({ id }, s3);
+    if (!upload?.UploadId) {
+        throw new Error("upload not found");
+    }
+    if (upload.parts.some((p) => p.PartNumber === partNumber)) {
+        throw new Error(`Part ${partNumber} already uploaded`);
+    }
+    await s3.send(new UploadPartCommand({
+        Bucket: upload.Bucket,
+        Key: upload.Key,
+        UploadId: upload.UploadId,
+        PartNumber: partNumber,
+        Body: chunk,
+    }));
+    await refreshUploadTTL(id, UPLOAD_TTL_SECONDS, s3);
+    return upload;
+}
+export async function finishUpload({ id, }, s3 = s3Client) {
+    const redisCache = await getUploadMeta(id);
+    const { fileSize: cachedSize, Key, Bucket } = redisCache || {};
+    const size = cachedSize ? Number(cachedSize) : undefined;
+    const upload = await findUpload({ id }, s3);
+    if (!upload?.uploaded) {
+        throw new Error("File not uploaded");
+    }
+    if (size && size !== upload.uploaded) {
+        throw new Error("File size mismatch");
+    }
+    await s3.send(new CompleteMultipartUploadCommand({
+        Bucket,
+        Key,
+        UploadId: id,
+        MultipartUpload: {
+            Parts: upload.parts.sort((a, b) => a.PartNumber - b.PartNumber),
+        },
+    }));
+    await deleteUploadMeta(id, Key);
+    return {
+        uploaded: upload.uploaded,
+    };
+}
+export async function abortUpload({ id, }, s3 = s3Client) {
+    const meta = await getUploadMeta(id);
+    if (!meta) {
+        return {
+            aborted: false,
+        };
+    }
+    await s3.send(new AbortMultipartUploadCommand({
+        Bucket: meta.Bucket,
+        Key: meta.Key,
+        UploadId: id,
+    }));
+    await deleteUploadMeta(id, meta.Key);
+    return {
+        aborted: true,
+    };
+}
+export async function getUploadStatus({ id, Key, Bucket, }, s3 = s3Client) {
+    const redisCache = id ? await getUploadMeta(id) : undefined;
+    const { fileSize: cachedSize, Key: redisKey, Bucket: redisBucket, } = redisCache || {};
+    if (!redisKey && Key && Bucket) {
+        const res = await s3
+            .send(new HeadObjectCommand({
+            Bucket,
+            Key,
+        }))
+            .catch(() => false);
+        return {
+            exists: !!res,
+            uploaded: res?.ContentLength,
+        };
+    }
+    const parts = await s3
+        .send(new ListPartsCommand({
+        Bucket: redisBucket,
+        Key: redisKey,
+        UploadId: id,
+    }))
+        .catch(() => ({
+        Parts: [],
+    }));
+    const res = id && !parts.Parts?.length
+        ? await s3
+            .send(new HeadObjectCommand({
+            Bucket: redisBucket,
+            Key: redisKey,
+        }))
+            .catch(() => false)
+        : false;
+    return {
+        exists: !!res || !!parts?.Parts?.length,
+        finished: !!res,
+        uploaded: res
+            ? res.ContentLength
+            : parts?.Parts?.reduce((acc, curr) => acc + (curr.Size || 0), 0) || 0,
+        id,
+        size: cachedSize ? Number(cachedSize) : undefined,
+        parts: parts?.Parts?.map((p) => ({
+            ETag: p.ETag,
+            PartNumber: p.PartNumber,
+        })) || [],
+    };
+}

package/dist/server/plugins/upload/startUpload.d.ts

@@ -2,7 +2,7 @@ export default function startUpload({ host, id, fileName, size, subdir, }: {
     host?: string;
     id?: string;
     fileName: string;
-    size?: number;
+    size: number;
     subdir?: string;
 }): Promise<any>;
 //# sourceMappingURL=startUpload.d.ts.map

package/dist/server/plugins/upload/startUpload.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"startUpload.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/startUpload.ts"],"names":[],"mappings":"AAaA,wBAA8B,WAAW,CAAC,EACxC,IAAI,EACJ,EAAE,EACF,QAAQ,EACR,IAAI,EACJ,MAAM,GACP,EAAE;IACD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,gBAkEA"}
+{"version":3,"file":"startUpload.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/startUpload.ts"],"names":[],"mappings":"AAiBA,wBAA8B,WAAW,CAAC,EACxC,IAAI,EACJ,EAAE,EACF,QAAQ,EACR,IAAI,EACJ,MAAM,GACP,EAAE;IACD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,gBA+EA"}

package/dist/server/plugins/upload/startUpload.js

@@ -1,8 +1,11 @@
 import path from "node:path";
 import { randomUUID } from "node:crypto";
 import { mkdir, writeFile } from "node:fs/promises";
+import config from "../../../config.js";
 import getUploadStatus from "./getUploadStatus.js";
 import { prefix, fileDir, metaDir, uploadChunkDirectory, fetchTimeoutMs, } from "./index.js";
+import mimes from "../file/providers/mime/mimes.js";
+import { startUpload as startUploadS3 } from "./s3.js";
 export default async function startUpload({ host, id, fileName, size, subdir, }) {
     if (subdir && (typeof subdir !== "string" || subdir.includes(".."))) {
         return {
@@ -11,11 +14,25 @@ export default async function startUpload({ host, id, fileName, size, subdir, })
         };
     }
     if (!host) {
-        const extension = path.extname(fileName).substring(1);
         const id1 = id || randomUUID();
+        const extension = path.extname(fileName).substring(1);
+        const relativeDirpath = path
+            .join(uploadChunkDirectory, subdir || "")
+            .replace(/\\/g, "/");
+        if (config.s3) {
+            const filepath = path.posix.join(config.folder || '', relativeDirpath, `${id1}.${extension}`);
+            const { UploadId } = await startUploadS3({
+                Bucket: config.s3?.containerName || "work",
+                Key: filepath[0] === "/" ? filepath?.slice(1) : filepath,
+                ContentType: mimes[extension],
+                fileSize: size,
+            });
+            return { id: UploadId, UploadId, provider: "s3" };
+        }
         const key = id1.split("-").pop();
         const meta = {
             id: id1,
+            UploadId: id1,
             key,
             fileName,
             size,
@@ -33,9 +50,6 @@ export default async function startUpload({ host, id, fileName, size, subdir, })
             return meta;
         }
         // if not started - start new upload
-        const relativeDirpath = path
-            .join(uploadChunkDirectory, subdir || "")
-            .replace(/\\/g, "/");
         await mkdir(path.join(fileDir, "tmp"), { recursive: true });
         // create metadata for resumable upload
         const metaData = {
@@ -43,6 +57,7 @@ export default async function startUpload({ host, id, fileName, size, subdir, })
             subdir,
             metaPath: undefined,
             relativeDirpath,
+            provider: "fs"
         };
         await writeFile(path.join(metaDir, `${id1}.json`), JSON.stringify(metaData, null, 2));
         return { ...meta, metaPath: undefined };

package/dist/server/plugins/upload/uploadChunk.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"uploadChunk.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/uploadChunk.ts"],"names":[],"mappings":"AAaA,wBAA8B,WAAW,CAAC,EACxC,IAAI,EACJ,EAAE,EACF,IAAI,EACJ,MAAM,EACN,GAAG,EACH,IAAI,GACL,EAAE;IACD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,GAAG,CAAC;IACV,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;CACd,gBA+EA"}
+{"version":3,"file":"uploadChunk.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/uploadChunk.ts"],"names":[],"mappings":"AAqBA,wBAA8B,WAAW,CAAC,EACxC,IAAI,EACJ,EAAE,EACF,IAAI,EACJ,MAAM,EACN,GAAG,EACH,IAAI,GACL,EAAE;IACD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,GAAG,CAAC;IACV,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;CACd,gBAiIA"}

package/dist/server/plugins/upload/uploadChunk.js

@@ -2,10 +2,28 @@ import path from "node:path";
 import { existsSync } from "node:fs";
 import { readFile, writeFile } from "node:fs/promises";
 import { prefix, metaDir, fetchTimeoutMs, uploadChunkDirectory, } from "./index.js";
+import config from "../../../config.js";
 import uploadFile from "../file/uploadFile.js";
 import applyHook from "../hook/applyHook.js";
+import getFileSize from "../file/getFileSize.js";
+import { finishUpload as finishUploadS3, uploadChunk as uploadChunkS3, getUploadStatus as getUploadStatusS3, findUpload as findUploadS3, } from "./s3.js";
 export default async function uploadChunk({ host, id, body, offset, end, size, }) {
     if (!host) {
+        if (config.s3) {
+            const partNumber = Math.floor(offset / config.chunkSize) + 1;
+            const { fileSize } = await findUploadS3({ id });
+            await uploadChunkS3({
+                id,
+                chunk: body,
+                partNumber,
+            });
+            const currentState = await getUploadStatusS3({ id });
+            const finished = currentState.uploaded === fileSize;
+            if (finished) {
+                return finishUploadS3({ id });
+            }
+            return currentState;
+        }
         const metaExists = existsSync(path.join(metaDir, `${id}.json`));
         if (!metaExists) {
             return { error: "File not found", code: 404 };
@@ -20,6 +38,17 @@ export default async function uploadChunk({ host, id, body, offset, end, size, }
         if (body?.length !== end - offset + 1) {
             return { error: "Chunk size mismatch", code: 400 };
         }
+        const relpath = path.posix.join(uploadChunkDirectory, `${id}.${meta.extension}`);
+        const filepath = path.posix.join(config.folder, uploadChunkDirectory, `${id}.${meta.extension}`);
+        const res = await getFileSize(relpath, { fallback: false }).catch((err) => ({
+            error: err.toString(),
+        }));
+        if (res.error && res.error !== "Error: file not found") {
+            throw new Error(res.error);
+        }
+        if (meta.size > res) {
+            throw new Error("uploaded size exceeds file size");
+        }
         // append chunk to existing file
         await uploadFile(path
             .join(uploadChunkDirectory, `${id}.${meta.extension}`)
@@ -47,10 +76,15 @@ export default async function uploadChunk({ host, id, body, offset, end, size, }
     if (!end || !size) {
         return { error: "not enough params: offset/end/size", code: 400 };
     }
-    const res = await fetch(`${host}/${prefix}/${id}`, {
+    const url = `${host}/${prefix}/${id}`;
+    const range = `bytes ${offset || 0}-${end}/${size}`;
+    if (config.trace || process.env.NODE_ENV === "test") {
+        console.log("PATCH request", url, range, body.length);
+    }
+    const res = await fetch(url, {
         method: "PATCH",
         headers: {
-            "Content-Range": `bytes ${offset || 0}-${end}/${size}`,
+            "Content-Range": range,
         },
         body,
         signal: AbortSignal.timeout(fetchTimeoutMs),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengis/fastify-table",
-  "version": "2.2.12",
+  "version": "2.3.1",
   "type": "module",
   "description": "core-plugins",
   "keywords": [