@opengis/fastify-table 2.5.53 → 3.0.0-beta.0

package/dist/server/plugins/upload/s3.js

@@ -1,464 +0,0 @@
-import path from "node:path";
-import { CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, HeadObjectCommand, ListPartsCommand, AbortMultipartUploadCommand, } from "@aws-sdk/client-s3";
-import config from "../../../config.js";
-import s3Client from "../file/providers/s3/client.js";
-import rclient from "../redis/client.js";
-import logger from "../logger/getLogger.js";
-import inspectContent from "../file/utils/inspectContent.js";
-import { ConflictError } from "../../../errors.js";
-const uploads = new Map();
-const resumableUploads = new Map();
-// non-redis upload timeout handlers
-const uploadTimeouts = new Map();
-const UPLOAD_TTL_SECONDS = 60 * 60;
-// expire gracefully to give cleanup worker time to abort stale uploads
-const CLEANUP_GRACE_SECONDS = 60 * 60 * 24;
-const REDIS_TTL = UPLOAD_TTL_SECONDS + CLEANUP_GRACE_SECONDS;
-const uploadMetaKey = (id) => `upload:${id}:meta`;
-const uploadExpireKey = (id) => `upload:${id}:expires`;
-const resumableKey = (key) => `resumable:${key}`;
-function clearUploadTimeout(id) {
-    const timeout = uploadTimeouts.get(id);
-    if (timeout) {
-        clearTimeout(timeout);
-        uploadTimeouts.delete(id);
-    }
-}
-async function scheduleNonRedisCleanup(id, s3 = s3Client) {
-    if (!s3)
-        throw new Error("No S3 client");
-    if (config.redis)
-        return;
-    clearUploadTimeout(id);
-    const timeout = setTimeout(async () => {
-        try {
-            const meta = uploads.get(id);
-            if (!meta)
-                return;
-            try {
-                await s3.send(new AbortMultipartUploadCommand({
-                    Bucket: meta.Bucket,
-                    Key: meta.Key,
-                    UploadId: id,
-                }));
-            }
-            catch (err) {
-                console.error("AbortMultipartUpload failed", {
-                    uploadId: id,
-                    Bucket: meta.Bucket,
-                    Key: meta.Key,
-                    error: err?.message,
-                });
-                logger.file("upload/abort/error", {
-                    uploadId: id,
-                    Bucket: meta.Bucket,
-                    Key: meta.Key,
-                    error: err?.message,
-                });
-            }
-            uploads.delete(id);
-            if (resumableUploads.get(meta.Key) === id) {
-                resumableUploads.delete(meta.Key);
-            }
-        }
-        catch (err) {
-            console.error(err);
-            logger.error(err);
-        }
-        finally {
-            clearUploadTimeout(id);
-        }
-    }, UPLOAD_TTL_SECONDS * 1000);
-    // don't keep node process alive
-    timeout.unref?.();
-    uploadTimeouts.set(id, timeout);
-}
-// used via onClose hook to avoid memory leaks
-export async function cleanupNonRedisUploads(s3 = s3Client) {
-    if (!s3)
-        throw new Error("No S3 client");
-    if (config.redis)
-        return;
-    console.log("cleanupNonRedisUploads start");
-    const entries = [...uploads.entries()];
-    for (const [id, meta] of entries) {
-        try {
-            await s3.send(new AbortMultipartUploadCommand({
-                Bucket: meta.Bucket,
-                Key: meta.Key,
-                UploadId: id,
-            }));
-        }
-        catch (err) {
-            console.error("AbortMultipartUpload failed", {
-                uploadId: id,
-                Bucket: meta.Bucket,
-                Key: meta.Key,
-                error: err?.message,
-            });
-            logger.file("upload/abort/error", {
-                uploadId: id,
-                Bucket: meta.Bucket,
-                Key: meta.Key,
-                error: err?.message,
-            });
-        }
-        clearUploadTimeout(id);
-        uploads.delete(id);
-        if (resumableUploads.get(meta.Key) === id) {
-            resumableUploads.delete(meta.Key);
-        }
-    }
-    console.log("open unfinished multipart uploads closed", entries.length);
-}
-export async function saveUploadMeta(id, meta, ttl = UPLOAD_TTL_SECONDS) {
-    if (!config.redis)
-        return;
-    const expiresAt = Date.now() + UPLOAD_TTL_SECONDS * 1000;
-    await Promise.all([
-        rclient.set(uploadMetaKey(id), JSON.stringify(meta), "EX", REDIS_TTL),
-        rclient.set(uploadExpireKey(id), String(expiresAt), "EX", ttl),
-        rclient.set(resumableKey(meta.Key), id, "EX", ttl),
-    ]);
-}
-export async function refreshUploadTTL(id, ttl = UPLOAD_TTL_SECONDS, s3 = s3Client) {
-    if (!s3)
-        throw new Error("No S3 client");
-    if (!config.redis) {
-        await scheduleNonRedisCleanup(id, s3);
-        return;
-    }
-    const meta = await getUploadMeta(id);
-    if (!meta)
-        return;
-    const expiresAt = Date.now() + ttl * 1000;
-    await Promise.all([
-        rclient.expire(uploadMetaKey(id), ttl),
-        rclient.set(uploadExpireKey(id), String(expiresAt), "EX", ttl),
-        rclient.expire(resumableKey(meta.Key), ttl),
-    ]);
-}
-export async function deleteUploadMeta(id, key) {
-    clearUploadTimeout(id);
-    await deleteUploadParts(id);
-    if (!config.redis) {
-        const meta = uploads.get(id);
-        uploads.delete(id);
-        const redisKey = key || meta?.Key;
-        if (redisKey && resumableUploads.get(redisKey) === id) {
-            resumableUploads.delete(redisKey);
-        }
-        return;
-    }
-    const redisKey = key || (await getUploadMeta(id))?.Key;
-    const keys = [uploadMetaKey(id), uploadExpireKey(id)];
-    if (redisKey) {
-        keys.push(resumableKey(redisKey));
-    }
-    await rclient.del(keys);
-}
-export async function getUploadMeta(id) {
-    if (!config.redis) {
-        return uploads.get(id);
-    }
-    const raw = await rclient.get(uploadMetaKey(id));
-    return raw ? JSON.parse(raw) : undefined;
-}
-export async function getResumableUploadId(key) {
-    if (!config.redis) {
-        return resumableUploads.get(key);
-    }
-    return rclient.get(resumableKey(key));
-}
-export async function cleanupStaleUploads(s3 = s3Client) {
-    if (!s3)
-        throw new Error("No S3 client");
-    if (!config.redis) {
-        await cleanupNonRedisUploads(s3);
-        return;
-    }
-    let cursor = "0";
-    do {
-        const [nextCursor, keys] = await rclient.scan(cursor, "MATCH", "upload:*:expires", "COUNT", 100);
-        cursor = nextCursor;
-        for (const expireKey of keys) {
-            try {
-                const expiresAt = Number(await rclient.get(expireKey));
-                if (!expiresAt || expiresAt > Date.now()) {
-                    continue;
-                }
-                const uploadId = expireKey
-                    .replace("upload:", "")
-                    .replace(":expires", "");
-                const meta = await getUploadMeta(uploadId);
-                if (!meta) {
-                    await rclient.del(expireKey);
-                    continue;
-                }
-                const { Bucket, Key } = meta;
-                try {
-                    await s3.send(new AbortMultipartUploadCommand({
-                        Bucket,
-                        Key,
-                        UploadId: uploadId,
-                    }));
-                }
-                catch (err) {
-                    console.error("AbortMultipartUpload failed", {
-                        uploadId,
-                        Bucket,
-                        Key,
-                        error: err?.message,
-                    });
-                    logger.file("upload/abort/error", {
-                        uploadId,
-                        Bucket,
-                        Key,
-                        error: err?.message,
-                    });
-                }
-                await deleteUploadMeta(uploadId, Key);
-            }
-            catch (err) {
-                console.error(err);
-                logger.error(err);
-            }
-        }
-    } while (cursor !== "0");
-}
-export async function findUpload({ id }, s3 = s3Client) {
-    if (!s3)
-        throw new Error("No S3 client");
-    const cachedInfo = await getUploadMeta(id);
-    const { Key, Bucket, fileSize: cacheSize, ContentType: cacheContentType, } = cachedInfo || {};
-    const parts = await s3
-        .send(new ListPartsCommand({
-        Bucket,
-        Key,
-        UploadId: id,
-    }))
-        .catch(() => null);
-    return {
-        UploadId: parts ? id : undefined,
-        Key,
-        Bucket,
-        fileSize: cacheSize,
-        ContentType: cacheContentType,
-        uploaded: parts?.Parts?.reduce((sum, part) => sum + (part.Size || 0), 0) || 0,
-        parts: parts?.Parts?.map((p) => ({
-            ETag: p.ETag,
-            PartNumber: p.PartNumber,
-        })) || [],
-    };
-}
-// todo: optional - keep originalFileName, else - id + extension
-//  + implement rotateFilenameS3 logic, like const filename = await rotateFilenameS3({ Name: originalFilename, Bucket, Prefix }, s3Client);
-// ! if originalFileName - check names with spaces - s3 mights throw error - encode request or at upload?
-export async function startUpload({ Bucket, Key, ContentType, fileSize, }, s3 = s3Client) {
-    if (!s3)
-        throw new Error("No S3 client");
-    const cachedUploadId = await getResumableUploadId(Key);
-    const cachedMeta = cachedUploadId
-        ? await getUploadMeta(cachedUploadId)
-        : undefined;
-    if (cachedUploadId && cachedMeta) {
-        await refreshUploadTTL(cachedUploadId, UPLOAD_TTL_SECONDS, s3);
-        return {
-            Key,
-            Bucket: cachedMeta.Bucket,
-            UploadId: cachedUploadId,
-            fileSize: cachedMeta.fileSize,
-            extension: cachedMeta.extension,
-            ContentType: cachedMeta.ContentType,
-            cache: true,
-        };
-    }
-    const res = await s3.send(new CreateMultipartUploadCommand({
-        Bucket,
-        Key,
-        ContentType,
-    }));
-    const extension = path.extname(Key).substring(1);
-    const meta = {
-        Key,
-        Bucket,
-        fileSize,
-        extension,
-        ContentType,
-    };
-    if (config.redis) {
-        await saveUploadMeta(res.UploadId, meta);
-    }
-    else {
-        uploads.set(res.UploadId, meta);
-        resumableUploads.set(Key, res.UploadId);
-        await scheduleNonRedisCleanup(res.UploadId, s3);
-    }
-    return {
-        UploadId: res.UploadId,
-    };
-}
-const uploadPartsKey = (id) => `upload:${id}:parts`;
-// non-redis in-memory parts cache
-const uploadParts = new Map();
-export async function savePart(id, part) {
-    if (!config.redis) {
-        const existing = uploadParts.get(id) || [];
-        // overwrite existing part number (S3 semantics)
-        const filtered = existing.filter((p) => p.PartNumber !== part.PartNumber);
-        filtered.push(part);
-        uploadParts.set(id, filtered);
-        return;
-    }
-    // use redis hash keyed by part number
-    await rclient.hset(uploadPartsKey(id), String(part.PartNumber), JSON.stringify(part));
-    // keep ttl aligned with upload meta
-    await rclient.expire(uploadPartsKey(id), REDIS_TTL);
-}
-export async function getStoredParts(id) {
-    if (!config.redis) {
-        return (uploadParts.get(id) || []).sort((a, b) => a.PartNumber - b.PartNumber);
-    }
-    const parts = await rclient.hgetall(uploadPartsKey(id));
-    return Object.values(parts)
-        .map((v) => JSON.parse(v))
-        .sort((a, b) => a.PartNumber - b.PartNumber);
-}
-export async function deleteUploadParts(id) {
-    if (!config.redis) {
-        uploadParts.delete(id);
-        return;
-    }
-    await rclient.del(uploadPartsKey(id));
-}
-export async function uploadChunk({ id, chunk, partNumber, }, s3 = s3Client) {
-    if (!s3)
-        throw new Error("No S3 client");
-    const upload = await findUpload({ id }, s3);
-    if (!upload?.UploadId) {
-        throw new Error("upload not found");
-    }
-    if (upload.parts.some((p) => p.PartNumber === partNumber)) {
-        throw new Error(`Part ${partNumber} already uploaded`);
-    }
-    if (partNumber === 1) {
-        const { isValid } = await inspectContent(chunk, path.extname(upload.Key).substring(1));
-        if (!isValid) {
-            throw ConflictError("file type spoofing attempt detected");
-        }
-    }
-    const res = await s3.send(new UploadPartCommand({
-        Bucket: upload.Bucket,
-        Key: upload.Key,
-        UploadId: upload.UploadId,
-        PartNumber: partNumber,
-        Body: chunk,
-    }));
-    await savePart(id, {
-        ETag: res.ETag,
-        PartNumber: partNumber,
-    });
-    await refreshUploadTTL(id, UPLOAD_TTL_SECONDS, s3);
-    return upload;
-}
-export async function finishUpload({ id, }, s3 = s3Client) {
-    if (!s3)
-        throw new Error("No S3 client");
-    const redisCache = await getUploadMeta(id);
-    const { fileSize: cachedSize, Key, Bucket } = redisCache || {};
-    const size = cachedSize ? Number(cachedSize) : undefined;
-    // const upload = await findUpload({ id }, s3);
-    // if (!upload?.uploaded) {
-    //   throw new Error("File not uploaded");
-    // }
-    // if (size && size !== upload.uploaded) {
-    //   throw new Error("File size mismatch");
-    // }
-    const parts = await getStoredParts(id);
-    await s3.send(new CompleteMultipartUploadCommand({
-        Bucket,
-        Key,
-        UploadId: id,
-        MultipartUpload: {
-            Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber),
-            // Parts: upload.parts.sort(
-            //   (a: any, b: any) => a.PartNumber - b.PartNumber
-            // ),
-        },
-    }));
-    await deleteUploadMeta(id, Key);
-    return {
-        finished: true,
-        // uploaded: upload.uploaded,
-        uploaded: size,
-        filepath: Key.replace(config.folder, ""),
-    };
-}
-export async function abortUpload({ id, }, s3 = s3Client) {
-    if (!s3)
-        throw new Error("No S3 client");
-    const meta = await getUploadMeta(id);
-    if (!meta) {
-        return {
-            aborted: false,
-        };
-    }
-    await s3.send(new AbortMultipartUploadCommand({
-        Bucket: meta.Bucket,
-        Key: meta.Key,
-        UploadId: id,
-    }));
-    await deleteUploadMeta(id, meta.Key);
-    return {
-        aborted: true,
-    };
-}
-export async function getUploadStatus({ id, Key, Bucket, }, s3 = s3Client) {
-    if (!s3)
-        throw new Error("No S3 client");
-    const redisCache = id ? await getUploadMeta(id) : undefined;
-    const { fileSize: cachedSize, Key: redisKey, Bucket: redisBucket, } = redisCache || {};
-    if (!redisKey && Key && Bucket) {
-        const res = await s3
-            .send(new HeadObjectCommand({
-            Bucket,
-            Key,
-        }))
-            .catch(() => false);
-        return {
-            exists: !!res,
-            uploaded: res?.ContentLength,
-        };
-    }
-    // const parts = id ? await getStoredParts(id) : { Parts: [] };
-    const parts = await s3
-        .send(new ListPartsCommand({
-        Bucket: redisBucket,
-        Key: redisKey,
-        UploadId: id,
-    }))
-        .catch(() => ({
-        Parts: [],
-    }));
-    const res = id && !parts.Parts?.length
-        ? await s3
-            .send(new HeadObjectCommand({
-            Bucket: redisBucket,
-            Key: redisKey,
-        }))
-            .catch(() => false)
-        : false;
-    return {
-        exists: !!res || !!parts?.Parts?.length,
-        finished: !!res,
-        uploaded: res
-            ? res.ContentLength
-            : parts?.Parts?.reduce((acc, curr) => acc + (curr.Size || 0), 0) || 0,
-        id,
-        size: cachedSize ? Number(cachedSize) : undefined,
-        parts: parts?.Parts?.map((p) => ({
-            ETag: p.ETag,
-            PartNumber: p.PartNumber,
-        })) || [],
-    };
-}

package/dist/server/plugins/upload/s3minio.d.ts

@@ -1,59 +0,0 @@
-export declare function close(): void;
-export declare function startUpload({ Bucket, Key, fileSize: size, ContentType, }: {
-    Bucket?: string;
-    Key: string;
-    fileSize: number;
-    ContentType: string;
-}): Promise<{
-    UploadId: any;
-}>;
-export declare function findUpload({ Bucket, Key, id: UploadId, }: {
-    Bucket?: string;
-    Key?: string;
-    id?: string;
-}): Promise<{
-    UploadId: string;
-    fileSize: any;
-} | {
-    UploadId: any;
-    fileSize?: undefined;
-}>;
-export declare function uploadChunk({ id: UploadId, chunk: Body, partNumber: PartNumber, }: {
-    id: string;
-    chunk: any;
-    partNumber: number;
-}): Promise<{
-    uploaded: any;
-    ETag: any;
-    PartNumber: number;
-}>;
-export declare function finishUpload({ id: UploadId }: {
-    id: string;
-}): Promise<{
-    finished: boolean;
-    uploaded: any;
-    filepath: any;
-}>;
-export declare function getUploadStatus({ id, Bucket, Key, }: {
-    id?: string;
-    Key?: string;
-    Bucket?: string;
-}): Promise<{
-    exists: boolean;
-    finished: boolean;
-    uploaded: any;
-    id: any;
-    size: number | undefined;
-    parts: {
-        ETag: any;
-        PartNumber: any;
-    }[];
-}>;
-export declare function cleanupMultipartUploads({ Bucket, }?: {
-    Bucket: string;
-}): Promise<PromiseSettledResult<{
-    Key: any;
-    UploadId: any;
-}>[]>;
-export declare function deleteUploadMeta(id: string): Promise<void>;
-//# sourceMappingURL=s3minio.d.ts.map

package/dist/server/plugins/upload/s3minio.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"s3minio.d.ts","sourceRoot":"","sources":["../../../../server/plugins/upload/s3minio.ts"],"names":[],"mappings":"AAuBA,wBAAgB,KAAK,SAIpB;AAwBD,wBAAsB,WAAW,CAAC,EAChC,MAA2C,EAC3C,GAAG,EACH,QAAQ,EAAE,IAAI,EACd,WAAW,GACZ,EAAE;IACD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;CACrB;;GAqBA;AAED,wBAAsB,UAAU,CAAC,EAC/B,MAA2C,EAC3C,GAAG,EACH,EAAE,EAAE,QAAQ,GACb,EAAE;IACD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,EAAE,CAAC,EAAE,MAAM,CAAC;CACb;;;;;;GAuBA;AAED,wBAAsB,WAAW,CAAC,EAChC,EAAE,EAAE,QAAQ,EACZ,KAAK,EAAE,IAAI,EACX,UAAU,EAAE,UAAU,GACvB,EAAE;IACD,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,GAAG,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;CACpB;;;;GA8BA;AAED,wBAAsB,YAAY,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE;;;;GAwClE;AAED,wBAAsB,eAAe,CAAC,EACpC,EAAE,EACF,MAAM,EACN,GAAG,GACJ,EAAE;IACD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;;;;;;;;;;GA4CA;AAED,wBAAsB,uBAAuB,CAC3C,EACE,MAAM,GACP,GAAE;IACD,MAAM,EAAE,MAAM,CAAC;CACiC;;;MAqCnD;AAED,wBAAsB,gBAAgB,CAAC,EAAE,EAAE,MAAM,iBAEhD"}