@leogps/file-uploader 2.0.0

package/src/model/progress_utils.ts

@@ -0,0 +1,17 @@
+import {Progress} from './progress';
+
+export class ProgressUtils {
+    public static calculateTransferRate(progress: Progress): number {
+      const transferSamples = progress.transferSamples;
+      if (!transferSamples || transferSamples.length < 2) {
+        return 0;
+      }
+      const first = transferSamples[0];
+      const last = transferSamples[transferSamples.length - 1];
+
+      const dataSize = last.bytesReceived - first.bytesReceived;
+      const timeIntervalSeconds = (last.timestamp - first.timestamp) / 1000;
+
+      return dataSize / timeIntervalSeconds;
+    }
+  }

package/src/routes/uploadChunk.ts

@@ -0,0 +1,125 @@
+import { Router, Request, Response } from "express";
+import formidable, { File } from "formidable";
+import fs from "fs/promises";
+import path from "path";
+import crypto from "crypto";
+import {
+    uploadsProgressMap,
+    getUploadsDir,
+    throttledBroadcaster
+} from "../globals";
+import { FileTransferProgress } from "../model/progress";
+
+export const router = Router();
+
+router.post("/", (req: Request, res: Response) => {
+    (async () => {
+        const fileId = req.query.fileId as string;
+        const chunkIndex = Number(req.query.chunkIndex);
+        const clientHash = req.query.hash as string;
+
+        if (!fileId || !uploadsProgressMap.has(fileId)) {
+            return res.status(400).json({ msg: "Invalid or unknown fileId" });
+        }
+        if (isNaN(chunkIndex) || chunkIndex < 0) {
+            return res.status(400).json({ msg: "Invalid chunk index" });
+        }
+        if (!clientHash) {
+            return res.status(400).json({ msg: "Missing SHA-1 hash for chunk" });
+        }
+
+        const progress = uploadsProgressMap.get(fileId)! as FileTransferProgress;
+
+        const uploadDir = getUploadsDir();
+        await fs.mkdir(uploadDir, { recursive: true });
+
+        const form = formidable({
+            multiples: false,
+            keepExtensions: true,
+            uploadDir
+        });
+
+        let receivedFile: File | null = null;
+        let errorDuringFileWrite: any = null;
+
+        form.on("file", (_: string, file: File) => {
+            (async () => {
+                receivedFile = file;
+
+                try {
+                    const finalPath = path.join(uploadDir, progress.fileName!);
+
+                    // Sparse file creation
+                    try {
+                        await fs.access(finalPath);
+                    } catch {
+                        const fh = await fs.open(finalPath, "w");
+                        await fh.truncate(progress.bytesExpected);
+                        await fh.close();
+                    }
+
+                    const chunkBuffer = await fs.readFile(file.filepath);
+                    const offset = chunkIndex * progress.chunkSize!;
+
+                    const fd = await fs.open(finalPath, "r+");
+                    await fd.write(chunkBuffer, 0, chunkBuffer.length, offset);
+
+                    const verifyBuffer = Buffer.alloc(chunkBuffer.length);
+                    await fd.read(verifyBuffer, 0, chunkBuffer.length, offset);
+                    await fd.close();
+
+                    const hash = crypto.createHash("sha1").update(verifyBuffer).digest("hex");
+                    if (hash !== clientHash) {
+                        errorDuringFileWrite = {
+                            status: 400,
+                            body: {
+                                msg: "Chunk hash mismatch",
+                                expected: clientHash,
+                                got: hash
+                            }
+                        };
+                    } else {
+                        progress.uploadingChunks.delete(chunkIndex);
+                        progress.addUploadedChunk(chunkIndex);
+                        progress.markSample();
+                    }
+                    throttledBroadcaster();
+
+                    await fs.unlink(file.filepath);
+                } catch (err) {
+                    console.error("Chunk write error:", err);
+                    errorDuringFileWrite = {
+                        status: 500,
+                        body: { msg: "Internal write error", error: err }
+                    };
+                }
+            })()
+        });
+
+        form.parse(req, (err) => {
+            if (err) {
+                console.error("Form parse error:", err);
+                return res.status(500).json({ msg: "Error receiving chunk", error: err });
+            }
+
+            if (errorDuringFileWrite) {
+                return res.status(errorDuringFileWrite.status).json(errorDuringFileWrite.body);
+            }
+
+            if (!receivedFile) {
+                return res.status(400).json({ msg: "No chunk received" });
+            }
+
+            console.log(`uploaded-chunk: ${chunkIndex} for ${progress.fileName}`);
+            return res.json({
+                msg: "Chunk uploaded successfully",
+                fileId,
+                chunkIndex,
+                hashMatches: true,
+                bytesReceived: progress.bytesReceived,
+                bytesExpected: progress.bytesExpected,
+                uploadedChunks: Array.from(progress.uploadedChunks)
+            });
+        });
+    })()
+});

package/src/routes/uploadComplete.ts

@@ -0,0 +1,53 @@
+import {Request, Response, Router} from 'express';
+import {getProgressWriter, progresses, uploadsProgressMap} from '../globals';
+import {FileTransferProgress, UploadStatus} from "../model/progress";
+
+export const router = Router();
+
+// POST /upload/complete?fileId=UUID
+router.post('/', (req: Request, res: Response) => {
+    const fileId = req.query.fileId as string;
+    const markUploadFailed: boolean = req.query.markUploadFailed === "true";
+
+    if (!fileId || !uploadsProgressMap.has(fileId)) {
+        return res.status(400).json({ msg: 'Invalid or unknown fileId' });
+    }
+
+    const progress: FileTransferProgress = uploadsProgressMap.get(fileId)! as FileTransferProgress;
+    progress.lastState = UploadStatus.FINISHING;
+
+    // Ensure uploadedChunks and totalChunks exist
+    if (!progress.uploadedChunks) {
+        progress.resetUploadedChunks();
+    }
+    if (!progress.totalChunks && progress.chunkSize && progress.bytesExpected) {
+        progress.totalChunks = Math.ceil(progress.bytesExpected / progress.chunkSize);
+    }
+
+    // Check all chunks uploaded
+    if (progress.uploadedChunks.size !== progress.totalChunks) {
+        if (markUploadFailed) {
+            console.log(`Marking upload failed for file ${progress.fileName} ${progress.uuid}`);
+            progress.lastState = UploadStatus.FAILED;
+        }
+        getProgressWriter().writeProgress(progresses);
+        return res.status(400).json({
+            msg: 'File incomplete',
+            uploadedChunks: Array.from(progress.uploadedChunks),
+            totalChunks: progress.totalChunks
+        });
+    }
+
+    progress.completed = Date.now();
+    progress.lastState = UploadStatus.COMPLETE;
+    getProgressWriter().writeProgress(progresses);
+
+    res.json({
+        msg: 'File upload complete',
+        fileName: progress.fileName,
+        savedLocation: progress.savedLocation,
+        bytesReceived: progress.bytesReceived,
+        totalChunks: progress.totalChunks,
+        chunkSize: progress.chunkSize
+    });
+});

package/src/routes/uploadInit.ts

@@ -0,0 +1,83 @@
+import { Router, Request, Response } from 'express';
+import { v4 as uuidv4 } from 'uuid';
+import {uploadsProgressMap, progresses, getUploadsDir, MAX_PARALLEL_CHUNK_UPLOADS, MAX_CHUNK_SIZE} from '../globals';
+import { FileTransferProgress } from '../model/progress';
+import path from "path";
+import fs from "fs";
+
+export const router = Router();
+
+/**
+ * POST /upload/init
+ * Accepts fileName and fileSize either in JSON body or query parameters.
+ * Returns a fileId to be used for chunked uploads.
+ */
+router.post('/', (req: Request, res: Response) => {
+    console.log("Req body: " + JSON.stringify(req.body));
+    let fileName: string | undefined = req.body?.fileName;
+    let fileSize: number | undefined = req.body?.fileSize;
+
+    // Fallback to query params
+    if (!fileName || !fileSize) {
+        fileName = req.query.fileName as string;
+        fileSize = parseInt(req.query.fileSize as string, 10);
+    }
+
+    console.log("File name: " + fileName);
+    console.log("File size: " + fileSize);
+    if (!fileName || !fileSize || isNaN(fileSize)) {
+        return res.status(400).json({ msg: 'Missing or invalid fileName/fileSize' });
+    }
+
+    const finalPath = path.join(getUploadsDir(), fileName);
+    let progress: FileTransferProgress;
+    let fileId: string;
+
+    // Check if file already exists
+    if (fs.existsSync(finalPath)) {
+        // Try to find existing progress in memory
+        const existingProgress = Array.from(uploadsProgressMap.values())
+            .find(p => p.fileName === fileName && p.bytesExpected === fileSize) as FileTransferProgress | undefined;
+
+        if (existingProgress) {
+            progress = existingProgress;
+            fileId = progress.uuid!;
+        } else {
+            // File exists but no memory entry (server restarted)
+            fileId = uuidv4();
+            progress = new FileTransferProgress(fileId, Date.now());
+            progress.fileName = fileName;
+            progress.bytesExpected = fileSize;
+            progress.chunkSize = 5 * 1024 * 1024;
+            progress.totalChunks = Math.ceil(fileSize / progress.chunkSize);
+            progress.bytesReceived = fs.statSync(finalPath).size; // resume
+            progress.resetUploadedChunks();
+            uploadsProgressMap.set(fileId, progress);
+            progresses.push(progress);
+        }
+    } else {
+        // File does not exist, create new progress
+        console.log(`creating file ${finalPath}`);
+        fs.writeFileSync(finalPath, Buffer.alloc(0));
+        fileId = uuidv4();
+        progress = new FileTransferProgress(fileId, Date.now());
+        progress.fileName = fileName;
+        progress.bytesExpected = fileSize;
+        progress.chunkSize = MAX_CHUNK_SIZE;
+        progress.totalChunks = Math.ceil(fileSize / progress.chunkSize);
+        progress.resetUploadedChunks();
+        uploadsProgressMap.set(fileId, progress);
+        progresses.push(progress);
+    }
+    progress.savedLocation = finalPath;
+
+    progress.resetVerificationCount();
+    progress.uploadingChunks = new Set<number>();
+    res.json({
+        fileId,
+        chunkSize: progress.chunkSize,
+        totalChunks: progress.totalChunks,
+        maxParallel: MAX_PARALLEL_CHUNK_UPLOADS,
+        bytesReceived: progress.bytesReceived || 0 // client can skip uploaded chunks
+    });
+});

package/src/routes/uploadStatus.ts

@@ -0,0 +1,137 @@
+import {Request, Response, Router} from 'express';
+import fs from 'fs';
+import crypto from 'crypto';
+import {throttledBroadcaster, uploadsProgressMap} from '../globals';
+import {FileTransferProgress, UploadStatus} from "../model/progress";
+
+export const router = Router();
+
+/* ------------------------ Helper: uniform error JSON ------------------------ */
+const sendError = (res: Response, code: number, msg: string): Response =>
+    res.status(code).json({ msg });
+
+const hashMatched = (fileId: string,
+                     chunkIndex: number,
+                     res: Response,
+                     progress: FileTransferProgress): Response => {
+    console.log(`✅ Hash match for chunk ${chunkIndex} of file ${fileId}`);
+    progress.addUploadedChunk(chunkIndex);
+    progress.chunkVerified(chunkIndex);
+    throttledBroadcaster();
+    return res.json({
+        fileId,
+        chunkIndex,
+        hashMatches: true,
+        bytesReceived: progress.bytesReceived,
+        bytesExpected: progress.bytesExpected
+    });
+}
+
+const hashMismatched = (fileId: string,
+                        chunkIndex: number,
+                        res: Response,
+                        progress: FileTransferProgress): Response => {
+    console.log(`❌ Hash mismatch for chunk ${chunkIndex} of file ${fileId}`);
+    progress.uploadingChunks.add(chunkIndex);
+    progress.chunkVerified(chunkIndex);
+    throttledBroadcaster();
+    return res.json({
+        fileId,
+        chunkIndex,
+        hashMatches: false,
+        bytesReceived: progress.bytesReceived,
+        bytesExpected: progress.bytesExpected
+    });
+}
+
+/* ------------------------ Helper: read chunk from disk ---------------------- */
+const readChunk = (filePath: string,
+                   start: number,
+                   length: number): Buffer<ArrayBuffer> | null => {
+    const fd = fs.openSync(filePath, 'r');
+
+    const stats = fs.fstatSync(fd);
+    const fileSize = stats.size;
+
+    // Chunk beyond EOF → return null (means "not written")
+    if (start >= fileSize) {
+        fs.closeSync(fd);
+        return null;
+    }
+
+    const realEnd = Math.min(start + length, fileSize);
+    const realLength = realEnd - start;
+
+    const buffer = Buffer.allocUnsafe(realLength);
+    fs.readSync(fd, buffer, 0, realLength, start);
+    fs.closeSync(fd);
+
+    return buffer;
+}
+
+// GET /upload/status?fileId=UUID&chunkIndex=N&chunkSize=BYTES&hash=SHA1
+router.get('/', (req: Request, res: Response) => {
+    // Cast query params to string explicitly
+    const fileId = (req.query.fileId as string | undefined);
+    const chunkIdxStr = req.query.chunkIndex as string | undefined;
+    const chunkSizeStr = req.query.chunkSize as string | undefined;
+    const clientHash = req.query.hash as string | undefined;
+
+    if (!fileId) {
+        return res.status(400).json({ msg: `Missing query parameter fileId`});
+    }
+    if (!chunkIdxStr) {
+        return res.status(400).json({ msg: `Missing query parameter chunkIndex`});
+    }
+    if (!chunkSizeStr) {
+        return res.status(400).json({ msg: `Missing query parameter chunkSize`});
+    }
+    if (!clientHash) {
+        return res.status(400).json({ msg: `Missing query parameter hash`});
+    }
+
+    const chunkIndex = parseInt(chunkIdxStr, 10);
+    const chunkSize = parseInt(chunkSizeStr, 10);
+
+    if (!uploadsProgressMap.has(fileId)) {
+        return res.status(400).json({ msg: 'Invalid or unknown fileId' });
+    }
+
+    const progress = uploadsProgressMap.get(fileId)! as FileTransferProgress;
+    progress.lastState = UploadStatus.UPLOADING;
+    throttledBroadcaster();
+
+    if (!chunkSize || chunkSize <= 0 || !progress.bytesExpected) {
+        return sendError(res, 400, "Invalid chunk size");
+    }
+
+    const totalChunks = Math.ceil(progress.bytesExpected / chunkSize);
+    if (isNaN(chunkIndex) || chunkIndex < 0 || chunkIndex >= totalChunks) {
+        return sendError(res, 400, "Invalid chunk index");
+    }
+
+    const filePath = progress.savedLocation;
+    if (!filePath || !fs.existsSync(filePath)) {
+        return sendError(res, 404, "File not found");
+    }
+
+    // ---- Compute read boundaries ----
+    const start = chunkIndex * chunkSize;
+    const length = Math.min(chunkSize, progress.bytesExpected - start);
+
+    // ---- Read chunk ----
+    const buffer = readChunk(filePath, start, length);
+    if (!buffer) {
+        // Chunk not written yet (beyond EOF)
+        return hashMismatched(fileId, chunkIndex, res, progress);
+    }
+
+    // ---- Hash verification ----
+    const serverHash = crypto.createHash('sha1').update(buffer).digest('hex');
+    const hashMatches = serverHash === clientHash;
+    if (hashMatches) {
+        return hashMatched(fileId, chunkIndex, res, progress);
+
+    }
+    return hashMismatched(fileId, chunkIndex, res, progress);
+});

package/src/service/progress_writer.ts

@@ -0,0 +1,52 @@
+import {FileTransferProgress, Progress, TransferSample} from "../model/progress";
+import { Server } from "socket.io";
+
+export class ProgressWriter {
+    io: Server;
+
+    constructor(io: Server) {
+        this.io = io;
+    }
+
+    public writeProgress(progresses: Progress[]) {
+        // Clone progresses to safely emit without sending full transferSamples
+        const progressesEmittable: FileTransferProgress[] = progresses.map(p => p as FileTransferProgress).map(p => {
+            const cloned = this.cloneObjectExceptField(p, "transferSamples") as any;
+
+            // Only include first and last sample for minimal data
+            cloned.transferSamples = [] as TransferSample[];
+            if (p.transferSamples && p.transferSamples.length >= 2) {
+                cloned.transferSamples.push(p.transferSamples[0]);
+                cloned.transferSamples.push(p.transferSamples[p.transferSamples.length - 1]);
+            }
+
+            // Ensure uploadedChunks, verifyingChunks, and uploadingChunks are serialized as arrays
+            cloned.uploadedChunks = p.uploadedChunks instanceof Set ? Array.from(p.uploadedChunks)
+                : Array.isArray(p.uploadedChunks) ? p.uploadedChunks : [];
+
+            cloned.uploadingChunks = p.uploadingChunks instanceof Set ? Array.from(p.uploadingChunks)
+                : Array.isArray(p.uploadingChunks) ? p.uploadingChunks : [];
+
+            return cloned as FileTransferProgress;
+        });
+
+        // Emit to all connected clients
+        this.io.emit("progresses", progressesEmittable);
+    }
+
+    private cloneObjectExceptField<T extends Record<string, any>, K extends keyof T>(
+        obj: T,
+        fieldToExclude: Extract<keyof T, string>
+    ): Omit<T, K> {
+        const clonedObject = {} as Omit<T, K>;
+
+        for (const key in obj) {
+            if (obj.hasOwnProperty(key) && key !== fieldToExclude) {
+                const c: Record<string, any> = clonedObject;
+                c[key.toString()] = obj[key];
+            }
+        }
+
+        return clonedObject;
+    }
+}