@leogps/file-uploader 2.0.0

package/dist/index.js.LICENSE.txt

@@ -0,0 +1,273 @@
+/*
+object-assign
+(c) Sindre Sorhus
+@license MIT
+*/
+
+/*!
+ * accepts
+ * Copyright(c) 2014 Jonathan Ong
+ * Copyright(c) 2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * base64id v0.1.0
+ */
+
+/*!
+ * body-parser
+ * Copyright(c) 2014 Jonathan Ong
+ * Copyright(c) 2014-2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * body-parser
+ * Copyright(c) 2014-2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * bytes
+ * Copyright(c) 2012-2014 TJ Holowaychuk
+ * Copyright(c) 2015 Jed Watson
+ * MIT Licensed
+ */
+
+/*!
+ * content-disposition
+ * Copyright(c) 2014-2017 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * content-type
+ * Copyright(c) 2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * cookie
+ * Copyright(c) 2012-2014 Roman Shtylman
+ * Copyright(c) 2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * depd
+ * Copyright(c) 2014-2018 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * destroy
+ * Copyright(c) 2014 Jonathan Ong
+ * Copyright(c) 2015-2022 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * ee-first
+ * Copyright(c) 2014 Jonathan Ong
+ * MIT Licensed
+ */
+
+/*!
+ * encodeurl
+ * Copyright(c) 2016 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * escape-html
+ * Copyright(c) 2012-2013 TJ Holowaychuk
+ * Copyright(c) 2015 Andreas Lubbe
+ * Copyright(c) 2015 Tiancheng "Timothy" Gu
+ * MIT Licensed
+ */
+
+/*!
+ * etag
+ * Copyright(c) 2014-2016 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * express
+ * Copyright(c) 2009-2013 TJ Holowaychuk
+ * Copyright(c) 2013 Roman Shtylman
+ * Copyright(c) 2014-2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * express
+ * Copyright(c) 2009-2013 TJ Holowaychuk
+ * Copyright(c) 2014-2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * finalhandler
+ * Copyright(c) 2014-2022 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * forwarded
+ * Copyright(c) 2014-2017 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * fresh
+ * Copyright(c) 2012 TJ Holowaychuk
+ * Copyright(c) 2016-2017 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * http-errors
+ * Copyright(c) 2014 Jonathan Ong
+ * Copyright(c) 2016 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * media-typer
+ * Copyright(c) 2014 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * merge-descriptors
+ * Copyright(c) 2014 Jonathan Ong
+ * Copyright(c) 2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * methods
+ * Copyright(c) 2013-2014 TJ Holowaychuk
+ * Copyright(c) 2015-2016 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * mime-db
+ * Copyright(c) 2014 Jonathan Ong
+ * Copyright(c) 2015-2022 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * mime-types
+ * Copyright(c) 2014 Jonathan Ong
+ * Copyright(c) 2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * negotiator
+ * Copyright(c) 2012 Federico Romero
+ * Copyright(c) 2012-2014 Isaac Z. Schlueter
+ * Copyright(c) 2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * on-finished
+ * Copyright(c) 2013 Jonathan Ong
+ * Copyright(c) 2014 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * parseurl
+ * Copyright(c) 2014 Jonathan Ong
+ * Copyright(c) 2014-2017 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * proxy-addr
+ * Copyright(c) 2014-2016 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * range-parser
+ * Copyright(c) 2012-2014 TJ Holowaychuk
+ * Copyright(c) 2015-2016 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * raw-body
+ * Copyright(c) 2013-2014 Jonathan Ong
+ * Copyright(c) 2014-2022 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * send
+ * Copyright(c) 2012 TJ Holowaychuk
+ * Copyright(c) 2014-2022 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * serve-static
+ * Copyright(c) 2010 Sencha Inc.
+ * Copyright(c) 2011 TJ Holowaychuk
+ * Copyright(c) 2014-2016 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * statuses
+ * Copyright(c) 2014 Jonathan Ong
+ * Copyright(c) 2016 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * toidentifier
+ * Copyright(c) 2016 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * type-is
+ * Copyright(c) 2014 Jonathan Ong
+ * Copyright(c) 2014-2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * unpipe
+ * Copyright(c) 2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*!
+ * vary
+ * Copyright(c) 2014-2017 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+/*! https://mths.be/utf8js v2.1.2 by @mathias */
+
+/*! noble-hashes - MIT License (c) 2022 Paul Miller (paulmillr.com) */
+
+/*! safe-buffer. MIT License. Feross Aboukhadijeh <https://feross.org/opensource> */
+
+/**
+ * @license
+ * Lodash <https://lodash.com/>
+ * Copyright OpenJS Foundation and other contributors <https://openjsf.org/>
+ * Released under MIT license <https://lodash.com/license>
+ * Based on Underscore.js 1.8.3 <http://underscorejs.org/LICENSE>
+ * Copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
+ */

package/package-gzip.js

@@ -0,0 +1,30 @@
+import targz from 'targz';
+import fs from 'fs';
+
+const DEST = 'file-uploader.tar.gz';
+const SRC = 'dist';
+
+if (!fs.existsSync(SRC)) {
+    console.error(`Source folder "${SRC}" does not exist. Build first!`);
+    process.exit(1);
+}
+
+// Remove existing archive
+if (fs.existsSync(DEST)) {
+    fs.unlinkSync(DEST);
+    console.log(`Removed existing archive: ${DEST}`);
+}
+
+console.log(`Creating archive from "${SRC}" → ${DEST}...`);
+
+targz.compress({
+    src: SRC,
+    dest: DEST,
+}, (err) => {
+    if (err) {
+        console.error('Packaging failed:', err);
+        process.exit(1);
+    } else {
+        console.log(`Packaging complete: ${DEST}`);
+    }
+});

package/package.json

@@ -0,0 +1,107 @@
+{
+  "name": "@leogps/file-uploader",
+  "version": "2.0.0",
+  "description": "Facilitates file uploader server.",
+  "main": "src/index.ts",
+  "bin": {
+    "file-uploader": "./dist/index.js"
+  },
+  "scripts": {
+    "clean": "rimraf dist file-uploader.tar.gz",
+    "precompile": "eslint -c .eslintrc.js --fix --ext .ts src src-client",
+    "compile-server": "./node_modules/webpack-cli/bin/cli.js --config webpack.config.js",
+    "compile-client-dev": "./node_modules/webpack-cli/bin/cli.js --config webpack-client.dev.js",
+    "compile-client-prod": "./node_modules/webpack-cli/bin/cli.js --config webpack-client.prod.js",
+    "compile-dev": "npm run precompile && npm run compile-server && npm run compile-client-dev",
+    "build-dev": "npm run compile-dev",
+    "compile-prod": "npm run precompile && npm run compile-server && npm run compile-client-prod",
+    "build-prod": "npm run compile-prod",
+    "start": "npm run build-dev && node dist/index.js",
+    "package": "cross-env NODE_ENV=production npm run build-prod && chmod +x ./dist/index.js && node ./package-gzip.js",
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "build:server:once": "npm run compile-server && npm run compile-client-dev",
+    "dev:server": "npm run build:server:once && npm-run-all --parallel nodemon:prod watch:client",
+    "watch:client": "cross-env NODE_ENV=development ./node_modules/webpack-cli/bin/cli.js --config webpack-client.dev.js --watch",
+    "nodemon:prod": "cross-env NODE_ENV=development ./node_modules/node-dev/bin/node-dev ./dist/index.js",
+    "pre-publish": "npm run clean && npm run package"
+  },
+  "nodemonConfig": {
+    "watch": [
+      "./src",
+      "./src-client",
+      "./node_modules"
+    ],
+    "ext": "ts,js,html,css,scss",
+    "delay": 2500
+  },
+  "keywords": [
+    "file",
+    "uploader",
+    "upload",
+    "server",
+    "client"
+  ],
+  "author": "Paul Gundarapu",
+  "license": "ISC",
+  "dependencies": {
+    "@fortawesome/fontawesome-free": "^5.15.4",
+    "bulma": "^1.0.4",
+    "express": "^4.18.2",
+    "formidable": "^3.5.4",
+    "jquery": "^3.7.1",
+    "jquery-blockui": "^2.7.0",
+    "js-sha1": "^0.7.0",
+    "lodash.throttle": "^4.1.1",
+    "moment": "^2.29.4",
+    "mv": "^2.1.1",
+    "pretty-bytes": "^6.1.1",
+    "rimraf": "^5.0.5",
+    "serve-favicon": "^2.5.0",
+    "socket.io": "^4.7.2",
+    "socket.io-client": "^4.7.2",
+    "toastify-js": "^1.12.0",
+    "uuid": "^9.0.1",
+    "yargs": "^17.7.2"
+  },
+  "devDependencies": {
+    "@types/express": "^4.17.20",
+    "@types/formidable": "^3.4.6",
+    "@types/jquery": "^3.5.24",
+    "@types/jquery.blockui": "^0.0.31",
+    "@types/lodash": "^4.14.184",
+    "@types/mv": "^2.1.2",
+    "@types/serve-favicon": "^2.5.6",
+    "@types/toastify-js": "^1.12.2",
+    "@types/uuid": "^9.0.6",
+    "@types/yargs": "^17.0.12",
+    "@typescript-eslint/eslint-plugin": "^8.48.1",
+    "@typescript-eslint/parser": "^8.48.1",
+    "clean-webpack-plugin": "^4.0.0",
+    "concurrently": "^7.3.0",
+    "cross-env": "^7.0.3",
+    "css-loader": "^6.7.1",
+    "css-minimizer-webpack-plugin": "^4.0.0",
+    "eslint": "^9.39.1",
+    "eslint-config-prettier": "^10.1.8",
+    "eslint-plugin-jsdoc": "^61.4.1",
+    "eslint-plugin-prefer-arrow": "^1.2.3",
+    "gulp-sass": "^6.0.1",
+    "html-webpack-plugin": "^5.5.0",
+    "mini-css-extract-plugin": "^2.6.1",
+    "node-dev": "^8.0.0",
+    "nodemon": "^3.0.1",
+    "npm-run-all": "^4.1.5",
+    "postcss-loader": "^7.0.1",
+    "prettier": "^3.7.4",
+    "sass": "^1.94.2",
+    "sass-loader": "^13.3.3",
+    "style-loader": "^3.3.1",
+    "targz": "^1.0.1",
+    "ts-loader": "^9.5.4",
+    "tsc-watch": "^5.0.3",
+    "typescript": "^5.9.3",
+    "webpack": "^5.74.0",
+    "webpack-cli": "^6.0.1",
+    "webpack-merge": "^5.8.0"
+  }
+}

package/src/globals.ts

@@ -0,0 +1,23 @@
+import { Progress } from './model/progress';
+import { ProgressWriter } from './service/progress_writer';
+import _ from "lodash";
+
+export const MAX_CHUNK_SIZE = 2 * 1024 * 1024;
+export const MAX_PARALLEL_CHUNK_UPLOADS = 10;
+export let uploadsDir: string;
+export const progresses: Progress[] = [];
+export const uploadsProgressMap: Map<string, Progress> = new Map();
+let progressWriter: ProgressWriter;
+export const throttleWaitTimeInMillis = 250;
+
+export const setUploadsDir = (dir: string) => { uploadsDir = dir; };
+export const getUploadsDir = () => uploadsDir;
+
+export const setProgressWriter = (writer: ProgressWriter) => {
+    progressWriter = writer;
+};
+export const getProgressWriter = (): ProgressWriter => progressWriter;
+
+export const throttledBroadcaster = _.throttle(() => {
+    getProgressWriter().writeProgress(progresses);
+}, throttleWaitTimeInMillis);

package/src/index.ts

@@ -0,0 +1,87 @@
+
+
+import express, { Express, Request, Response } from 'express';
+import {createServer, Server} from "http"
+import {router as uploadInitRouter} from "./routes/uploadInit";
+import {router as uploadChunkRouter} from "./routes/uploadChunk";
+import {router as uploadCompleteRouter} from "./routes/uploadComplete";
+import {router as uploadStatusRouter} from "./routes/uploadStatus";
+import {ProgressWriter} from "./service/progress_writer";
+import * as socketio from "socket.io";
+import yargs from "yargs";
+import {hideBin} from "yargs/helpers";
+import * as os from 'os';
+import {getProgressWriter, progresses, setProgressWriter, setUploadsDir} from "./globals";
+
+const homedir = os.homedir();
+let port = 8082;
+let uploadsDir = homedir + "/Downloads/uploads/"
+const argv: any = yargs(hideBin(process.argv))
+  .option('upload_location', {
+    alias: 'l',
+    type: 'string',
+    description: 'upload location',
+    default: uploadsDir
+  })
+  .option('port', {
+    alias: 'p',
+    type: 'number',
+    description: 'server port'
+  })
+  .help()
+  .argv
+
+if (argv.port) {
+  port = argv.port
+}
+if (argv.upload_location) {
+  uploadsDir = argv.upload_location.endsWith('/') ? argv.upload_location: argv.upload_location + '/'
+}
+setUploadsDir(uploadsDir)
+
+console.log("Upload location: " + uploadsDir)
+console.log("Server port: " + port)
+
+const app: Express = express();
+const httpServer: Server = createServer(app)
+const io: socketio.Server = new socketio.Server(httpServer);
+
+setProgressWriter(new ProgressWriter(io));
+
+app.use(express.json());
+app.use(express.urlencoded({ extended: true }));
+app.use('/upload/init', uploadInitRouter);
+app.use('/upload/chunk', uploadChunkRouter);
+app.use('/upload/complete', uploadCompleteRouter);
+app.use('/upload/status', uploadStatusRouter);
+
+app.get('/', (_, res) => {
+  res.sendFile(__dirname + '/client/index.html');
+});
+
+app.get('/progresses', (_: Request, res: Response) => {
+  console.log("Progresses requested...");
+  res.writeHead(200, { 'content-type': 'application/json' });
+  res.write(JSON.stringify(progresses));
+  res.end();
+});
+
+io.on('connection', (socket: socketio.Socket) => {
+  console.log('a user connected');
+  // socket.emit('progresses', progresses);
+  getProgressWriter().writeProgress(progresses);
+  socket.on('disconnect', () => {
+    console.log('user disconnected');
+  });
+  socket.on('message', (msg) => {
+    console.log('message: ' + msg);
+  });
+});
+
+app.use('/', [
+  express.static(__dirname + '/client/')
+]);
+
+httpServer.listen(port, () => {
+  console.log('Server listening on ' + port + ' ...');
+});

package/src/model/progress.ts

@@ -0,0 +1,175 @@
+import prettyBytes from 'pretty-bytes'
+
+const TRANSFER_SAMPLE_FREQ = 1000; // 1second
+const MIN_SAMPLE_FREQ = TRANSFER_SAMPLE_FREQ / 4;
+
+export enum UploadStatus {
+    INITIATED = "INITIATED",
+    UPLOADING = "UPLOADING",
+    FINISHING = "FINISHING",
+    COMPLETE = "COMPLETE",
+    FAILED = "FAILED"
+}
+
+export interface Progress {
+  uuid?: any,
+  type?: string,
+  timestamp?: number,
+  bytesReceived?: number,
+  bytesExpected?: number,
+  bytesReceivedPretty?: string,
+  bytesExpectedPretty?: string,
+  fileName?: string,
+  savedLocation?: string,
+  completed?: number,
+  lastState: UploadStatus,
+  transferSamples?: TransferSample[]
+
+  markSample(): void
+}
+
+export interface TransferSample {
+    bytesReceived: number;      // total bytes received at this sample
+    timestamp: number;          // timestamp in ms
+    chunkIndex?: number;        // optional: index of chunk this sample refers to
+    chunkBytes?: number;        // optional: bytes in this chunk
+}
+
+export class FileTransferProgress implements Progress {
+  uuid?: any
+  type?: string | undefined
+  timestamp?: number | undefined
+  bytesReceived?: number | undefined
+  bytesExpected?: number | undefined
+  bytesReceivedPretty?: string | undefined
+  bytesExpectedPretty?: string | undefined
+  fileName?: string | undefined
+  savedLocation?: string | undefined
+  completed?: number | undefined
+  lastState: UploadStatus = UploadStatus.INITIATED;
+  transferSamples: TransferSample[] = []
+
+  chunkSize?: number;            // size of each chunk
+  totalChunks?: number;          // total number of chunks
+  chunkVerificationCount = 0;
+  uploadedChunks: Set<number> = new Set(); // track uploaded chunk indices
+  uploadingChunks: Set<number> = new Set();
+
+  constructor(uuid: string, timestamp: number) {
+    this.uuid = uuid;
+    this.type = 'progress';
+    this.timestamp = timestamp;
+    this.bytesReceived = 0;
+    this.bytesExpected = 0;
+    this.bytesReceivedPretty = prettyBytes(0);
+    this.bytesExpectedPretty = prettyBytes(0);
+  }
+
+  markSample(): void {
+    const timestampInMillis = new Date().getTime();
+    if (!this.shouldSample(timestampInMillis)) {
+      return;
+    }
+
+    const bytesReceived = this.bytesReceived as number;
+    this.transferSamples.push(
+      {
+        bytesReceived,
+        timestamp: timestampInMillis
+      }
+    );
+
+    this.cleanupSamples();
+  }
+
+  private shouldSample(millis: number): boolean {
+    if (this.transferSamples.length < 1) {
+      return true;
+    }
+    const lastSample = this.transferSamples[this.transferSamples.length - 1];
+    return (millis - lastSample.timestamp > MIN_SAMPLE_FREQ);
+  }
+
+  private cleanupSamples(): void {
+    if (this.transferSamples.length < 1) {
+      return;
+    }
+    const firstSample = this.transferSamples[0];
+    if (!firstSample) {
+      return;
+    }
+    const firstSampleTimestamp: number = firstSample.timestamp;
+    const nowInMillis = new Date().getTime();
+    if (firstSampleTimestamp - nowInMillis > TRANSFER_SAMPLE_FREQ) {
+      this.transferSamples.shift();
+    }
+  }
+
+    /** Calculate size of last chunk */
+    public lastChunkSize(): number {
+        if (!this.bytesExpected || !this.chunkSize || !this.totalChunks) {
+            return 0;
+        }
+        const remainder = this.bytesExpected % this.chunkSize;
+        return remainder > 0 ? remainder : this.chunkSize;
+    }
+
+    /** Calculate bytesReceived based on uploadedChunks */
+    public calculateBytesReceived(): number {
+        if (!this.chunkSize || !this.totalChunks) {
+            return this.bytesReceived || 0;
+        }
+
+        const lastChunkIndex = (this.totalChunks - 1);
+        const hasLastChunk = this.uploadedChunks.has(lastChunkIndex);
+        const chunksCount = this.uploadedChunks.size;
+
+        if (hasLastChunk) {
+            return ((chunksCount - 1) * this.chunkSize) + this.lastChunkSize();
+        }
+        return chunksCount * this.chunkSize;
+    }
+
+    /** Update bytesReceived and its pretty representation */
+    private updateBytesReceived(): void {
+        this.bytesReceived = this.calculateBytesReceived();
+        this.bytesReceivedPretty = prettyBytes(this.bytesReceived || 0);
+        this.bytesExpectedPretty = prettyBytes(this.bytesExpected || 0);
+    }
+
+    public addUploadedChunk(chunkIndex: number): void {
+        if (chunkIndex < 0) {
+            console.warn(`Invalid chunkIndex ${chunkIndex}. Must be >= 0.`);
+            return;
+        }
+        if (!this.totalChunks) {
+            console.warn(`Total chunks not set yet. Cannot add chunk ${chunkIndex}.`);
+            return;
+        }
+        if (chunkIndex >= this.totalChunks) {
+            console.warn(`Chunk index ${chunkIndex} exceeds totalChunks (${this.totalChunks - 1}).`);
+            return;
+        }
+        if (this.uploadedChunks.has(chunkIndex)) {
+            console.warn(`Chunk index ${chunkIndex} already marked as uploaded for ${this.fileName}.`);
+            return;
+        }
+
+        this.uploadedChunks.add(chunkIndex);
+        this.updateBytesReceived();
+    }
+
+    public resetUploadedChunks(): void {
+        this.uploadedChunks = new Set();
+    }
+
+    public chunkVerified(chunkIndex: number): void {
+        this.chunkVerificationCount++;
+        console.debug(`chunk ${chunkIndex} verified. Total verified: ${this.chunkVerificationCount} for ${this.fileName}.`)
+    }
+
+    /** Optional: reset */
+    public resetVerificationCount(): void {
+        this.chunkVerificationCount = 0;
+    }
+}