@es-labs/jslib 0.0.1

package/node/oss-files/s3-uploader-client.js

@@ -0,0 +1,217 @@
+/**
+ * S3 Large File Uploader (Client-Side)
+ * Supports multipart upload for files > 5MB using pre-signed URLs
+ * 
+ * Usage:
+ *   const uploader = new S3Uploader({ getSignedUrlEndpoint: '/api/s3/sign' });
+ *   await uploader.upload(file, { onProgress: (pct) => console.log(pct) });
+ */
+
+const CHUNK_SIZE = 10 * 1024 * 1024; // 10MB per part (min 5MB for S3 multipart)
+const SINGLE_UPLOAD_LIMIT = 5 * 1024 * 1024; // Use multipart above 5MB
+
+class S3Uploader {
+  /**
+   * @param {object} options
+   * @param {string} options.getSignedUrlEndpoint - Your backend endpoint that returns signed URLs
+   * @param {number} [options.chunkSize]          - Bytes per part (default: 10MB)
+   * @param {number} [options.maxConcurrent]      - Parallel part uploads (default: 3)
+   */
+  constructor(options = {}) {
+    this.endpoint = options.getSignedUrlEndpoint;
+    this.chunkSize = options.chunkSize || CHUNK_SIZE;
+    this.maxConcurrent = options.maxConcurrent || 3;
+
+    if (!this.endpoint) {
+      throw new Error('getSignedUrlEndpoint is required');
+    }
+  }
+
+  // ─── Public API ─────────────────────────────────────────────────────────────
+
+  /**
+   * Upload a File or Blob to S3.
+   * @param {File|Blob} file
+   * @param {object}   [opts]
+   * @param {string}   [opts.key]        - S3 object key (defaults to file.name)
+   * @param {Function} [opts.onProgress] - Called with 0–100 progress percentage
+   * @param {AbortSignal} [opts.signal]  - AbortController signal to cancel upload
+   * @returns {Promise<{ key: string, location: string }>}
+   */
+  async upload(file, opts = {}) {
+    const key = opts.key || file.name;
+    const onProgress = opts.onProgress || (() => {});
+    const signal = opts.signal || null;
+
+    if (file.size <= SINGLE_UPLOAD_LIMIT) {
+      return this._singleUpload(file, key, onProgress, signal);
+    }
+    return this._multipartUpload(file, key, onProgress, signal);
+  }
+
+  // ─── Single-Part Upload (≤5MB) ───────────────────────────────────────────────
+
+  async _singleUpload(file, key, onProgress, signal) {
+    onProgress(0);
+
+    // 1. Get signed PUT URL from your backend
+    const { signedUrl, location } = await this._requestSignedUrl({
+      type: 'single',
+      key,
+      contentType: file.type,
+      size: file.size,
+    });
+
+    // 2. PUT the file directly to S3
+    await this._putToS3(signedUrl, file, file.type, (pct) => onProgress(pct), signal);
+
+    onProgress(100);
+    return { key, location };
+  }
+
+  // ─── Multipart Upload (>5MB) ─────────────────────────────────────────────────
+
+  async _multipartUpload(file, key, onProgress, signal) {
+    // 1. Initiate multipart upload — get uploadId from your backend
+    const { uploadId } = await this._requestSignedUrl({
+      type: 'initiate',
+      key,
+      contentType: file.type,
+    });
+
+    const chunks = this._splitFile(file);
+    const totalParts = chunks.length;
+    const partProgress = new Array(totalParts).fill(0);
+
+    const updateProgress = () => {
+      const totalUploaded = partProgress.reduce((a, b) => a + b, 0);
+      onProgress(Math.round((totalUploaded / file.size) * 100));
+    };
+
+    // 2. Upload parts with concurrency control
+    let partNumber = 1;
+    const completedParts = [];
+    const queue = [...chunks.entries()]; // [[index, blob], ...]
+
+    const uploadWorker = async () => {
+      while (queue.length > 0) {
+        const [index, chunk] = queue.shift();
+        const currentPart = index + 1;
+
+        if (signal?.aborted) throw new DOMException('Upload aborted', 'AbortError');
+
+        // Get a signed URL for this part
+        const { signedUrl } = await this._requestSignedUrl({
+          type: 'part',
+          key,
+          uploadId,
+          partNumber: currentPart,
+        });
+
+        // Upload the part and capture the ETag
+        const etag = await this._putToS3(
+          signedUrl,
+          chunk,
+          file.type,
+          (bytesDone) => {
+            partProgress[index] = bytesDone;
+            updateProgress();
+          },
+          signal,
+          true // returnEtag
+        );
+
+        completedParts.push({ PartNumber: currentPart, ETag: etag });
+      }
+    };
+
+    // Run N workers in parallel
+    const workers = Array.from({ length: this.maxConcurrent }, () => uploadWorker());
+    await Promise.all(workers);
+
+    // 3. Complete the multipart upload
+    completedParts.sort((a, b) => a.PartNumber - b.PartNumber);
+
+    const { location } = await this._requestSignedUrl({
+      type: 'complete',
+      key,
+      uploadId,
+      parts: completedParts,
+    });
+
+    onProgress(100);
+    return { key, location };
+  }
+
+  // ─── Helpers ─────────────────────────────────────────────────────────────────
+
+  /** Split a File/Blob into chunks */
+  _splitFile(file) {
+    const chunks = [];
+    let offset = 0;
+    while (offset < file.size) {
+      chunks.push(file.slice(offset, offset + this.chunkSize));
+      offset += this.chunkSize;
+    }
+    return chunks;
+  }
+
+  /**
+   * Call your backend to get signed URLs / manage multipart lifecycle.
+   * Adapt the request/response shape to match your backend API.
+   */
+  async _requestSignedUrl(payload) {
+    const res = await fetch(this.endpoint, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(payload),
+    });
+
+    if (!res.ok) {
+      const text = await res.text();
+      throw new Error(`Signed URL request failed (${res.status}): ${text}`);
+    }
+    return res.json();
+  }
+
+  /**
+   * PUT a blob to a pre-signed S3 URL, reporting progress.
+   * Returns ETag if returnEtag=true (needed for multipart complete).
+   */
+  _putToS3(signedUrl, blob, contentType, onChunkProgress, signal, returnEtag = false) {
+    return new Promise((resolve, reject) => {
+      const xhr = new XMLHttpRequest();
+
+      xhr.open('PUT', signedUrl);
+      xhr.setRequestHeader('Content-Type', contentType);
+
+      xhr.upload.addEventListener('progress', (e) => {
+        if (e.lengthComputable) onChunkProgress(e.loaded);
+      });
+
+      xhr.addEventListener('load', () => {
+        if (xhr.status >= 200 && xhr.status < 300) {
+          if (returnEtag) {
+            const etag = xhr.getResponseHeader('ETag');
+            resolve(etag);
+          } else {
+            resolve(xhr.responseURL || signedUrl); // rough "location"
+          }
+        } else {
+          reject(new Error(`S3 upload failed: HTTP ${xhr.status}`));
+        }
+      });
+
+      xhr.addEventListener('error', () => reject(new Error('Network error during upload')));
+      xhr.addEventListener('abort', () => reject(new DOMException('Upload aborted', 'AbortError')));
+
+      if (signal) {
+        signal.addEventListener('abort', () => xhr.abort());
+      }
+
+      xhr.send(blob);
+    });
+  }
+}
+
+export default S3Uploader;

package/node/oss-files/s3-uploader-server.js

@@ -0,0 +1,123 @@
+/**
+ * S3 Signed URL Backend (Node.js / Express)
+ *
+ * Install deps:
+ *   npm install @aws-sdk/client-s3 @aws-sdk/s3-request-presigner express
+ *
+ * Env vars required:
+ *   AWS_REGION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_BUCKET_NAME
+ */
+
+import express from 'express';
+import {
+  S3Client,
+  CreateMultipartUploadCommand,
+  UploadPartCommand,
+  CompleteMultipartUploadCommand,
+  AbortMultipartUploadCommand,
+  PutObjectCommand,
+} from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+
+const app = express();
+app.use(express.json());
+
+const s3 = new S3Client({
+  region: process.env.AWS_REGION || 'us-east-1',
+  credentials: {
+    accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+  },
+});
+
+const BUCKET = process.env.S3_BUCKET_NAME;
+const URL_EXPIRY = 3600; // signed URL valid for 1 hour
+
+// ─── Single endpoint that handles all upload phases ──────────────────────────
+// POST /api/s3/sign
+// Body: { type, key, contentType?, size?, uploadId?, partNumber?, parts? }
+
+app.post('/api/s3/sign', async (req, res) => {
+  const { type, key, contentType, size, uploadId, partNumber, parts } = req.body;
+
+  try {
+    switch (type) {
+      // ── 1. Single-file upload (≤5MB) ─────────────────────────────────────
+      case 'single': {
+        const command = new PutObjectCommand({
+          Bucket: BUCKET,
+          Key: key,
+          ContentType: contentType,
+          ContentLength: size,
+          // Optional: enforce public/private ACL, SSE, etc.
+          // ServerSideEncryption: 'AES256',
+        });
+        const signedUrl = await getSignedUrl(s3, command, { expiresIn: URL_EXPIRY });
+        const location = `https://${BUCKET}.s3.amazonaws.com/${key}`;
+        return res.json({ signedUrl, location });
+      }
+
+      // ── 2. Initiate multipart upload ──────────────────────────────────────
+      case 'initiate': {
+        const command = new CreateMultipartUploadCommand({
+          Bucket: BUCKET,
+          Key: key,
+          ContentType: contentType,
+        });
+        const response = await s3.send(command);
+        return res.json({ uploadId: response.UploadId });
+      }
+
+      // ── 3. Sign an individual part ────────────────────────────────────────
+      case 'part': {
+        if (!uploadId || !partNumber) {
+          return res.status(400).json({ error: 'uploadId and partNumber are required' });
+        }
+        const command = new UploadPartCommand({
+          Bucket: BUCKET,
+          Key: key,
+          UploadId: uploadId,
+          PartNumber: partNumber,
+        });
+        const signedUrl = await getSignedUrl(s3, command, { expiresIn: URL_EXPIRY });
+        return res.json({ signedUrl });
+      }
+
+      // ── 4. Complete multipart upload ──────────────────────────────────────
+      case 'complete': {
+        if (!uploadId || !parts?.length) {
+          return res.status(400).json({ error: 'uploadId and parts[] are required' });
+        }
+        const command = new CompleteMultipartUploadCommand({
+          Bucket: BUCKET,
+          Key: key,
+          UploadId: uploadId,
+          MultipartUpload: { Parts: parts }, // [{ PartNumber, ETag }, ...]
+        });
+        const response = await s3.send(command);
+        return res.json({ location: response.Location });
+      }
+
+      // ── 5. Abort multipart upload (cleanup on error/cancel) ───────────────
+      case 'abort': {
+        if (!uploadId) return res.status(400).json({ error: 'uploadId is required' });
+        const command = new AbortMultipartUploadCommand({
+          Bucket: BUCKET,
+          Key: key,
+          UploadId: uploadId,
+        });
+        await s3.send(command);
+        return res.json({ success: true });
+      }
+
+      default:
+        return res.status(400).json({ error: `Unknown type: ${type}` });
+    }
+  } catch (err) {
+    console.error('[S3 sign error]', err);
+    res.status(500).json({ error: err.message });
+  }
+});
+
+app.listen(3000, () => console.log('Server running on http://localhost:3000'));
+export default app;

package/node/oss-files/s3-uploader-usage.js

@@ -0,0 +1,77 @@
+/**
+ * Usage example — drop this in your frontend
+ * Works with vanilla JS, React, Vue, etc.
+ */
+import S3Uploader from './s3-uploader-client.js';
+
+// ─── Setup ────────────────────────────────────────────────────────────────────
+
+const uploader = new S3Uploader({
+  getSignedUrlEndpoint: '/api/s3/sign', // your backend
+  chunkSize: 10 * 1024 * 1024,         // 10MB parts
+  maxConcurrent: 3,                    // 3 parallel part uploads
+});
+
+// ─── Basic upload ─────────────────────────────────────────────────────────────
+
+async function uploadFile(file) {
+  try {
+    const result = await uploader.upload(file, {
+      key: `uploads/${Date.now()}-${file.name}`, // custom S3 key (optional)
+      onProgress: (pct) => {
+        console.log(`Upload progress: ${pct}%`);
+        document.getElementById('progress').value = pct;
+      },
+    });
+
+    console.log('Upload complete!', result);
+    // result = { key: 'uploads/...', location: 'https://bucket.s3.amazonaws.com/...' }
+  } catch (err) {
+    console.error('Upload failed:', err.message);
+  }
+}
+
+// ─── With cancellation support ────────────────────────────────────────────────
+
+let controller;
+
+async function uploadWithCancel(file) {
+  controller = new AbortController();
+
+  try {
+    const result = await uploader.upload(file, {
+      onProgress: (pct) => console.log(`${pct}%`),
+      signal: controller.signal,
+    });
+    console.log('Done:', result);
+  } catch (err) {
+    if (err.name === 'AbortError') {
+      console.log('Upload cancelled by user');
+      // Optionally call your backend to abort the multipart upload in S3
+      // to avoid storage costs for incomplete uploads
+    } else {
+      console.error('Error:', err);
+    }
+  }
+}
+
+function cancelUpload() {
+  controller?.abort();
+}
+
+// ─── Wire up to a file input ──────────────────────────────────────────────────
+
+document.getElementById('fileInput').addEventListener('change', (e) => {
+  const file = e.target.files[0];
+  if (file) uploadWithCancel(file);
+});
+
+document.getElementById('cancelBtn').addEventListener('click', cancelUpload);
+
+/*
+  Minimal HTML to go with this:
+
+  <input type="file" id="fileInput" />
+  <progress id="progress" max="100" value="0"></progress>
+  <button id="cancelBtn">Cancel</button>
+*/

package/node/oss-files/s3-uploader-usage.md

@@ -0,0 +1,34 @@
+Here's a comprehensive JavaScript solution for reading and uploading large files to S3 using signed URLs, with multipart upload support for reliability:Here's a breakdown of what's in each file:
+
+---
+
+**`s3-uploader-client.js`** — Browser-side `S3Uploader` class
+- Files **≤ 5MB** → single `PUT` with a signed URL
+- Files **> 5MB** → S3 multipart upload (10MB chunks by default)
+- Concurrent part uploads (default: 3 in parallel) for speed
+- `XMLHttpRequest`-based for accurate `upload.progress` events
+- `AbortController` support to cancel mid-upload
+
+**`s3-uploader-server.js`** — Node.js/Express backend
+- Single `/api/s3/sign` endpoint handles all 5 phases: `single`, `initiate`, `part`, `complete`, `abort`
+- Uses the AWS SDK v3 (`@aws-sdk/client-s3` + `@aws-sdk/s3-request-presigner`)
+- Your AWS credentials never touch the browser
+
+**`s3-uploader-usage.js`** — Integration examples with cancel support
+
+---
+
+**Quick setup:**
+
+```bash
+# Backend
+npm install @aws-sdk/client-s3 @aws-sdk/s3-request-presigner express
+```
+
+Set env vars: `AWS_REGION`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `S3_BUCKET_NAME`
+
+**S3 CORS config** you'll also need on your bucket:
+```json
+[{ "AllowedOrigins": ["*"], "AllowedMethods": ["PUT"], "AllowedHeaders": ["*"], "ExposeHeaders": ["ETag"] }]
+```
+The `ETag` exposure is critical — S3 multipart complete requires the ETags from each part.

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@es-labs/jslib",
+  "version": "0.0.1",
+  "author": "Aaron Gong",
+  "license": "MIT",
+  "description": "Shareable JS library using ES modules",
+  "keywords": [
+    "javscript",
+    "module"
+  ],
+  "homepage": "https://github.com/es-labs/jscommon#readme",
+  "bugs": {
+    "url": "https://github.com/es-labs/jscommon/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/es-labs/jscommon.git"
+  },
+  "type": "module",
+  "exports": {
+    "./auth": "./auth/index.js",
+    "./auth/*": "./auth/*.js",
+    "./comms/*": "./comms/*.js",
+    "./express/controller/auth/*": "./express/controller/auth/*.js",
+    "./express/*": "./express/*.js",
+    "./iso/*": "./iso/*.js",
+    "./services": "./services/index.js",
+    "./services/*": "./services/*.js",
+    "./utils/*": "./utils/*.js",
+    "./web/*": "./web/*.js",
+    "./*": "./*.js"
+  },
+  "main": "index.js",
+  "scripts": {
+    "test": "node --test-reporter=spec --experimental-test-coverage --test **/__tests__/*.spec.js",
+    "cover": "node --experimental-test-coverage --test-reporter=lcov --test-reporter-destination=lcov.info --test __tests__/**/*.spec.js"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.1019.0",
+    "@node-saml/node-saml": "^5.1.0",
+    "ali-oss": "^6.23.0",
+    "bcryptjs": "^3.0.3",
+    "dotenv": "^17.3.1",
+    "ioredis": "^5.10.1",
+    "jsonwebtoken": "^9.0.3",
+    "keyv": "^5.6.0",
+    "knex": "^3.2.7",
+    "multer": "^2.1.1",
+    "otplib": "^13.4.0",
+    "web-push": "^3.6.7",
+    "ws": "^8.20.0"
+  }
+}

package/packageInfo.js

@@ -0,0 +1,9 @@
+import packageJson from './package.json' with { type: 'json' };
+
+const { name, version, dependencies } = packageJson;
+
+export {
+  name,
+  version,
+  dependencies
+}