@es-labs/jslib 0.0.1

package/node/oss-files/oss-uploader-client.js

@@ -0,0 +1,219 @@
+/**
+ * Alibaba Cloud OSS Large File Uploader (Client-Side)
+ * Uses the S3-compatible API via pre-signed URLs generated by your backend.
+ *
+ * OSS S3-compatibility docs:
+ *   https://www.alibabacloud.com/help/en/oss/developer-reference/compatibility-with-amazon-s3
+ *
+ * Key OSS-specific differences from AWS S3:
+ *  - Endpoint format: https://<bucket>.<region>.aliyuncs.com  (path-style not needed)
+ *  - Minimum multipart part size: 100KB (S3 is 5MB) — we still use 10MB for reliability
+ *  - Max parts: 10,000 (same as S3)
+ *  - ETag is returned without quotes in some OSS responses — we normalize this below
+ *  - CORS must expose the ETag header explicitly on the OSS bucket
+ *
+ * Usage:
+ *   const uploader = new OSSUploader({ signEndpoint: '/api/oss/sign' });
+ *   const result = await uploader.upload(file, { onProgress: pct => console.log(pct + '%') });
+ */
+
+const CHUNK_SIZE = 10 * 1024 * 1024;       // 10MB per part
+const MULTIPART_THRESHOLD = 5 * 1024 * 1024; // Use multipart above 5MB
+
+class OSSUploader {
+  /**
+   * @param {object} options
+   * @param {string} options.signEndpoint   - Your backend endpoint for signed URL generation
+   * @param {number} [options.chunkSize]    - Bytes per part (default 10MB, min 100KB for OSS)
+   * @param {number} [options.maxConcurrent]- Parallel part uploads (default 3)
+   */
+  constructor(options = {}) {
+    if (!options.signEndpoint) throw new Error('signEndpoint is required');
+    this.signEndpoint = options.signEndpoint;
+    this.chunkSize = options.chunkSize || CHUNK_SIZE;
+    this.maxConcurrent = options.maxConcurrent || 3;
+  }
+
+  // ─── Public API ──────────────────────────────────────────────────────────────
+
+  /**
+   * Upload a File or Blob to Alibaba OSS.
+   *
+   * @param {File|Blob} file
+   * @param {object}    [opts]
+   * @param {string}    [opts.key]         - OSS object key (defaults to file.name)
+   * @param {Function}  [opts.onProgress]  - Callback with integer 0–100
+   * @param {AbortSignal} [opts.signal]    - AbortController signal to cancel
+   * @returns {Promise<{ key: string, location: string }>}
+   */
+  async upload(file, opts = {}) {
+    const key = opts.key || file.name;
+    const onProgress = opts.onProgress || (() => {});
+    const signal = opts.signal || null;
+
+    if (file.size <= MULTIPART_THRESHOLD) {
+      return this._singleUpload(file, key, onProgress, signal);
+    }
+    return this._multipartUpload(file, key, onProgress, signal);
+  }
+
+  // ─── Single-Part Upload (≤5MB) ───────────────────────────────────────────────
+
+  async _singleUpload(file, key, onProgress, signal) {
+    onProgress(0);
+
+    const { signedUrl, location } = await this._callBackend({
+      type: 'single',
+      key,
+      contentType: file.type || 'application/octet-stream',
+      size: file.size,
+    });
+
+    await this._putBlob(signedUrl, file, file.type, (loaded) => {
+      onProgress(Math.round((loaded / file.size) * 100));
+    }, signal);
+
+    onProgress(100);
+    return { key, location };
+  }
+
+  // ─── Multipart Upload (>5MB) ─────────────────────────────────────────────────
+
+  async _multipartUpload(file, key, onProgress, signal) {
+    // Step 1 — Initiate: backend calls CreateMultipartUpload, returns uploadId
+    const { uploadId } = await this._callBackend({
+      type: 'initiate',
+      key,
+      contentType: file.type || 'application/octet-stream',
+    });
+
+    const chunks = this._splitFile(file);
+    const partProgress = new Array(chunks.length).fill(0);
+
+    const reportProgress = () => {
+      const uploaded = partProgress.reduce((s, v) => s + v, 0);
+      onProgress(Math.round((uploaded / file.size) * 100));
+    };
+
+    // Step 2 — Upload parts concurrently
+    const completedParts = [];
+    const queue = chunks.map((chunk, i) => ({ chunk, partNumber: i + 1, index: i }));
+
+    const worker = async () => {
+      while (queue.length > 0) {
+        const { chunk, partNumber, index } = queue.shift();
+
+        if (signal?.aborted) throw new DOMException('Upload aborted', 'AbortError');
+
+        // Get a signed URL for this specific part
+        const { signedUrl } = await this._callBackend({
+          type: 'part',
+          key,
+          uploadId,
+          partNumber,
+        });
+
+        // PUT the chunk; OSS returns ETag in response header
+        const rawETag = await this._putBlob(
+          signedUrl,
+          chunk,
+          file.type || 'application/octet-stream',
+          (loaded) => { partProgress[index] = loaded; reportProgress(); },
+          signal,
+          /* returnETag= */ true,
+        );
+
+        // OSS sometimes returns ETag without surrounding quotes — normalise
+        const etag = rawETag?.replace(/"/g, '') ? `"${rawETag.replace(/"/g, '')}"` : rawETag;
+
+        completedParts.push({ PartNumber: partNumber, ETag: etag });
+      }
+    };
+
+    const workers = Array.from({ length: this.maxConcurrent }, worker);
+    try {
+      await Promise.all(workers);
+    } catch (err) {
+      // Best-effort abort to avoid orphaned multipart uploads costing storage
+      await this._callBackend({ type: 'abort', key, uploadId }).catch(() => {});
+      throw err;
+    }
+
+    // Step 3 — Complete: parts must be sorted by PartNumber
+    completedParts.sort((a, b) => a.PartNumber - b.PartNumber);
+
+    const { location } = await this._callBackend({
+      type: 'complete',
+      key,
+      uploadId,
+      parts: completedParts,
+    });
+
+    onProgress(100);
+    return { key, location };
+  }
+
+  // ─── Helpers ─────────────────────────────────────────────────────────────────
+
+  _splitFile(file) {
+    const chunks = [];
+    for (let offset = 0; offset < file.size; offset += this.chunkSize) {
+      chunks.push(file.slice(offset, offset + this.chunkSize));
+    }
+    return chunks;
+  }
+
+  async _callBackend(payload) {
+    const res = await fetch(this.signEndpoint, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(payload),
+    });
+    if (!res.ok) {
+      const text = await res.text();
+      throw new Error(`Backend error (${res.status}): ${text}`);
+    }
+    return res.json();
+  }
+
+  /**
+   * PUT a Blob to a pre-signed OSS URL via XHR (for upload progress events).
+   *
+   * OSS S3-compatible PUT requires:
+   *   - Content-Type header must match the value used when signing
+   *   - Do NOT send Content-MD5 unless you included it in the signed headers
+   *
+   * @returns {Promise<string|undefined>} ETag value when returnETag=true
+   */
+  _putBlob(signedUrl, blob, contentType, onProgress, signal, returnETag = false) {
+    return new Promise((resolve, reject) => {
+      const xhr = new XMLHttpRequest();
+      xhr.open('PUT', signedUrl);
+
+      // Content-Type must match what was signed on the backend
+      if (contentType) xhr.setRequestHeader('Content-Type', contentType);
+
+      xhr.upload.addEventListener('progress', (e) => {
+        if (e.lengthComputable) onProgress(e.loaded);
+      });
+
+      xhr.addEventListener('load', () => {
+        if (xhr.status >= 200 && xhr.status < 300) {
+          resolve(returnETag ? xhr.getResponseHeader('ETag') : undefined);
+        } else {
+          // OSS returns XML error bodies — surface them for easier debugging
+          reject(new Error(`OSS PUT failed (HTTP ${xhr.status}): ${xhr.responseText}`));
+        }
+      });
+
+      xhr.addEventListener('error', () => reject(new Error('Network error during OSS upload')));
+      xhr.addEventListener('abort', () => reject(new DOMException('Upload aborted', 'AbortError')));
+
+      if (signal) signal.addEventListener('abort', () => xhr.abort(), { once: true });
+
+      xhr.send(blob);
+    });
+  }
+}
+
+export default OSSUploader;

package/node/oss-files/oss-uploader-server.js

@@ -0,0 +1,199 @@
+/**
+ * Alibaba Cloud OSS — Signed URL Backend (Node.js / Express)
+ * Uses the AWS SDK v3 pointed at OSS's S3-compatible endpoint.
+ *
+ * Install:
+ *   npm install @aws-sdk/client-s3 @aws-sdk/s3-request-presigner express
+ *
+ * Required env vars:
+ *   OSS_REGION          e.g. oss-ap-southeast-1
+ *   OSS_BUCKET          your bucket name
+ *   OSS_ACCESS_KEY_ID   Alibaba RAM user AccessKey ID
+ *   OSS_ACCESS_KEY_SECRET
+ *
+ * OSS S3-compatible endpoint format:
+ *   https://<region>.aliyuncs.com          (global/internal)
+ *   https://<region>.aliyuncs.com          (use forcePathStyle: false — virtual-hosted default)
+ *
+ * Compatibility notes:
+ *   - OSS supports Signature V4 (same as S3) via the S3-compatible API
+ *   - Use forcePathStyle: false (virtual-hosted style) — OSS prefers bucket in hostname
+ *   - OSS presigned URLs use x-oss-* headers, but the S3 SDK generates x-amz-* — this is
+ *     fine because OSS accepts both via the S3-compat layer
+ *   - ETag exposure in CORS must be configured on the OSS bucket (see notes at bottom)
+ */
+
+import express from 'express';
+import {
+  S3Client,
+  PutObjectCommand,
+  CreateMultipartUploadCommand,
+  UploadPartCommand,
+  CompleteMultipartUploadCommand,
+  AbortMultipartUploadCommand,
+} from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+
+const app = express();
+app.use(express.json());
+
+// ─── OSS Client via S3-compatible endpoint ────────────────────────────────────
+
+const REGION = process.env.OSS_REGION;           // e.g. 'oss-ap-southeast-1'
+const BUCKET = process.env.OSS_BUCKET;
+const URL_TTL = 3600;                            // signed URL valid for 1 hour
+
+if (!REGION || !BUCKET) {
+  throw new Error('OSS_REGION and OSS_BUCKET env vars are required');
+}
+
+const ossClient = new S3Client({
+  // OSS S3-compatible endpoint — region subdomain, no bucket prefix here
+  endpoint: `https://${REGION}.aliyuncs.com`,
+
+  region: REGION,
+
+  // Virtual-hosted style: bucket is placed in the hostname (recommended for OSS)
+  // e.g. https://my-bucket.oss-ap-southeast-1.aliyuncs.com/object-key
+  forcePathStyle: false,
+
+  credentials: {
+    accessKeyId: process.env.OSS_ACCESS_KEY_ID,
+    secretAccessKey: process.env.OSS_ACCESS_KEY_SECRET,
+  },
+});
+
+// ─── Single endpoint handling all upload lifecycle phases ─────────────────────
+// POST /api/oss/sign
+// Body: { type, key, contentType?, size?, uploadId?, partNumber?, parts? }
+
+app.post('/api/oss/sign', async (req, res) => {
+  const { type, key, contentType, size, uploadId, partNumber, parts } = req.body;
+
+  if (!key) return res.status(400).json({ error: 'key is required' });
+
+  try {
+    switch (type) {
+
+      // ── 1. Single PUT (≤5MB) ───────────────────────────────────────────────
+      case 'single': {
+        const cmd = new PutObjectCommand({
+          Bucket: BUCKET,
+          Key: key,
+          ContentType: contentType || 'application/octet-stream',
+          ContentLength: size,
+        });
+        const signedUrl = await getSignedUrl(ossClient, cmd, { expiresIn: URL_TTL });
+
+        // OSS virtual-hosted URL for the completed object
+        const location = `https://${BUCKET}.${REGION}.aliyuncs.com/${key}`;
+        return res.json({ signedUrl, location });
+      }
+
+      // ── 2. Initiate multipart ──────────────────────────────────────────────
+      case 'initiate': {
+        const cmd = new CreateMultipartUploadCommand({
+          Bucket: BUCKET,
+          Key: key,
+          ContentType: contentType || 'application/octet-stream',
+        });
+        const response = await ossClient.send(cmd);
+        return res.json({ uploadId: response.UploadId });
+      }
+
+      // ── 3. Sign a part URL ─────────────────────────────────────────────────
+      case 'part': {
+        if (!uploadId || !partNumber) {
+          return res.status(400).json({ error: 'uploadId and partNumber are required' });
+        }
+        const cmd = new UploadPartCommand({
+          Bucket: BUCKET,
+          Key: key,
+          UploadId: uploadId,
+          PartNumber: Number(partNumber),
+        });
+        const signedUrl = await getSignedUrl(ossClient, cmd, { expiresIn: URL_TTL });
+        return res.json({ signedUrl });
+      }
+
+      // ── 4. Complete multipart ──────────────────────────────────────────────
+      case 'complete': {
+        if (!uploadId || !Array.isArray(parts) || parts.length === 0) {
+          return res.status(400).json({ error: 'uploadId and parts[] are required' });
+        }
+        const cmd = new CompleteMultipartUploadCommand({
+          Bucket: BUCKET,
+          Key: key,
+          UploadId: uploadId,
+          MultipartUpload: {
+            // OSS is strict: parts must be sorted by PartNumber
+            Parts: parts
+              .slice()
+              .sort((a, b) => a.PartNumber - b.PartNumber)
+              .map(({ PartNumber, ETag }) => ({ PartNumber, ETag })),
+          },
+        });
+        const response = await ossClient.send(cmd);
+        const location = response.Location ||
+          `https://${BUCKET}.${REGION}.aliyuncs.com/${key}`;
+        return res.json({ location });
+      }
+
+      // ── 5. Abort multipart (clean up on cancel/error) ──────────────────────
+      case 'abort': {
+        if (!uploadId) return res.status(400).json({ error: 'uploadId is required' });
+        const cmd = new AbortMultipartUploadCommand({
+          Bucket: BUCKET,
+          Key: key,
+          UploadId: uploadId,
+        });
+        await ossClient.send(cmd);
+        return res.json({ success: true });
+      }
+
+      default:
+        return res.status(400).json({ error: `Unknown type: "${type}"` });
+    }
+  } catch (err) {
+    console.error('[OSS sign error]', err);
+    res.status(500).json({ error: err.message });
+  }
+});
+
+app.listen(3000, () => console.log('OSS sign server running on http://localhost:3000'));
+export default app;
+
+/*
+ * ─── OSS Bucket CORS Configuration (required) ────────────────────────────────
+ *
+ * In the Alibaba Cloud OSS console → Bucket → Data Security → CORS:
+ *
+ * Rule:
+ *   Allowed Origins:   https://your-frontend-domain.com  (or * for dev)
+ *   Allowed Methods:   PUT
+ *   Allowed Headers:   *
+ *   Exposed Headers:   ETag                  ← CRITICAL for multipart complete
+ *   Max Age (seconds): 3600
+ *
+ * Without exposing ETag, the browser cannot read it from the PUT response
+ * and multipart complete will fail.
+ *
+ * ─── OSS RAM Policy (minimum permissions for the signing user) ───────────────
+ *
+ * {
+ *   "Statement": [{
+ *     "Effect": "Allow",
+ *     "Action": [
+ *       "oss:PutObject",
+ *       "oss:InitiateMultipartUpload",
+ *       "oss:UploadPart",
+ *       "oss:CompleteMultipartUpload",
+ *       "oss:AbortMultipartUpload"
+ *     ],
+ *     "Resource": [
+ *       "acs:oss:*:*:your-bucket-name",
+ *       "acs:oss:*:*:your-bucket-name/*"
+ *     ]
+ *   }]
+ * }
+ */

package/node/oss-files/oss-uploader-usage.js

@@ -0,0 +1,121 @@
+/**
+ * Alibaba OSS Uploader — Usage Examples
+ */
+
+// ─── 1. Vanilla JS ────────────────────────────────────────────────────────────
+import OSSUploader from './oss-uploader-client.js';
+
+const uploader = new OSSUploader({
+  signEndpoint: '/api/oss/sign',
+  chunkSize: 10 * 1024 * 1024,  // 10MB
+  maxConcurrent: 3,
+});
+
+// Basic upload
+async function uploadFile(file) {
+  try {
+    const result = await uploader.upload(file, {
+      key: `uploads/${Date.now()}-${file.name}`,
+      onProgress: (pct) => {
+        console.log(`${pct}%`);
+        document.getElementById('progress').value = pct;
+      },
+    });
+    console.log('Done:', result.location);
+  } catch (err) {
+    console.error('Upload failed:', err.message);
+  }
+}
+
+// With cancel support
+let controller = null;
+
+async function uploadWithCancel(file) {
+  controller = new AbortController();
+  try {
+    const result = await uploader.upload(file, {
+      key: `uploads/${file.name}`,
+      onProgress: (pct) => console.log(`${pct}%`),
+      signal: controller.signal,
+    });
+    console.log('Uploaded to:', result.location);
+  } catch (err) {
+    if (err.name === 'AbortError') {
+      console.log('Upload cancelled');
+      // The client already called /api/oss/sign with type=abort to clean up OSS
+    } else {
+      console.error(err);
+    }
+  }
+}
+
+document.getElementById('fileInput').addEventListener('change', (e) => {
+  if (e.target.files[0]) uploadWithCancel(e.target.files[0]);
+});
+document.getElementById('cancelBtn').addEventListener('click', () => controller?.abort());
+
+
+// ─── 2. React component ───────────────────────────────────────────────────────
+/*
+import { useState, useRef } from 'react';
+import OSSUploader from './oss-uploader-client.js';
+
+const uploader = new OSSUploader({ signEndpoint: '/api/oss/sign' });
+
+export function OSSFileUpload() {
+  const [progress, setProgress] = useState(0);
+  const [status, setStatus]     = useState('idle'); // idle | uploading | done | error
+  const [location, setLocation] = useState('');
+  const controllerRef           = useRef(null);
+
+  const handleFileChange = async (e) => {
+    const file = e.target.files[0];
+    if (!file) return;
+
+    controllerRef.current = new AbortController();
+    setStatus('uploading');
+    setProgress(0);
+
+    try {
+      const result = await uploader.upload(file, {
+        key: `uploads/${Date.now()}-${file.name}`,
+        onProgress: setProgress,
+        signal: controllerRef.current.signal,
+      });
+      setStatus('done');
+      setLocation(result.location);
+    } catch (err) {
+      if (err.name === 'AbortError') {
+        setStatus('idle');
+      } else {
+        setStatus('error');
+        console.error(err);
+      }
+    }
+  };
+
+  return (
+    <div>
+      <input type="file" onChange={handleFileChange} disabled={status === 'uploading'} />
+      {status === 'uploading' && (
+        <>
+          <progress value={progress} max={100} />
+          <span>{progress}%</span>
+          <button onClick={() => controllerRef.current?.abort()}>Cancel</button>
+        </>
+      )}
+      {status === 'done' && <p>Uploaded: <a href={location}>{location}</a></p>}
+      {status === 'error' && <p style={{ color: 'red' }}>Upload failed</p>}
+    </div>
+  );
+}
+*/
+
+
+/*
+  Minimal HTML (vanilla):
+
+  <input type="file" id="fileInput" />
+  <progress id="progress" max="100" value="0"></progress>
+  <button id="cancelBtn">Cancel</button>
+*/

package/node/oss-files/oss-uploader-usage.md

@@ -0,0 +1,34 @@
+Here's a comprehensive JavaScript solution for reading and uploading large files to S3 using signed URLs, with multipart upload support for reliability:Here's a breakdown of what's in each file:
+
+---
+
+**`s3-uploader-client.js`** — Browser-side `S3Uploader` class
+- Files **≤ 5MB** → single `PUT` with a signed URL
+- Files **> 5MB** → S3 multipart upload (10MB chunks by default)
+- Concurrent part uploads (default: 3 in parallel) for speed
+- `XMLHttpRequest`-based for accurate `upload.progress` events
+- `AbortController` support to cancel mid-upload
+
+**`s3-uploader-server.js`** — Node.js/Express backend
+- Single `/api/s3/sign` endpoint handles all 5 phases: `single`, `initiate`, `part`, `complete`, `abort`
+- Uses the AWS SDK v3 (`@aws-sdk/client-s3` + `@aws-sdk/s3-request-presigner`)
+- Your AWS credentials never touch the browser
+
+**`s3-uploader-usage.js`** — Integration examples with cancel support
+
+---
+
+**Quick setup:**
+
+```bash
+# Backend
+npm install @aws-sdk/client-s3 @aws-sdk/s3-request-presigner express
+```
+
+Set env vars: `AWS_REGION`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `S3_BUCKET_NAME`
+
+**S3 CORS config** you'll also need on your bucket:
+```json
+[{ "AllowedOrigins": ["*"], "AllowedMethods": ["PUT"], "AllowedHeaders": ["*"], "ExposeHeaders": ["ETag"] }]
+```
+The `ETag` exposure is critical — S3 multipart complete requires the ETags from each part.