@myelinbridge/cli 0.1.0 → 0.2.0

package/README.md

@@ -16,7 +16,10 @@ npx @myelinbridge/cli push  ./run_042 --dataset onco1-wes --submit
 ```
 
 - **Resume = re-run.** `push` is idempotent: already-uploaded files are skipped
-  (path + size), interrupted uploads resume mid-file over TUS.
+  (path + size), and within a large file, parts that already landed are skipped
+  too (S3 multipart). Uploads go direct to storage over short-lived presigned
+  URLs — no credential is stored on your machine, and revoking the API key cuts
+  off signing immediately.
 - **`check` costs nothing.** It evaluates your local manifest against the dataset's
   quality checks server-side — same engine, same verdicts as submit — without
   uploading. Exit code 2 means a blocking rule fails.

package/bin/myelin.js

@@ -11,7 +11,6 @@
 import { readdirSync, statSync, createReadStream, readFileSync } from 'node:fs'
 import { resolve, join, relative, sep, basename } from 'node:path'
 import process from 'node:process'
-import * as tus from 'tus-js-client'
 
 const API_URL = (process.env.MYELIN_API_URL ?? 'https://myelinbridge.com/api/v1').replace(/\/$/, '')
 const KEY = process.env.MYELIN_API_KEY
@@ -97,24 +96,71 @@ function fmtBytes(n) {
   return n + ' B'
 }
 
-function tusUpload(absPath, size, storagePath, upload) {
-  return new Promise((resolveP, rejectP) => {
-    new tus.Upload(createReadStream(absPath), {
-      endpoint: upload.endpoint,
-      chunkSize: upload.chunk_size_bytes,
-      uploadSize: size,
-      retryDelays: [0, 1000, 5000],
-      headers: { authorization: `Bearer ${upload.token}`, apikey: upload.apikey, ...upload.headers },
-      metadata: {
-        bucketName: upload.bucket,
-        objectName: storagePath,
-        contentType: 'application/octet-stream',
-        cacheControl: '3600',
-      },
-      onError: rejectP,
-      onSuccess: () => resolveP(),
-    }).start()
+// Upload a file via S3 presigned multipart (SEC-TOKEN-01: no bearer token ever
+// reaches the client — declare tells us the part size, and each part gets a
+// short-lived presigned PUT URL scoped to exactly that one part of one object).
+// Resumable: the sign step reports parts that already landed, so a re-run skips
+// them. Returns { upload_id, parts: [{part_number, etag}] } for confirm.
+const SIGN_WINDOW = 1000 // ≤ the server's max_parts_per_sign
+
+async function readPart(absPath, start, end) {
+  const chunks = []
+  await new Promise((res, rej) => {
+    const s = createReadStream(absPath, { start, end }) // end inclusive
+    s.on('data', (c) => chunks.push(c))
+    s.on('end', res)
+    s.on('error', rej)
   })
+  return Buffer.concat(chunks)
+}
+
+async function uploadFileMultipart(absPath, size, fileId, partSize, onPartDone) {
+  const numParts = Math.max(1, Math.ceil(size / partSize))
+  const allParts = Array.from({ length: numParts }, (_, i) => i + 1)
+  const etags = new Map() // part_number -> etag
+  let uploadId = null
+
+  for (let i = 0; i < allParts.length; i += SIGN_WINDOW) {
+    const window = allParts.slice(i, i + SIGN_WINDOW)
+    const j = expectOk(await api('POST', `/files/${fileId}/upload-parts`, { part_numbers: window }), 'sign upload parts')
+    uploadId = j.upload_id
+    for (const p of j.uploaded_parts) etags.set(p.part_number, p.etag) // already landed → skip
+    const urlByPart = new Map(j.urls.map((u) => [u.part_number, u.url]))
+
+    // upload the still-missing parts in this window, up to 4 concurrent.
+    // Presigned URLs expire (~30 min); on a rejection we re-sign that one part
+    // and retry once — cheap, since upload-parts is idempotent.
+    const missing = window.filter((n) => !etags.has(n))
+    const queue = [...missing]
+    const resign = async (partNo) => {
+      const j = expectOk(await api('POST', `/files/${fileId}/upload-parts`, { part_numbers: [partNo] }), 're-sign upload part')
+      const fresh = j.urls.find((u) => u.part_number === partNo)?.url
+      if (!fresh) die(`upload part ${partNo}: could not re-sign (already finalized?)`)
+      urlByPart.set(partNo, fresh)
+      return fresh
+    }
+    const workers = Array.from({ length: Math.min(4, queue.length) }, async () => {
+      for (;;) {
+        const partNo = queue.shift()
+        if (partNo === undefined) return
+        const start = (partNo - 1) * partSize
+        const end = Math.min(start + partSize, size) - 1
+        const bytes = await readPart(absPath, start, end)
+        let put = await fetch(urlByPart.get(partNo), { method: 'PUT', body: bytes }).catch(() => null)
+        if (!put || put.status === 403 || put.status === 401) {
+          put = await fetch(await resign(partNo), { method: 'PUT', body: bytes }).catch((e) => die(`upload part ${partNo}: ${e.message}`))
+        }
+        if (put.status !== 200) die(`upload part ${partNo}: HTTP ${put.status}`)
+        const etag = put.headers.get('etag')
+        if (!etag) die(`upload part ${partNo}: storage returned no ETag`)
+        etags.set(partNo, etag)
+        if (onPartDone) onPartDone()
+      }
+    })
+    await Promise.all(workers)
+  }
+
+  return { upload_id: uploadId, parts: allParts.map((n) => ({ part_number: n, etag: etags.get(n) })) }
 }
 
 // ——— commands ———
@@ -226,20 +272,22 @@ async function cmdPush() {
     declared.push(...r.json.files)
     upload = r.json.upload
   }
+  const partSize = upload?.part_size ?? 64 * 1024 * 1024
 
-  // 3. upload the delta (anything not already 'uploaded'), 4 in parallel
+  // 3. upload the delta (anything not already 'uploaded') via S3 multipart,
+  //    2 files in parallel (each file uploads its parts 4-wide internally)
   const byPath = new Map(files.map((f) => [f.path, f]))
   const pending = declared.filter((d) => d.upload_state !== 'uploaded')
   out(`Uploading ${pending.length}/${declared.length} files (${fmtBytes(pending.reduce((s, d) => s + (byPath.get(d.path)?.size ?? 0), 0))}) · resumable`)
   let done = 0
   const queue = [...pending]
-  const workers = Array.from({ length: Math.min(4, queue.length) }, async () => {
+  const workers = Array.from({ length: Math.min(2, queue.length) }, async () => {
     for (;;) {
       const d = queue.shift()
       if (!d) return
       const local = byPath.get(d.path)
-      await tusUpload(local.abs, local.size, d.storage_path, upload)
-      const c = await api('POST', `/files/${d.file_id}/confirm`)
+      const { upload_id, parts } = await uploadFileMultipart(local.abs, local.size, d.file_id, partSize)
+      const c = await api('POST', `/files/${d.file_id}/confirm`, { upload_id, parts })
       if (c.status !== 200) die(`confirm ${d.path}: [${c.json?.code}] ${c.json?.detail ?? c.status}`)
       done++
       out(`  ✓ ${d.path} (${done}/${pending.length})`)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@myelinbridge/cli",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Myelin Partner Ingestion CLI — push R&D data deliveries from a pipeline: preflight against the client's quality rules, resumable upload, submit, track review outcomes.",
   "type": "module",
   "bin": {
@@ -13,14 +13,11 @@
   "engines": {
     "node": ">=20"
   },
-  "dependencies": {
-    "tus-js-client": "^4.1.0"
-  },
   "keywords": [
     "myelin",
     "pharma",
     "ingestion",
-    "tus"
+    "s3"
   ],
   "license": "UNLICENSED"
 }