@myelinbridge/cli 0.8.1 → 0.9.0

package/README.md

@@ -38,6 +38,11 @@ npx @myelinbridge/cli push  ./run_042 --dataset onco1-wes --submit
   off, and retries up to 4 attempts before surfacing — an unattended pipeline
   run survives a blip. A retried `--submit` whose first attempt actually landed
   gets a success echo (`already_submitted: true`), not a false failure.
+- **Slow connections shrink their parts.** Since 0.9.0, when the same part dies
+  twice on a timeout (the edge kills any PUT that runs too long — HTTP 524),
+  `push` restarts the file with smaller parts, 64 → 16 → 5 MB, instead of
+  retrying the same slice into the same wall. A hospital-grade uplink down to
+  roughly 30 KB/s can now complete a delivery; it is slower, but it finishes.
 - **Every file is fingerprinted.** Since 0.7.0, `push` computes an MD5 of each
   file while it uploads and records it with the delivery. If the client's
   quality contract includes a checksum-manifest check, your delivery verifies

package/bin/myelin.js

@@ -195,6 +195,43 @@ function fmtBytes(n) {
 // them. Returns { upload_id, parts: [{part_number, etag}] } for confirm.
 const SIGN_WINDOW = 1000 // ≤ the server's max_parts_per_sign
 
+// Adaptive part geometry (mirrors lib/uploads/part-geometry.ts — this file
+// ships dependency-free and cannot import it). The edge in front of the S3
+// endpoint kills a PUT that outlives its patience (observed: HTTP 524 at
+// ~175 s), so a fixed 64 MiB part gives a slow uplink (under ~375 KiB/s) no
+// way to ever land one — every retry re-sends the same slice into the same
+// wall. The second timeout-shaped failure on the same part restarts the file
+// with smaller parts: 64 → 16 → 5 MiB (the S3 floor for any part but the
+// last), which moves the workable floor to ~30 KiB/s.
+const MIN_PART_SIZE = 5 * 1024 * 1024
+const SLOW_FAILURE_MS = 45_000
+const nextSmallerPartSize = (n) => Math.max(MIN_PART_SIZE, Math.floor(n / 4))
+const isTimeoutClassFailure = (status, elapsedMs) =>
+  status === 408 || status === 504 || status === 522 || status === 524 ||
+  (status === 0 && elapsedMs >= SLOW_FAILURE_MS)
+// A landed part is skippable on resume only when it holds the bytes the
+// CURRENT geometry puts at that part number — a leftover from an abandoned
+// geometry re-uploads under the same number, which replaces it. The part
+// listing reports no sizes, but a part's ETag IS the MD5 of its bytes, so the
+// check is: MD5 of the local slice == the landed ETag. This also catches a
+// file edited between runs, which skip-by-part-number silently stitched into
+// a mixed object.
+const etagMd5 = (etag) => {
+  const m = /^(?:W\/)?"?([0-9a-fA-F]{32})"?$/.exec(etag ?? '')
+  return m ? m[1].toLowerCase() : null
+}
+const md5Hex = (buf) => createHash('md5').update(buf).digest('hex')
+// Geometries a previous run may have used, most likely first: the requested
+// size, then the downshift ladder from the 64 MiB default.
+const partSizeCandidates = (requested) => {
+  const ladder = [requested]
+  for (let s = 64 * 1024 * 1024; ; s = nextSmallerPartSize(s)) {
+    if (!ladder.includes(s)) ladder.push(s)
+    if (s <= MIN_PART_SIZE) break
+  }
+  return ladder
+}
+
 async function readPart(absPath, start, end) {
   const chunks = []
   await new Promise((res, rej) => {
@@ -228,56 +265,133 @@ async function md5File(absPath) {
 }
 
 async function uploadFileMultipart(absPath, size, fileId, partSize, onPartDone) {
-  const numParts = Math.max(1, Math.ceil(size / partSize))
-  const allParts = Array.from({ length: numParts }, (_, i) => i + 1)
+  let adoptLandedGeometry = true
+  for (;;) {
+    const attempt = await uploadFileParts(absPath, size, fileId, partSize, adoptLandedGeometry, onPartDone)
+    if (!attempt.tooSlow) return attempt.result
+    const smaller = nextSmallerPartSize(attempt.partSize)
+    out(`  ${fmtBytes(attempt.partSize)} parts keep timing out on this connection — restarting with ${fmtBytes(smaller)} parts`)
+    partSize = smaller
+    // The restart deliberately abandons the old geometry — re-adopting part
+    // 1's old size from the landed listing would undo it.
+    adoptLandedGeometry = false
+  }
+}
+
+async function uploadFileParts(absPath, size, fileId, partSize, adoptLandedGeometry, onPartDone) {
+  let numParts = Math.max(1, Math.ceil(size / partSize))
   const etags = new Map() // part_number -> etag
+  const timeoutStrikes = new Map() // part_number -> timeout-shaped failures
   let uploadId = null
-
-  for (let i = 0; i < allParts.length; i += SIGN_WINDOW) {
-    const window = allParts.slice(i, i + SIGN_WINDOW)
-    const j = expectOk(await api('POST', `/files/${fileId}/upload-parts`, { part_numbers: window }), 'sign upload parts')
+  let tooSlow = false
+
+  for (let i = 0; i < numParts; i += SIGN_WINDOW) {
+    const windowParts = () =>
+      Array.from({ length: Math.min(SIGN_WINDOW, numParts - i) }, (_, k) => i + k + 1)
+    let j = expectOk(await api('POST', `/files/${fileId}/upload-parts`, { part_numbers: windowParts() }), 'sign upload parts')
+    if (i === 0 && adoptLandedGeometry && size > 0) {
+      // Resume across runs: an earlier run may have downshifted on a slow
+      // link, so the landed parts can belong to a smaller geometry. The
+      // ladder geometry whose local part-1 slice hashes to the landed part-1
+      // ETag is the geometry that run used — adopt it, or every landed part
+      // fails the content check and re-uploads for nothing.
+      const landedFirst = j.uploaded_parts.find((p) => p.part_number === 1)
+      const landedFirstMd5 = landedFirst ? etagMd5(landedFirst.etag) : null
+      if (landedFirstMd5) {
+        for (const candidate of partSizeCandidates(partSize)) {
+          const probe = await readPart(absPath, 0, Math.min(candidate, size) - 1)
+          if (md5Hex(probe) !== landedFirstMd5) continue
+          if (candidate !== partSize) {
+            partSize = candidate
+            numParts = Math.max(1, Math.ceil(size / partSize))
+            j = expectOk(await api('POST', `/files/${fileId}/upload-parts`, { part_numbers: windowParts() }), 'sign upload parts')
+          }
+          break
+        }
+      }
+    }
     uploadId = j.upload_id
-    for (const p of j.uploaded_parts) etags.set(p.part_number, p.etag) // already landed → skip
+    const landedByPart = new Map() // part_number -> { etag, md5 } (content-checkable landed parts)
+    for (const p of j.uploaded_parts) {
+      const md5 = etagMd5(p.etag)
+      if (md5) landedByPart.set(p.part_number, { etag: p.etag, md5 })
+      // An ETag that is not MD5-shaped cannot be content-checked; trust it by
+      // number (the pre-adaptive behavior) unless this attempt is a geometry
+      // restart, where the listing is known stale and the part re-uploads.
+      else if (adoptLandedGeometry) etags.set(p.part_number, p.etag)
+    }
     const urlByPart = new Map(j.urls.map((u) => [u.part_number, u.url]))
 
     // upload the still-missing parts in this window, up to 4 concurrent.
-    const missing = window.filter((n) => !etags.has(n))
+    const missing = windowParts().filter((n) => !etags.has(n))
     const queue = [...missing]
     // Re-sign one part. Returns a fresh URL — or null when the server reports
-    // the part ALREADY LANDED: a 524/timeout can kill the response after S3
-    // committed the bytes, and the server then returns no URL for that part,
-    // only its etag in uploaded_parts. That case is a success to record, not
-    // a dead end to die on.
-    const resign = async (partNo) => {
+    // the part ALREADY LANDED with the bytes we are sending: a 524/timeout
+    // can kill the response after S3 committed the bytes. That case is a
+    // success to record, not a dead end to die on.
+    const resign = async (partNo, bytes) => {
       const j = expectOk(await api('POST', `/files/${fileId}/upload-parts`, { part_numbers: [partNo] }), 're-sign upload part')
       uploadId = j.upload_id
       const landed = j.uploaded_parts.find((p) => p.part_number === partNo)
-      if (landed) { etags.set(partNo, landed.etag); return null }
+      if (landed) {
+        const md5 = etagMd5(landed.etag)
+        // Trust an unverifiable ETag here: this is our own attempt's commit.
+        if (!md5 || md5 === md5Hex(bytes)) { etags.set(partNo, landed.etag); return null }
+      }
       const fresh = j.urls.find((u) => u.part_number === partNo)?.url
-      if (!fresh) die(`upload part ${partNo}: could not re-sign (already finalized?)`)
+      if (!fresh) {
+        if (landed) die(`upload part ${partNo}: landed under a different part geometry and this server will not re-sign a landed part — delete the file from the draft and push again`)
+        die(`upload part ${partNo}: could not re-sign (already finalized?)`)
+      }
       urlByPart.set(partNo, fresh)
       return fresh
     }
     const workers = Array.from({ length: Math.min(4, queue.length) }, async () => {
       for (;;) {
+        if (tooSlow) return
         const partNo = queue.shift()
         if (partNo === undefined) return
         const start = (partNo - 1) * partSize
         const end = Math.min(start + partSize, size) - 1
         const bytes = await readPart(absPath, start, end)
+        // Resume: a landed part holding exactly these bytes is recorded, not
+        // re-sent. A mismatch (abandoned geometry, or the file changed on
+        // disk since the last run) falls through and re-uploads the part
+        // number, which replaces it.
+        const landed = landedByPart.get(partNo)
+        if (landed && md5Hex(bytes) === landed.md5) {
+          etags.set(partNo, landed.etag)
+          if (onPartDone) onPartDone()
+          continue
+        }
         // Expired URLs (401/403), transient storage errors (5xx), edge kills
         // (524) and throttling all deserve the same treatment: re-sign — always
         // safe, upload-parts is idempotent — back off, try again. Bounded at 4
         // attempts, then the error surfaces. An unattended pipeline should
         // survive a hiccup, not page a human for it; `push` stays resumable
-        // either way.
+        // either way. EXCEPT timeout-shaped failures: the second one on the
+        // same part flags a geometry restart — retrying the same slice into
+        // the same timeout gets nowhere on a slow uplink.
         let put = null
         for (let attempt = 1; ; attempt++) {
-          const url = attempt === 1 ? urlByPart.get(partNo) : await resign(partNo)
+          if (tooSlow) return
+          let url = attempt === 1 ? urlByPart.get(partNo) : undefined
+          if (url === undefined) url = await resign(partNo, bytes)
           if (url === null) break // part landed server-side — resign() recorded its etag
+          const startedAt = Date.now()
           put = await fetch(url, { method: 'PUT', body: bytes }).catch(() => null)
           if (put && put.status === 200) break
+          const status = put ? put.status : 0
           const reason = put ? `HTTP ${put.status}` : 'network error'
+          if (isTimeoutClassFailure(status, Date.now() - startedAt)) {
+            const strikes = (timeoutStrikes.get(partNo) ?? 0) + 1
+            timeoutStrikes.set(partNo, strikes)
+            if (strikes >= 2 && partSize > MIN_PART_SIZE) {
+              out(`  part ${partNo}: ${reason} again — parts too large for this connection`)
+              tooSlow = true
+              return
+            }
+          }
           if (attempt >= 4) die(`upload part ${partNo}: ${reason} after ${attempt} attempts — re-run to resume`)
           const wait = [1, 4, 10][attempt - 1] + Math.random()
           out(`  part ${partNo}: ${reason} — retrying in ${Math.round(wait)}s (${attempt}/3)`)
@@ -292,9 +406,15 @@ async function uploadFileMultipart(absPath, size, fileId, partSize, onPartDone)
       }
     })
     await Promise.all(workers)
+    if (tooSlow) return { tooSlow: true, partSize }
   }
 
-  return { upload_id: uploadId, parts: allParts.map((n) => ({ part_number: n, etag: etags.get(n) })) }
+  const allParts = Array.from({ length: numParts }, (_, n) => n + 1)
+  return {
+    tooSlow: false,
+    partSize,
+    result: { upload_id: uploadId, parts: allParts.map((n) => ({ part_number: n, etag: etags.get(n) })) },
+  }
 }
 
 // ——— commands ———

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@myelinbridge/cli",
-  "version": "0.8.1",
+  "version": "0.9.0",
   "description": "Myelin Partner Ingestion CLI — push R&D data deliveries from a pipeline: preflight against the client's quality rules, resumable upload, submit, track review outcomes.",
   "type": "module",
   "bin": {