@myelinbridge/cli 0.6.1 → 0.8.1

package/README.md

@@ -22,15 +22,26 @@ npx @myelinbridge/cli push  ./run_042 --dataset onco1-wes --submit
   sample sits — `0` = the delivery root is one sample, `1` (default) = each top-level
   folder is a sample, `2` = one level deeper. Sample-scoped quality checks group by it,
   so getting it right up front is what makes per-sample verdicts mean anything. It is
-  **idempotent** (safe to assert on every pipeline run) and **locks once your first
-  batch leaves draft**, so that verdicts stay comparable across deliveries — after that
-  the command exits `2`. This is the only dataset field you can write.
+  **idempotent** (safe to assert on every pipeline run, before and after the lock) and
+  **locks once your first batch leaves draft**, so that verdicts stay comparable across
+  deliveries — after that, re-asserting the current value still succeeds and only a
+  *change* exits `2`. This is the only dataset field you can write.
 
 - **Resume = re-run.** `push` is idempotent: already-uploaded files are skipped
   (path + size), and within a large file, parts that already landed are skipped
   too (S3 multipart). Uploads go direct to storage over short-lived presigned
   URLs — no credential is stored on your machine, and revoking the API key cuts
   off signing immediately.
+- **Transient failures retry themselves.** Since 0.8.0 a rate limit (`429`)
+  waits out `Retry-After` and retries on every command, and a part upload that
+  hits a storage hiccup (5xx, an edge timeout, an expired URL) re-signs, backs
+  off, and retries up to 4 attempts before surfacing — an unattended pipeline
+  run survives a blip. A retried `--submit` whose first attempt actually landed
+  gets a success echo (`already_submitted: true`), not a false failure.
+- **Every file is fingerprinted.** Since 0.7.0, `push` computes an MD5 of each
+  file while it uploads and records it with the delivery. If the client's
+  quality contract includes a checksum-manifest check, your delivery verifies
+  against your own `md5_manifest.csv` instead of reading as "unverifiable".
 - **Read the contract before you build the delivery.** `contract` prints what the
   client expects — every check as one plain sentence, grouped by what it answers
   (completeness, structure, validity, consistency, integrity, privacy), and marked
@@ -61,7 +72,7 @@ partner's data, scope it to that partner's projects.
 ```bash
 export MYELIN_API_KEY=myl_live_…
 
-# What has landed, newest first
+# What has landed, newest first (page with --cursor from the printed next_cursor)
 myelin deliveries --dataset <dataset-id>
 
 # What is this object, exactly?
@@ -90,6 +101,12 @@ A partner key calling these gets `403 wrong_key_side`, and vice versa.
 Every command takes `--json`. Exit codes: `0` ok · `1` error · `2` blocked
 (blocking preflight failure, locked delivery, blocked submit, locked sample depth).
 
+API errors carry structure, not just prose: in `--json` mode an error is
+`{ "error", "code", "status", "request_id" }` — branch on `code`, and quote the
+`request_id` when reporting a problem (it is printed in human mode too; it lets
+Myelin find the exact server-side log line). `myelin version` prints the CLI
+version.
+
 ## Environment
 
 | Variable | |

package/bin/myelin.js

@@ -10,10 +10,12 @@
 
 import { readdirSync, statSync, createReadStream, readFileSync } from 'node:fs'
 import { resolve, join, relative, sep, basename } from 'node:path'
+import { createHash } from 'node:crypto'
 import process from 'node:process'
 
 const API_URL = (process.env.MYELIN_API_URL ?? 'https://myelinbridge.com/api/v1').replace(/\/$/, '')
 const KEY = process.env.MYELIN_API_KEY
+const VERSION = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version
 
 // Extra headers sent with every API call — newline-separated "Name: value"
 // pairs in MYELIN_API_HEADER. For deployments fronted by something that
@@ -48,13 +50,19 @@ class CliExit extends Error {}
 // assertion on Windows, which printed a C stack trace after a perfectly good
 // error message and replaced the documented exit code with 127. Node exits on
 // its own once the request settles — measured at ~130 ms, no keep-alive stall.
-const die = (message, code = 1) => {
-  if (JSON_MODE) console.log(JSON.stringify({ error: message }))
-  else console.error(`✗ ${message}`)
+// `extra` fields ride into the --json error object as structured data.
+const die = (message, code = 1, extra = null) => {
+  if (JSON_MODE) console.log(JSON.stringify({ error: message, ...(extra ?? {}) }))
+  else {
+    console.error(`✗ ${message}`)
+    if (extra?.request_id) console.error(`  request id ${extra.request_id} — include it if you report this`)
+  }
   process.exitCode = code
   throw new CliExit(message)
 }
 
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
+
 // Column widths come from the content. padEnd() alone silently ran a long
 // value into the next column — a 28-character dataset slug swallowed the
 // STATUS header's gutter. Pass headers = null for an unheadered list.
@@ -79,27 +87,61 @@ function opt(name) {
   return i >= 0 && args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : null
 }
 
+// 429s are handled HERE so every command inherits the platform's documented
+// contract (honour Retry-After): bounded — 4 waits, then the error surfaces
+// like any other. The wait is capped at 60 s so a pathological header cannot
+// stall a pipeline for an hour.
 async function api(method, path, body, raw) {
-  const res = await fetch(API_URL + path, {
-    method,
-    headers: {
-      ...EXTRA_HEADERS,
-      Authorization: `Bearer ${KEY}`,
-      ...(body && !raw ? { 'Content-Type': 'application/json' } : {}),
-    },
-    body: raw ? body : body ? JSON.stringify(body) : undefined,
-  }).catch((err) => die(`Cannot reach ${API_URL}: ${err.message}`))
-  let json = null
-  try { json = await res.json() } catch { /* non-JSON body */ }
-  return { status: res.status, json }
+  for (let attempt = 1; ; attempt++) {
+    const res = await fetch(API_URL + path, {
+      method,
+      headers: {
+        ...EXTRA_HEADERS,
+        Authorization: `Bearer ${KEY}`,
+        ...(body && !raw ? { 'Content-Type': 'application/json' } : {}),
+      },
+      body: raw ? body : body ? JSON.stringify(body) : undefined,
+    }).catch((err) => die(`Cannot reach ${API_URL}: ${err.message}`))
+    let json = null
+    try { json = await res.json() } catch { /* non-JSON body */ }
+    const requestId = res.headers.get('myelin-request-id') ?? json?.request_id ?? null
+    if (res.status === 429 && attempt <= 4) {
+      const wait = Math.min(Number(res.headers.get('retry-after') ?? json?.retry_after) || 5, 60)
+      out(`  rate limited — retrying in ${wait}s (${attempt}/4)`)
+      await sleep(wait * 1000 + Math.random() * 500)
+      continue
+    }
+    return { status: res.status, json, requestId }
+  }
 }
 
-function expectOk(r, context) {
-  if (r.status >= 200 && r.status < 300) return r.json
+const BLOCKED_CODES = ['delivery_locked', 'submit_blocked', 'dataset_not_active', 'api_disabled', 'bridge_paused', 'sample_depth_locked']
+
+// Every API failure surfaces the stable `code` AND the request id the server
+// echoed — the id is what lets Myelin find the exact server-side log line for
+// a support report. In --json mode both are structured fields, never prose a
+// script has to regex out of the message.
+function apiDie(context, r) {
   const detail = r.json?.detail ?? `HTTP ${r.status}`
   const code = r.json?.code ?? 'error'
-  const blocked = ['delivery_locked', 'submit_blocked', 'dataset_not_active', 'api_disabled', 'bridge_paused', 'sample_depth_locked'].includes(code)
-  die(`${context}: [${code}] ${detail}`, blocked ? 2 : 1)
+  die(`${context}: [${code}] ${detail}`, BLOCKED_CODES.includes(code) ? 2 : 1, {
+    code,
+    status: r.status,
+    ...(r.requestId ? { request_id: r.requestId } : {}),
+  })
+}
+
+function expectOk(r, context) {
+  if (r.status >= 200 && r.status < 300) {
+    // A 2xx that isn't JSON is almost always a front door answering instead
+    // of Myelin (a gateway login page, an SSO-protected preview whose pass
+    // lapsed). Name the likely cause instead of crashing on j.<field>.
+    if (r.json == null) {
+      die(`${context}: the server answered ${r.status} but not with JSON — if this deployment sits behind a gateway or front door, your pass (MYELIN_API_HEADER) may have lapsed. Re-mint it and retry.`)
+    }
+    return r.json
+  }
+  apiDie(context, r)
 }
 
 // ——— dataset resolution (id or slug) ———
@@ -122,9 +164,15 @@ async function resolveDataset(ref) {
 function walkDir(root) {
   const files = []
   const walk = (dir) => {
-    for (const entry of readdirSync(dir)) {
+    let entries
+    try { entries = readdirSync(dir) } catch (err) { die(`Cannot read directory ${dir}: ${err.message}`) }
+    for (const entry of entries) {
       const full = join(dir, entry)
-      const st = statSync(full)
+      // A broken symlink or unreadable file gets a named error, not a stack —
+      // everything under the directory ships, so an unreadable entry is the
+      // partner's to fix or remove, and the message should say which one.
+      let st
+      try { st = statSync(full) } catch (err) { die(`Cannot read ${full}: ${err.message} — fix or remove it (every file under the directory is included)`) }
       if (st.isDirectory()) walk(full)
       else files.push({ abs: full, path: '/' + relative(root, full).split(sep).join('/'), size: st.size })
     }
@@ -158,6 +206,27 @@ async function readPart(absPath, start, end) {
   return Buffer.concat(chunks)
 }
 
+// Whole-file MD5, lowercase hex — the exact format md5sum-style manifests use
+// and the only one the platform's manifest_checksums_match check can verify.
+// A separate sequential read (parts upload 4-wide, out of order — a streaming
+// hash cannot ride along), overlapped with the upload by the caller. Returns
+// null on read failure: a hash miss must degrade the delivery to
+// "unverifiable", never kill the push.
+async function md5File(absPath) {
+  try {
+    const hash = createHash('md5')
+    await new Promise((res, rej) => {
+      const s = createReadStream(absPath)
+      s.on('data', (c) => hash.update(c))
+      s.on('end', res)
+      s.on('error', rej)
+    })
+    return hash.digest('hex')
+  } catch {
+    return null
+  }
+}
+
 async function uploadFileMultipart(absPath, size, fileId, partSize, onPartDone) {
   const numParts = Math.max(1, Math.ceil(size / partSize))
   const allParts = Array.from({ length: numParts }, (_, i) => i + 1)
@@ -172,12 +241,18 @@ async function uploadFileMultipart(absPath, size, fileId, partSize, onPartDone)
     const urlByPart = new Map(j.urls.map((u) => [u.part_number, u.url]))
 
     // upload the still-missing parts in this window, up to 4 concurrent.
-    // Presigned URLs expire (~30 min); on a rejection we re-sign that one part
-    // and retry once — cheap, since upload-parts is idempotent.
     const missing = window.filter((n) => !etags.has(n))
     const queue = [...missing]
+    // Re-sign one part. Returns a fresh URL — or null when the server reports
+    // the part ALREADY LANDED: a 524/timeout can kill the response after S3
+    // committed the bytes, and the server then returns no URL for that part,
+    // only its etag in uploaded_parts. That case is a success to record, not
+    // a dead end to die on.
     const resign = async (partNo) => {
       const j = expectOk(await api('POST', `/files/${fileId}/upload-parts`, { part_numbers: [partNo] }), 're-sign upload part')
+      uploadId = j.upload_id
+      const landed = j.uploaded_parts.find((p) => p.part_number === partNo)
+      if (landed) { etags.set(partNo, landed.etag); return null }
       const fresh = j.urls.find((u) => u.part_number === partNo)?.url
       if (!fresh) die(`upload part ${partNo}: could not re-sign (already finalized?)`)
       urlByPart.set(partNo, fresh)
@@ -190,14 +265,29 @@ async function uploadFileMultipart(absPath, size, fileId, partSize, onPartDone)
         const start = (partNo - 1) * partSize
         const end = Math.min(start + partSize, size) - 1
         const bytes = await readPart(absPath, start, end)
-        let put = await fetch(urlByPart.get(partNo), { method: 'PUT', body: bytes }).catch(() => null)
-        if (!put || put.status === 403 || put.status === 401) {
-          put = await fetch(await resign(partNo), { method: 'PUT', body: bytes }).catch((e) => die(`upload part ${partNo}: ${e.message}`))
+        // Expired URLs (401/403), transient storage errors (5xx), edge kills
+        // (524) and throttling all deserve the same treatment: re-sign — always
+        // safe, upload-parts is idempotent — back off, try again. Bounded at 4
+        // attempts, then the error surfaces. An unattended pipeline should
+        // survive a hiccup, not page a human for it; `push` stays resumable
+        // either way.
+        let put = null
+        for (let attempt = 1; ; attempt++) {
+          const url = attempt === 1 ? urlByPart.get(partNo) : await resign(partNo)
+          if (url === null) break // part landed server-side — resign() recorded its etag
+          put = await fetch(url, { method: 'PUT', body: bytes }).catch(() => null)
+          if (put && put.status === 200) break
+          const reason = put ? `HTTP ${put.status}` : 'network error'
+          if (attempt >= 4) die(`upload part ${partNo}: ${reason} after ${attempt} attempts — re-run to resume`)
+          const wait = [1, 4, 10][attempt - 1] + Math.random()
+          out(`  part ${partNo}: ${reason} — retrying in ${Math.round(wait)}s (${attempt}/3)`)
+          await sleep(wait * 1000)
+        }
+        if (put && put.status === 200) {
+          const etag = put.headers.get('etag')
+          if (!etag) die(`upload part ${partNo}: storage returned no ETag`)
+          etags.set(partNo, etag)
         }
-        if (put.status !== 200) die(`upload part ${partNo}: HTTP ${put.status}`)
-        const etag = put.headers.get('etag')
-        if (!etag) die(`upload part ${partNo}: storage returned no ETag`)
-        etags.set(partNo, etag)
         if (onPartDone) onPartDone()
       }
     })
@@ -352,8 +442,27 @@ async function cmdPush() {
   if (files.length === 0) die(`No files under ${root}`)
   const dataset = await resolveDataset(dsRef)
 
-  // 1. create or resume the draft
-  const created = expectOk(await api('POST', `/datasets/${dataset.id}/batches`), 'create batch')
+  // 1. create or resume the draft. A 409 delivery_locked here can BE the
+  //    lost-response retry case: our previous run's --submit landed, the
+  //    batch left draft, and the pipeline re-ran the whole command. Before
+  //    claiming success, verify the in-flight batch actually IS this delivery
+  //    (same paths, same sizes) — an unrelated delivery under review must
+  //    keep the honest blocked exit.
+  const createRes = await api('POST', `/datasets/${dataset.id}/batches`)
+  if (createRes.status === 409 && createRes.json?.code === 'delivery_locked' && createRes.json?.batch_id && flag('submit')) {
+    const b = await api('GET', `/batches/${createRes.json.batch_id}`)
+    const remote = b.status === 200 ? (b.json.batch?.files ?? []) : []
+    const matches =
+      remote.length === files.length &&
+      files.every((f) => remote.some((r) => r.path === f.path && r.size_bytes === f.size))
+    if (matches) {
+      const j = expectOk(await api('POST', `/batches/${createRes.json.batch_id}/submit`), 'submit')
+      emit(j)
+      out(`✓ This delivery is already submitted (${j.status}) — nothing to redo. Track: myelin status ${createRes.json.batch_id} --watch`)
+      return
+    }
+  }
+  const created = expectOk(createRes, 'create batch')
   const batchId = created.batch_id
   out(`Draft ${created.resumed ? 'resumed' : 'created'} (${batchId.slice(0, 8)}…).`)
 
@@ -367,6 +476,7 @@ async function cmdPush() {
       replace: flag('replace'),
     })
     if (r.status !== 200 && r.status !== 409) expectOk(r, 'declare')
+    if (!r.json?.files) apiDie('declare', r) // 409 with a non-JSON body (a proxy page, an edge error) must not crash
     const conflicts = r.json.files.filter((f) => f.error)
     if (conflicts.length > 0) {
       for (const c of conflicts) out(`✗ ${c.path}: ${c.error}`)
@@ -389,9 +499,20 @@ async function cmdPush() {
       const d = queue.shift()
       if (!d) return
       const local = byPath.get(d.path)
-      const { upload_id, parts } = await uploadFileMultipart(local.abs, local.size, d.file_id, partSize)
-      const c = await api('POST', `/files/${d.file_id}/confirm`, { upload_id, parts })
-      if (c.status !== 200) die(`confirm ${d.path}: [${c.json?.code}] ${c.json?.detail ?? c.status}`)
+      // Hash and upload overlap — two independent reads of the same file. The
+      // checksum rides on confirm so files.checksum is populated for CLI
+      // pushes (0.7.0): without it every CLI delivery reads as "unverifiable"
+      // to the manifest_checksums_match quality check.
+      const [{ upload_id, parts }, checksum] = await Promise.all([
+        uploadFileMultipart(local.abs, local.size, d.file_id, partSize),
+        md5File(local.abs),
+      ])
+      const c = await api('POST', `/files/${d.file_id}/confirm`, {
+        upload_id,
+        parts,
+        ...(checksum ? { checksum, checksum_algorithm: 'md5' } : {}),
+      })
+      if (c.status !== 200) apiDie(`confirm ${d.path}`, c)
       done++
       out(`  ✓ ${d.path} (${done}/${pending.length})`)
     }
@@ -401,15 +522,17 @@ async function cmdPush() {
 
   // 4. optional submit
   if (flag('submit')) {
-    const r = await api('POST', `/batches/${batchId}/submit`)
-    if (r.status !== 200) {
-      const code = r.json?.code ?? 'error'
-      die(`submit: [${code}] ${r.json?.detail ?? r.status}`, code === 'submit_blocked' ? 2 : 1)
+    const j = expectOk(await api('POST', `/batches/${batchId}/submit`), 'submit')
+    emit(j)
+    if (j.already_submitted) {
+      // A retried submit whose first attempt actually landed: the API echoes
+      // success instead of failing, and so do we — nothing to redo.
+      out(`✓ Batch already submitted (${j.status}) — nothing to redo. Track: myelin status ${batchId} --watch`)
+    } else {
+      const a = j.auto_checks
+      out(`Submitting… auto-checks: ${a.passed} passed, ${a.flagged} flagged, ${a.failed} failed.`)
+      out(`✓ Batch submitted for review. Track: myelin status ${batchId} --watch`)
     }
-    const a = r.json.auto_checks
-    emit(r.json)
-    out(`Submitting… auto-checks: ${a.passed} passed, ${a.flagged} flagged, ${a.failed} failed.`)
-    out(`✓ Batch submitted for review. Track: myelin status ${batchId} --watch`)
   } else {
     emit({ batch_id: batchId, files: declared.length, submitted: false })
     out(`Draft ready (not submitted). Submit with: myelin push ${dir} --dataset ${dsRef} --submit`)
@@ -423,7 +546,7 @@ async function cmdStatus() {
   const findBatch = async () => {
     if (/^[0-9a-f]{8}-[0-9a-f]{4}-/i.test(ref)) {
       const r = await api('GET', `/batches/${ref}`)
-      if (r.status === 200) return r.json.batch
+      if (r.status === 200) return expectOk(r, 'batch').batch
       die(`Batch ${ref} not found in this key's scope`)
     }
     const list = expectOk(await api('GET', '/batches?limit=100'), 'batches')
@@ -463,7 +586,8 @@ async function cmdSandbox() {
   const file = args[1]
   if (!file) die('Usage: myelin sandbox <file>')
   const abs = resolve(file)
-  const size = statSync(abs).size
+  let size
+  try { size = statSync(abs).size } catch (err) { die(`Cannot read ${abs}: ${err.message}`) }
   if (size > 4 * 1024 * 1024) die('Sandbox files should stay under ~4 MB — this verifies connectivity, not throughput.')
 
   const me = expectOk(await api('GET', '/me'), 'auth')
@@ -523,22 +647,26 @@ Commands:
 
 Client-side commands (need a CLIENT key — Organisation → API keys):
   deliveries [--project <id>]             what landed in your destination bucket
-             [--dataset <id>] [--since <iso>] [--limit <n>]
+             [--dataset <id>] [--since <iso>] [--limit <n>] [--cursor <ts>]
   resolve <path|prefix|id>                what a path in your bucket actually is —
                                           project, dataset, batch, file
 
-Every command accepts --json. Exit codes: 0 ok · 1 error · 2 blocked.`)
+  version                                 print the CLI version
+
+Every command accepts --json. Exit codes: 0 ok · 1 error · 2 blocked.
+Errors print the API request id — include it when reporting a problem.`)
 }
 
 // ——— client-side commands (client keys only) ———
 //
-// These need a CLIENT key, minted by the client bridge owner in Bridge → API →
-// "Your read keys". A partner key gets 403 wrong_key_side, and the message says
-// so — the two sides deliberately do not silently degrade into each other.
+// These need a CLIENT key, minted by a tenant admin in Organisation → API keys
+// (tenant-scoped since 2026-08-08). A partner key gets 403 wrong_key_side, and
+// the message says so — the two sides deliberately do not silently degrade
+// into each other.
 
 async function cmdDeliveries() {
   const qs = new URLSearchParams()
-  const map = { project: 'project_id', dataset: 'dataset_id', since: 'since', limit: 'limit' }
+  const map = { project: 'project_id', dataset: 'dataset_id', since: 'since', limit: 'limit', cursor: 'cursor' }
   for (const [flagName, param] of Object.entries(map)) {
     const v = opt(flagName)
     if (v) qs.set(param, v)
@@ -559,7 +687,15 @@ async function cmdDeliveries() {
     d.destination_root ?? '—',
   ])
   for (const line of table(['PROJECT', 'DATASET', 'BATCH', 'FILES', 'DELIVERED', 'LOCATION'], rows)) out(line)
-  if (j.next_cursor) out(`\n… more — re-run with --since ${j.next_cursor}`)
+  if (j.next_cursor) {
+    // Carry the invocation's own filters into the hint — a hint that drops
+    // --dataset pages the unfiltered list and interleaves other datasets.
+    const carried = ['project', 'dataset', 'since', 'limit']
+      .filter((f) => opt(f))
+      .map((f) => `--${f} ${opt(f)}`)
+      .join(' ')
+    out(`\n… more — next page: myelin deliveries ${carried ? `${carried} ` : ''}--cursor ${j.next_cursor}`)
+  }
 }
 
 async function cmdResolve() {
@@ -590,6 +726,10 @@ if (!command || command === 'help' || command === '--help') {
   cmdHelp()
   process.exit(0)
 }
+if (command === 'version' || command === '--version' || command === '-v') {
+  console.log(JSON_MODE ? JSON.stringify({ version: VERSION }) : VERSION)
+  process.exit(0)
+}
 
 const commands = {
   ping: cmdPing,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@myelinbridge/cli",
-  "version": "0.6.1",
+  "version": "0.8.1",
   "description": "Myelin Partner Ingestion CLI — push R&D data deliveries from a pipeline: preflight against the client's quality rules, resumable upload, submit, track review outcomes.",
   "type": "module",
   "bin": {