@myelinbridge/cli 0.1.0 → 0.4.0

package/README.md

@@ -11,12 +11,33 @@ export MYELIN_API_KEY=myl_live_…     # created by your bridge owner in Bridge
 
 npx @myelinbridge/cli ping                 # verifies auth, prints your projects
 npx @myelinbridge/cli datasets             # what you can deliver to, and whose move it is
+npx @myelinbridge/cli contract --dataset onco1-wes            # what is expected of your delivery
+npx @myelinbridge/cli sample-depth 1 --dataset onco1-wes      # once, before your first submit
 npx @myelinbridge/cli check ./run_042 --dataset onco1-wes    # validate BEFORE uploading a byte
 npx @myelinbridge/cli push  ./run_042 --dataset onco1-wes --submit
 ```
 
+- **Declare your sample depth once, first.** The client creates and describes the
+  dataset; you own your output structure, so you tell Myelin at which folder depth a
+  sample sits — `0` = the delivery root is one sample, `1` (default) = each top-level
+  folder is a sample, `2` = one level deeper. Sample-scoped quality checks group by it,
+  so getting it right up front is what makes per-sample verdicts mean anything. It is
+  **idempotent** (safe to assert on every pipeline run) and **locks once your first
+  batch leaves draft**, so that verdicts stay comparable across deliveries — after that
+  the command exits `2`. This is the only dataset field you can write.
+
 - **Resume = re-run.** `push` is idempotent: already-uploaded files are skipped
-  (path + size), interrupted uploads resume mid-file over TUS.
+  (path + size), and within a large file, parts that already landed are skipped
+  too (S3 multipart). Uploads go direct to storage over short-lived presigned
+  URLs — no credential is stored on your machine, and revoking the API key cuts
+  off signing immediately.
+- **Read the contract before you build the delivery.** `contract` prints what the
+  client expects — every check as one plain sentence, grouped by what it answers
+  (completeness, structure, validity, consistency, integrity, privacy), and marked
+  `!` when a failure blocks validation. It also tells you which checks `check` can
+  verify locally and which only run once the files are uploaded, so nothing about
+  the bar is a surprise at review time.
+
 - **`check` costs nothing.** It evaluates your local manifest against the dataset's
   quality checks server-side — same engine, same verdicts as submit — without
   uploading. Exit code 2 means a blocking rule fails.
@@ -28,7 +49,7 @@ npx @myelinbridge/cli push  ./run_042 --dataset onco1-wes --submit
 ## Machine mode
 
 Every command takes `--json`. Exit codes: `0` ok · `1` error · `2` blocked
-(blocking preflight failure, locked delivery, blocked submit).
+(blocking preflight failure, locked delivery, blocked submit, locked sample depth).
 
 ## Webhooks instead of polling
 

package/bin/myelin.js

@@ -11,7 +11,6 @@
 import { readdirSync, statSync, createReadStream, readFileSync } from 'node:fs'
 import { resolve, join, relative, sep, basename } from 'node:path'
 import process from 'node:process'
-import * as tus from 'tus-js-client'
 
 const API_URL = (process.env.MYELIN_API_URL ?? 'https://myelinbridge.com/api/v1').replace(/\/$/, '')
 const KEY = process.env.MYELIN_API_KEY
@@ -55,7 +54,7 @@ function expectOk(r, context) {
   if (r.status >= 200 && r.status < 300) return r.json
   const detail = r.json?.detail ?? `HTTP ${r.status}`
   const code = r.json?.code ?? 'error'
-  const blocked = ['delivery_locked', 'submit_blocked', 'dataset_not_active', 'api_disabled', 'bridge_paused'].includes(code)
+  const blocked = ['delivery_locked', 'submit_blocked', 'dataset_not_active', 'api_disabled', 'bridge_paused', 'sample_depth_locked'].includes(code)
   die(`${context}: [${code}] ${detail}`, blocked ? 2 : 1)
 }
 
@@ -97,24 +96,71 @@ function fmtBytes(n) {
   return n + ' B'
 }
 
-function tusUpload(absPath, size, storagePath, upload) {
-  return new Promise((resolveP, rejectP) => {
-    new tus.Upload(createReadStream(absPath), {
-      endpoint: upload.endpoint,
-      chunkSize: upload.chunk_size_bytes,
-      uploadSize: size,
-      retryDelays: [0, 1000, 5000],
-      headers: { authorization: `Bearer ${upload.token}`, apikey: upload.apikey, ...upload.headers },
-      metadata: {
-        bucketName: upload.bucket,
-        objectName: storagePath,
-        contentType: 'application/octet-stream',
-        cacheControl: '3600',
-      },
-      onError: rejectP,
-      onSuccess: () => resolveP(),
-    }).start()
+// Upload a file via S3 presigned multipart (SEC-TOKEN-01: no bearer token ever
+// reaches the client — declare tells us the part size, and each part gets a
+// short-lived presigned PUT URL scoped to exactly that one part of one object).
+// Resumable: the sign step reports parts that already landed, so a re-run skips
+// them. Returns { upload_id, parts: [{part_number, etag}] } for confirm.
+const SIGN_WINDOW = 1000 // ≤ the server's max_parts_per_sign
+
+async function readPart(absPath, start, end) {
+  const chunks = []
+  await new Promise((res, rej) => {
+    const s = createReadStream(absPath, { start, end }) // end inclusive
+    s.on('data', (c) => chunks.push(c))
+    s.on('end', res)
+    s.on('error', rej)
   })
+  return Buffer.concat(chunks)
+}
+
+async function uploadFileMultipart(absPath, size, fileId, partSize, onPartDone) {
+  const numParts = Math.max(1, Math.ceil(size / partSize))
+  const allParts = Array.from({ length: numParts }, (_, i) => i + 1)
+  const etags = new Map() // part_number -> etag
+  let uploadId = null
+
+  for (let i = 0; i < allParts.length; i += SIGN_WINDOW) {
+    const window = allParts.slice(i, i + SIGN_WINDOW)
+    const j = expectOk(await api('POST', `/files/${fileId}/upload-parts`, { part_numbers: window }), 'sign upload parts')
+    uploadId = j.upload_id
+    for (const p of j.uploaded_parts) etags.set(p.part_number, p.etag) // already landed → skip
+    const urlByPart = new Map(j.urls.map((u) => [u.part_number, u.url]))
+
+    // upload the still-missing parts in this window, up to 4 concurrent.
+    // Presigned URLs expire (~30 min); on a rejection we re-sign that one part
+    // and retry once — cheap, since upload-parts is idempotent.
+    const missing = window.filter((n) => !etags.has(n))
+    const queue = [...missing]
+    const resign = async (partNo) => {
+      const j = expectOk(await api('POST', `/files/${fileId}/upload-parts`, { part_numbers: [partNo] }), 're-sign upload part')
+      const fresh = j.urls.find((u) => u.part_number === partNo)?.url
+      if (!fresh) die(`upload part ${partNo}: could not re-sign (already finalized?)`)
+      urlByPart.set(partNo, fresh)
+      return fresh
+    }
+    const workers = Array.from({ length: Math.min(4, queue.length) }, async () => {
+      for (;;) {
+        const partNo = queue.shift()
+        if (partNo === undefined) return
+        const start = (partNo - 1) * partSize
+        const end = Math.min(start + partSize, size) - 1
+        const bytes = await readPart(absPath, start, end)
+        let put = await fetch(urlByPart.get(partNo), { method: 'PUT', body: bytes }).catch(() => null)
+        if (!put || put.status === 403 || put.status === 401) {
+          put = await fetch(await resign(partNo), { method: 'PUT', body: bytes }).catch((e) => die(`upload part ${partNo}: ${e.message}`))
+        }
+        if (put.status !== 200) die(`upload part ${partNo}: HTTP ${put.status}`)
+        const etag = put.headers.get('etag')
+        if (!etag) die(`upload part ${partNo}: storage returned no ETag`)
+        etags.set(partNo, etag)
+        if (onPartDone) onPartDone()
+      }
+    })
+    await Promise.all(workers)
+  }
+
+  return { upload_id: uploadId, parts: allParts.map((n) => ({ part_number: n, etag: etags.get(n) })) }
 }
 
 // ——— commands ———
@@ -161,6 +207,60 @@ async function cmdDatasets() {
   }
 }
 
+// The delivery contract, before you deliver anything. Answers "what is expected
+// of me?" — which used to be discoverable only by submitting and being rejected.
+async function cmdContract() {
+  const dsRef = opt('dataset')
+  if (!dsRef) die('Usage: myelin contract --dataset <slug|id>')
+  const dataset = await resolveDataset(dsRef)
+  const j = expectOk(
+    await api('GET', `/datasets/${dataset.id}/quality-checks`),
+    'quality-checks',
+  )
+  emit(j)
+  if (JSON_MODE) return
+
+  if (!j.version) {
+    out(`${dataset.name}: no quality contract published yet — nothing is enforced.`)
+    return
+  }
+
+  const s = j.summary
+  out(`${dataset.name} — delivery contract v${j.version}`)
+  out(
+    `${s.total} checks · ${s.blocking} block validation · ` +
+      `${s.checkable_before_upload} checkable before upload` +
+      (s.needs_reviewer ? ` · ${s.needs_reviewer} reviewed by a person` : ''),
+  )
+  out('')
+
+  // Grouped by dimension so the contract reads as questions, not a flat list.
+  const byDim = new Map()
+  for (const c of j.checks) {
+    const key = c.dimension ?? 'review'
+    if (!byDim.has(key)) byDim.set(key, [])
+    byDim.get(key).push(c)
+  }
+  for (const d of s.dimensions) {
+    const items = byDim.get(d.key) ?? []
+    if (items.length === 0) continue
+    out(`${d.label.toUpperCase()} — ${d.question}`)
+    for (const c of items) {
+      const gate = c.severity === 'blocking' ? 'must' : 'should'
+      const when = c.runs_at === 'preflight' ? '' : ' (checked at submission)'
+      out(`  ${gate === 'must' ? '!' : '·'} ${c.assertion ?? c.name}${when}`)
+    }
+    out('')
+  }
+  const manual = byDim.get('review') ?? []
+  if (manual.length > 0) {
+    out('REVIEWER JUDGEMENT — decided by a person, not the engine')
+    for (const c of manual) out(`  · ${c.name}`)
+    out('')
+  }
+  out(`Run "myelin check <dir> --dataset ${dsRef}" to test ${s.checkable_before_upload} of these locally.`)
+}
+
 async function cmdCheck() {
   const dir = args[1]
   const dsRef = opt('dataset')
@@ -226,20 +326,22 @@ async function cmdPush() {
     declared.push(...r.json.files)
     upload = r.json.upload
   }
+  const partSize = upload?.part_size ?? 64 * 1024 * 1024
 
-  // 3. upload the delta (anything not already 'uploaded'), 4 in parallel
+  // 3. upload the delta (anything not already 'uploaded') via S3 multipart,
+  //    2 files in parallel (each file uploads its parts 4-wide internally)
   const byPath = new Map(files.map((f) => [f.path, f]))
   const pending = declared.filter((d) => d.upload_state !== 'uploaded')
   out(`Uploading ${pending.length}/${declared.length} files (${fmtBytes(pending.reduce((s, d) => s + (byPath.get(d.path)?.size ?? 0), 0))}) · resumable`)
   let done = 0
   const queue = [...pending]
-  const workers = Array.from({ length: Math.min(4, queue.length) }, async () => {
+  const workers = Array.from({ length: Math.min(2, queue.length) }, async () => {
     for (;;) {
       const d = queue.shift()
       if (!d) return
       const local = byPath.get(d.path)
-      await tusUpload(local.abs, local.size, d.storage_path, upload)
-      const c = await api('POST', `/files/${d.file_id}/confirm`)
+      const { upload_id, parts } = await uploadFileMultipart(local.abs, local.size, d.file_id, partSize)
+      const c = await api('POST', `/files/${d.file_id}/confirm`, { upload_id, parts })
       if (c.status !== 200) die(`confirm ${d.path}: [${c.json?.code}] ${c.json?.detail ?? c.status}`)
       done++
       out(`  ✓ ${d.path} (${done}/${pending.length})`)
@@ -324,6 +426,31 @@ async function cmdSandbox() {
   out(`✓ Sandbox upload succeeded — the partner-side test transfer checklist item is satisfied for bridge "${me.bridge.name}".`)
 }
 
+async function cmdSampleDepth() {
+  const dsRef = opt('dataset')
+  const raw = args[1]
+  if (!dsRef || raw === undefined) {
+    die('Usage: myelin sample-depth <0-5> --dataset <slug|id>')
+  }
+  const depth = Number(raw)
+  if (!Number.isInteger(depth) || depth < 0 || depth > 5) {
+    die('sample-depth must be an integer between 0 and 5')
+  }
+  const dataset = await resolveDataset(dsRef)
+  const j = expectOk(
+    await api('POST', `/datasets/${dataset.id}/sample-depth`, { sample_depth: depth }),
+    'sample-depth',
+  )
+  emit(j)
+  if (!JSON_MODE) {
+    out(
+      j.changed
+        ? `Sample depth for ${dataset.name} set to ${j.sample_depth}.`
+        : `Sample depth for ${dataset.name} already ${j.sample_depth} — nothing to change.`,
+    )
+  }
+}
+
 function cmdHelp() {
   console.log(`myelin — Partner Ingestion CLI
 
@@ -335,6 +462,8 @@ Commands:
   ping                                    verify the key; show bridge + projects
   projects                                list scoped projects
   datasets                                list datasets with "your move" hints
+  sample-depth <0-5> --dataset <slug|id>  declare the folder depth a sample sits at (locks after 1st submit)
+  contract --dataset <slug|id>            what this dataset expects of your delivery
   check <dir> --dataset <slug|id>         preflight local files against quality rules (no upload)
   push <dir> --dataset <slug|id>          create/resume a delivery and upload (resumable; re-run to resume)
        [--submit] [--replace]
@@ -355,6 +484,8 @@ const commands = {
   ping: cmdPing,
   projects: cmdProjects,
   datasets: cmdDatasets,
+  'sample-depth': cmdSampleDepth,
+  contract: cmdContract,
   check: cmdCheck,
   push: cmdPush,
   status: cmdStatus,

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@myelinbridge/cli",
-  "version": "0.1.0",
+  "version": "0.4.0",
   "description": "Myelin Partner Ingestion CLI — push R&D data deliveries from a pipeline: preflight against the client's quality rules, resumable upload, submit, track review outcomes.",
   "type": "module",
   "bin": {
-    "myelin": "./bin/myelin.js"
+    "myelin": "bin/myelin.js"
   },
   "files": [
     "bin/",
@@ -13,14 +13,11 @@
   "engines": {
     "node": ">=20"
   },
-  "dependencies": {
-    "tus-js-client": "^4.1.0"
-  },
   "keywords": [
     "myelin",
     "pharma",
     "ingestion",
-    "tus"
+    "s3"
   ],
   "license": "UNLICENSED"
 }