@myelinbridge/cli 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -4
- package/bin/myelin.js +288 -61
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -22,15 +22,27 @@ npx @myelinbridge/cli push ./run_042 --dataset onco1-wes --submit
|
|
|
22
22
|
sample sits — `0` = the delivery root is one sample, `1` (default) = each top-level
|
|
23
23
|
folder is a sample, `2` = one level deeper. Sample-scoped quality checks group by it,
|
|
24
24
|
so getting it right up front is what makes per-sample verdicts mean anything. It is
|
|
25
|
-
**idempotent** (safe to assert on every pipeline run
|
|
26
|
-
batch leaves draft**, so that verdicts stay comparable across
|
|
27
|
-
|
|
25
|
+
**idempotent** (safe to assert on every pipeline run, before and after the lock) and
|
|
26
|
+
**locks once your first batch leaves draft**, so that verdicts stay comparable across
|
|
27
|
+
deliveries — after that, re-asserting the current value still succeeds and only a
|
|
28
|
+
*change* exits `2`. This is the only dataset field you can write.
|
|
28
29
|
|
|
29
30
|
- **Resume = re-run.** `push` is idempotent: already-uploaded files are skipped
|
|
30
31
|
(path + size), and within a large file, parts that already landed are skipped
|
|
31
32
|
too (S3 multipart). Uploads go direct to storage over short-lived presigned
|
|
32
33
|
URLs — no credential is stored on your machine, and revoking the API key cuts
|
|
33
34
|
off signing immediately.
|
|
35
|
+
- **Transient failures retry themselves.** Since 0.8.0 a rate limit (`429`)
|
|
36
|
+
waits out `Retry-After` and retries on every command, and a part upload that
|
|
37
|
+
hits a storage hiccup (5xx, an edge timeout, an expired URL) re-signs, backs
|
|
38
|
+
off, and retries up to 4 attempts before surfacing — an unattended pipeline
|
|
39
|
+
run survives a blip. A retried `--submit` whose first attempt actually landed
|
|
40
|
+
gets a success echo (`already_submitted: true`), not a false failure.
|
|
41
|
+
- **Slow connections shrink their parts.** Since 0.9.0, when the same part dies
|
|
42
|
+
twice on a timeout (the edge kills any PUT that runs too long — HTTP 524),
|
|
43
|
+
`push` restarts the file with smaller parts, 64 → 16 → 5 MB, instead of
|
|
44
|
+
retrying the same slice into the same wall. A hospital-grade uplink down to
|
|
45
|
+
roughly 30 KB/s can now complete a delivery; it is slower, but it finishes.
|
|
34
46
|
- **Every file is fingerprinted.** Since 0.7.0, `push` computes an MD5 of each
|
|
35
47
|
file while it uploads and records it with the delivery. If the client's
|
|
36
48
|
quality contract includes a checksum-manifest check, your delivery verifies
|
|
@@ -65,7 +77,7 @@ partner's data, scope it to that partner's projects.
|
|
|
65
77
|
```bash
|
|
66
78
|
export MYELIN_API_KEY=myl_live_…
|
|
67
79
|
|
|
68
|
-
# What has landed, newest first
|
|
80
|
+
# What has landed, newest first (page with --cursor from the printed next_cursor)
|
|
69
81
|
myelin deliveries --dataset <dataset-id>
|
|
70
82
|
|
|
71
83
|
# What is this object, exactly?
|
|
@@ -94,6 +106,12 @@ A partner key calling these gets `403 wrong_key_side`, and vice versa.
|
|
|
94
106
|
Every command takes `--json`. Exit codes: `0` ok · `1` error · `2` blocked
|
|
95
107
|
(blocking preflight failure, locked delivery, blocked submit, locked sample depth).
|
|
96
108
|
|
|
109
|
+
API errors carry structure, not just prose: in `--json` mode an error is
|
|
110
|
+
`{ "error", "code", "status", "request_id" }` — branch on `code`, and quote the
|
|
111
|
+
`request_id` when reporting a problem (it is printed in human mode too; it lets
|
|
112
|
+
Myelin find the exact server-side log line). `myelin version` prints the CLI
|
|
113
|
+
version.
|
|
114
|
+
|
|
97
115
|
## Environment
|
|
98
116
|
|
|
99
117
|
| Variable | |
|
package/bin/myelin.js
CHANGED
|
@@ -15,6 +15,7 @@ import process from 'node:process'
|
|
|
15
15
|
|
|
16
16
|
const API_URL = (process.env.MYELIN_API_URL ?? 'https://myelinbridge.com/api/v1').replace(/\/$/, '')
|
|
17
17
|
const KEY = process.env.MYELIN_API_KEY
|
|
18
|
+
const VERSION = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version
|
|
18
19
|
|
|
19
20
|
// Extra headers sent with every API call — newline-separated "Name: value"
|
|
20
21
|
// pairs in MYELIN_API_HEADER. For deployments fronted by something that
|
|
@@ -49,13 +50,19 @@ class CliExit extends Error {}
|
|
|
49
50
|
// assertion on Windows, which printed a C stack trace after a perfectly good
|
|
50
51
|
// error message and replaced the documented exit code with 127. Node exits on
|
|
51
52
|
// its own once the request settles — measured at ~130 ms, no keep-alive stall.
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
53
|
+
// `extra` fields ride into the --json error object as structured data.
|
|
54
|
+
const die = (message, code = 1, extra = null) => {
|
|
55
|
+
if (JSON_MODE) console.log(JSON.stringify({ error: message, ...(extra ?? {}) }))
|
|
56
|
+
else {
|
|
57
|
+
console.error(`✗ ${message}`)
|
|
58
|
+
if (extra?.request_id) console.error(` request id ${extra.request_id} — include it if you report this`)
|
|
59
|
+
}
|
|
55
60
|
process.exitCode = code
|
|
56
61
|
throw new CliExit(message)
|
|
57
62
|
}
|
|
58
63
|
|
|
64
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
|
|
65
|
+
|
|
59
66
|
// Column widths come from the content. padEnd() alone silently ran a long
|
|
60
67
|
// value into the next column — a 28-character dataset slug swallowed the
|
|
61
68
|
// STATUS header's gutter. Pass headers = null for an unheadered list.
|
|
@@ -80,27 +87,61 @@ function opt(name) {
|
|
|
80
87
|
return i >= 0 && args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : null
|
|
81
88
|
}
|
|
82
89
|
|
|
90
|
+
// 429s are handled HERE so every command inherits the platform's documented
|
|
91
|
+
// contract (honour Retry-After): bounded — 4 waits, then the error surfaces
|
|
92
|
+
// like any other. The wait is capped at 60 s so a pathological header cannot
|
|
93
|
+
// stall a pipeline for an hour.
|
|
83
94
|
async function api(method, path, body, raw) {
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
95
|
+
for (let attempt = 1; ; attempt++) {
|
|
96
|
+
const res = await fetch(API_URL + path, {
|
|
97
|
+
method,
|
|
98
|
+
headers: {
|
|
99
|
+
...EXTRA_HEADERS,
|
|
100
|
+
Authorization: `Bearer ${KEY}`,
|
|
101
|
+
...(body && !raw ? { 'Content-Type': 'application/json' } : {}),
|
|
102
|
+
},
|
|
103
|
+
body: raw ? body : body ? JSON.stringify(body) : undefined,
|
|
104
|
+
}).catch((err) => die(`Cannot reach ${API_URL}: ${err.message}`))
|
|
105
|
+
let json = null
|
|
106
|
+
try { json = await res.json() } catch { /* non-JSON body */ }
|
|
107
|
+
const requestId = res.headers.get('myelin-request-id') ?? json?.request_id ?? null
|
|
108
|
+
if (res.status === 429 && attempt <= 4) {
|
|
109
|
+
const wait = Math.min(Number(res.headers.get('retry-after') ?? json?.retry_after) || 5, 60)
|
|
110
|
+
out(` rate limited — retrying in ${wait}s (${attempt}/4)`)
|
|
111
|
+
await sleep(wait * 1000 + Math.random() * 500)
|
|
112
|
+
continue
|
|
113
|
+
}
|
|
114
|
+
return { status: res.status, json, requestId }
|
|
115
|
+
}
|
|
96
116
|
}
|
|
97
117
|
|
|
98
|
-
|
|
99
|
-
|
|
118
|
+
const BLOCKED_CODES = ['delivery_locked', 'submit_blocked', 'dataset_not_active', 'api_disabled', 'bridge_paused', 'sample_depth_locked']
|
|
119
|
+
|
|
120
|
+
// Every API failure surfaces the stable `code` AND the request id the server
|
|
121
|
+
// echoed — the id is what lets Myelin find the exact server-side log line for
|
|
122
|
+
// a support report. In --json mode both are structured fields, never prose a
|
|
123
|
+
// script has to regex out of the message.
|
|
124
|
+
function apiDie(context, r) {
|
|
100
125
|
const detail = r.json?.detail ?? `HTTP ${r.status}`
|
|
101
126
|
const code = r.json?.code ?? 'error'
|
|
102
|
-
|
|
103
|
-
|
|
127
|
+
die(`${context}: [${code}] ${detail}`, BLOCKED_CODES.includes(code) ? 2 : 1, {
|
|
128
|
+
code,
|
|
129
|
+
status: r.status,
|
|
130
|
+
...(r.requestId ? { request_id: r.requestId } : {}),
|
|
131
|
+
})
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function expectOk(r, context) {
|
|
135
|
+
if (r.status >= 200 && r.status < 300) {
|
|
136
|
+
// A 2xx that isn't JSON is almost always a front door answering instead
|
|
137
|
+
// of Myelin (a gateway login page, an SSO-protected preview whose pass
|
|
138
|
+
// lapsed). Name the likely cause instead of crashing on j.<field>.
|
|
139
|
+
if (r.json == null) {
|
|
140
|
+
die(`${context}: the server answered ${r.status} but not with JSON — if this deployment sits behind a gateway or front door, your pass (MYELIN_API_HEADER) may have lapsed. Re-mint it and retry.`)
|
|
141
|
+
}
|
|
142
|
+
return r.json
|
|
143
|
+
}
|
|
144
|
+
apiDie(context, r)
|
|
104
145
|
}
|
|
105
146
|
|
|
106
147
|
// ——— dataset resolution (id or slug) ———
|
|
@@ -123,9 +164,15 @@ async function resolveDataset(ref) {
|
|
|
123
164
|
function walkDir(root) {
|
|
124
165
|
const files = []
|
|
125
166
|
const walk = (dir) => {
|
|
126
|
-
|
|
167
|
+
let entries
|
|
168
|
+
try { entries = readdirSync(dir) } catch (err) { die(`Cannot read directory ${dir}: ${err.message}`) }
|
|
169
|
+
for (const entry of entries) {
|
|
127
170
|
const full = join(dir, entry)
|
|
128
|
-
|
|
171
|
+
// A broken symlink or unreadable file gets a named error, not a stack —
|
|
172
|
+
// everything under the directory ships, so an unreadable entry is the
|
|
173
|
+
// partner's to fix or remove, and the message should say which one.
|
|
174
|
+
let st
|
|
175
|
+
try { st = statSync(full) } catch (err) { die(`Cannot read ${full}: ${err.message} — fix or remove it (every file under the directory is included)`) }
|
|
129
176
|
if (st.isDirectory()) walk(full)
|
|
130
177
|
else files.push({ abs: full, path: '/' + relative(root, full).split(sep).join('/'), size: st.size })
|
|
131
178
|
}
|
|
@@ -148,6 +195,43 @@ function fmtBytes(n) {
|
|
|
148
195
|
// them. Returns { upload_id, parts: [{part_number, etag}] } for confirm.
|
|
149
196
|
const SIGN_WINDOW = 1000 // ≤ the server's max_parts_per_sign
|
|
150
197
|
|
|
198
|
+
// Adaptive part geometry (mirrors lib/uploads/part-geometry.ts — this file
|
|
199
|
+
// ships dependency-free and cannot import it). The edge in front of the S3
|
|
200
|
+
// endpoint kills a PUT that outlives its patience (observed: HTTP 524 at
|
|
201
|
+
// ~175 s), so a fixed 64 MiB part gives a slow uplink (under ~375 KiB/s) no
|
|
202
|
+
// way to ever land one — every retry re-sends the same slice into the same
|
|
203
|
+
// wall. The second timeout-shaped failure on the same part restarts the file
|
|
204
|
+
// with smaller parts: 64 → 16 → 5 MiB (the S3 floor for any part but the
|
|
205
|
+
// last), which moves the workable floor to ~30 KiB/s.
|
|
206
|
+
const MIN_PART_SIZE = 5 * 1024 * 1024
|
|
207
|
+
const SLOW_FAILURE_MS = 45_000
|
|
208
|
+
const nextSmallerPartSize = (n) => Math.max(MIN_PART_SIZE, Math.floor(n / 4))
|
|
209
|
+
const isTimeoutClassFailure = (status, elapsedMs) =>
|
|
210
|
+
status === 408 || status === 504 || status === 522 || status === 524 ||
|
|
211
|
+
(status === 0 && elapsedMs >= SLOW_FAILURE_MS)
|
|
212
|
+
// A landed part is skippable on resume only when it holds the bytes the
|
|
213
|
+
// CURRENT geometry puts at that part number — a leftover from an abandoned
|
|
214
|
+
// geometry re-uploads under the same number, which replaces it. The part
|
|
215
|
+
// listing reports no sizes, but a part's ETag IS the MD5 of its bytes, so the
|
|
216
|
+
// check is: MD5 of the local slice == the landed ETag. This also catches a
|
|
217
|
+
// file edited between runs, which skip-by-part-number silently stitched into
|
|
218
|
+
// a mixed object.
|
|
219
|
+
const etagMd5 = (etag) => {
|
|
220
|
+
const m = /^(?:W\/)?"?([0-9a-fA-F]{32})"?$/.exec(etag ?? '')
|
|
221
|
+
return m ? m[1].toLowerCase() : null
|
|
222
|
+
}
|
|
223
|
+
const md5Hex = (buf) => createHash('md5').update(buf).digest('hex')
|
|
224
|
+
// Geometries a previous run may have used, most likely first: the requested
|
|
225
|
+
// size, then the downshift ladder from the 64 MiB default.
|
|
226
|
+
const partSizeCandidates = (requested) => {
|
|
227
|
+
const ladder = [requested]
|
|
228
|
+
for (let s = 64 * 1024 * 1024; ; s = nextSmallerPartSize(s)) {
|
|
229
|
+
if (!ladder.includes(s)) ladder.push(s)
|
|
230
|
+
if (s <= MIN_PART_SIZE) break
|
|
231
|
+
}
|
|
232
|
+
return ladder
|
|
233
|
+
}
|
|
234
|
+
|
|
151
235
|
async function readPart(absPath, start, end) {
|
|
152
236
|
const chunks = []
|
|
153
237
|
await new Promise((res, rej) => {
|
|
@@ -181,52 +265,156 @@ async function md5File(absPath) {
|
|
|
181
265
|
}
|
|
182
266
|
|
|
183
267
|
async function uploadFileMultipart(absPath, size, fileId, partSize, onPartDone) {
|
|
184
|
-
|
|
185
|
-
|
|
268
|
+
let adoptLandedGeometry = true
|
|
269
|
+
for (;;) {
|
|
270
|
+
const attempt = await uploadFileParts(absPath, size, fileId, partSize, adoptLandedGeometry, onPartDone)
|
|
271
|
+
if (!attempt.tooSlow) return attempt.result
|
|
272
|
+
const smaller = nextSmallerPartSize(attempt.partSize)
|
|
273
|
+
out(` ${fmtBytes(attempt.partSize)} parts keep timing out on this connection — restarting with ${fmtBytes(smaller)} parts`)
|
|
274
|
+
partSize = smaller
|
|
275
|
+
// The restart deliberately abandons the old geometry — re-adopting part
|
|
276
|
+
// 1's old size from the landed listing would undo it.
|
|
277
|
+
adoptLandedGeometry = false
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
async function uploadFileParts(absPath, size, fileId, partSize, adoptLandedGeometry, onPartDone) {
|
|
282
|
+
let numParts = Math.max(1, Math.ceil(size / partSize))
|
|
186
283
|
const etags = new Map() // part_number -> etag
|
|
284
|
+
const timeoutStrikes = new Map() // part_number -> timeout-shaped failures
|
|
187
285
|
let uploadId = null
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
const
|
|
286
|
+
let tooSlow = false
|
|
287
|
+
|
|
288
|
+
for (let i = 0; i < numParts; i += SIGN_WINDOW) {
|
|
289
|
+
const windowParts = () =>
|
|
290
|
+
Array.from({ length: Math.min(SIGN_WINDOW, numParts - i) }, (_, k) => i + k + 1)
|
|
291
|
+
let j = expectOk(await api('POST', `/files/${fileId}/upload-parts`, { part_numbers: windowParts() }), 'sign upload parts')
|
|
292
|
+
if (i === 0 && adoptLandedGeometry && size > 0) {
|
|
293
|
+
// Resume across runs: an earlier run may have downshifted on a slow
|
|
294
|
+
// link, so the landed parts can belong to a smaller geometry. The
|
|
295
|
+
// ladder geometry whose local part-1 slice hashes to the landed part-1
|
|
296
|
+
// ETag is the geometry that run used — adopt it, or every landed part
|
|
297
|
+
// fails the content check and re-uploads for nothing.
|
|
298
|
+
const landedFirst = j.uploaded_parts.find((p) => p.part_number === 1)
|
|
299
|
+
const landedFirstMd5 = landedFirst ? etagMd5(landedFirst.etag) : null
|
|
300
|
+
if (landedFirstMd5) {
|
|
301
|
+
for (const candidate of partSizeCandidates(partSize)) {
|
|
302
|
+
const probe = await readPart(absPath, 0, Math.min(candidate, size) - 1)
|
|
303
|
+
if (md5Hex(probe) !== landedFirstMd5) continue
|
|
304
|
+
if (candidate !== partSize) {
|
|
305
|
+
partSize = candidate
|
|
306
|
+
numParts = Math.max(1, Math.ceil(size / partSize))
|
|
307
|
+
j = expectOk(await api('POST', `/files/${fileId}/upload-parts`, { part_numbers: windowParts() }), 'sign upload parts')
|
|
308
|
+
}
|
|
309
|
+
break
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
}
|
|
192
313
|
uploadId = j.upload_id
|
|
193
|
-
|
|
314
|
+
const landedByPart = new Map() // part_number -> { etag, md5 } (content-checkable landed parts)
|
|
315
|
+
for (const p of j.uploaded_parts) {
|
|
316
|
+
const md5 = etagMd5(p.etag)
|
|
317
|
+
if (md5) landedByPart.set(p.part_number, { etag: p.etag, md5 })
|
|
318
|
+
// An ETag that is not MD5-shaped cannot be content-checked; trust it by
|
|
319
|
+
// number (the pre-adaptive behavior) unless this attempt is a geometry
|
|
320
|
+
// restart, where the listing is known stale and the part re-uploads.
|
|
321
|
+
else if (adoptLandedGeometry) etags.set(p.part_number, p.etag)
|
|
322
|
+
}
|
|
194
323
|
const urlByPart = new Map(j.urls.map((u) => [u.part_number, u.url]))
|
|
195
324
|
|
|
196
325
|
// upload the still-missing parts in this window, up to 4 concurrent.
|
|
197
|
-
|
|
198
|
-
// and retry once — cheap, since upload-parts is idempotent.
|
|
199
|
-
const missing = window.filter((n) => !etags.has(n))
|
|
326
|
+
const missing = windowParts().filter((n) => !etags.has(n))
|
|
200
327
|
const queue = [...missing]
|
|
201
|
-
|
|
328
|
+
// Re-sign one part. Returns a fresh URL — or null when the server reports
|
|
329
|
+
// the part ALREADY LANDED with the bytes we are sending: a 524/timeout
|
|
330
|
+
// can kill the response after S3 committed the bytes. That case is a
|
|
331
|
+
// success to record, not a dead end to die on.
|
|
332
|
+
const resign = async (partNo, bytes) => {
|
|
202
333
|
const j = expectOk(await api('POST', `/files/${fileId}/upload-parts`, { part_numbers: [partNo] }), 're-sign upload part')
|
|
334
|
+
uploadId = j.upload_id
|
|
335
|
+
const landed = j.uploaded_parts.find((p) => p.part_number === partNo)
|
|
336
|
+
if (landed) {
|
|
337
|
+
const md5 = etagMd5(landed.etag)
|
|
338
|
+
// Trust an unverifiable ETag here: this is our own attempt's commit.
|
|
339
|
+
if (!md5 || md5 === md5Hex(bytes)) { etags.set(partNo, landed.etag); return null }
|
|
340
|
+
}
|
|
203
341
|
const fresh = j.urls.find((u) => u.part_number === partNo)?.url
|
|
204
|
-
if (!fresh)
|
|
342
|
+
if (!fresh) {
|
|
343
|
+
if (landed) die(`upload part ${partNo}: landed under a different part geometry and this server will not re-sign a landed part — delete the file from the draft and push again`)
|
|
344
|
+
die(`upload part ${partNo}: could not re-sign (already finalized?)`)
|
|
345
|
+
}
|
|
205
346
|
urlByPart.set(partNo, fresh)
|
|
206
347
|
return fresh
|
|
207
348
|
}
|
|
208
349
|
const workers = Array.from({ length: Math.min(4, queue.length) }, async () => {
|
|
209
350
|
for (;;) {
|
|
351
|
+
if (tooSlow) return
|
|
210
352
|
const partNo = queue.shift()
|
|
211
353
|
if (partNo === undefined) return
|
|
212
354
|
const start = (partNo - 1) * partSize
|
|
213
355
|
const end = Math.min(start + partSize, size) - 1
|
|
214
356
|
const bytes = await readPart(absPath, start, end)
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
357
|
+
// Resume: a landed part holding exactly these bytes is recorded, not
|
|
358
|
+
// re-sent. A mismatch (abandoned geometry, or the file changed on
|
|
359
|
+
// disk since the last run) falls through and re-uploads the part
|
|
360
|
+
// number, which replaces it.
|
|
361
|
+
const landed = landedByPart.get(partNo)
|
|
362
|
+
if (landed && md5Hex(bytes) === landed.md5) {
|
|
363
|
+
etags.set(partNo, landed.etag)
|
|
364
|
+
if (onPartDone) onPartDone()
|
|
365
|
+
continue
|
|
366
|
+
}
|
|
367
|
+
// Expired URLs (401/403), transient storage errors (5xx), edge kills
|
|
368
|
+
// (524) and throttling all deserve the same treatment: re-sign — always
|
|
369
|
+
// safe, upload-parts is idempotent — back off, try again. Bounded at 4
|
|
370
|
+
// attempts, then the error surfaces. An unattended pipeline should
|
|
371
|
+
// survive a hiccup, not page a human for it; `push` stays resumable
|
|
372
|
+
// either way. EXCEPT timeout-shaped failures: the second one on the
|
|
373
|
+
// same part flags a geometry restart — retrying the same slice into
|
|
374
|
+
// the same timeout gets nowhere on a slow uplink.
|
|
375
|
+
let put = null
|
|
376
|
+
for (let attempt = 1; ; attempt++) {
|
|
377
|
+
if (tooSlow) return
|
|
378
|
+
let url = attempt === 1 ? urlByPart.get(partNo) : undefined
|
|
379
|
+
if (url === undefined) url = await resign(partNo, bytes)
|
|
380
|
+
if (url === null) break // part landed server-side — resign() recorded its etag
|
|
381
|
+
const startedAt = Date.now()
|
|
382
|
+
put = await fetch(url, { method: 'PUT', body: bytes }).catch(() => null)
|
|
383
|
+
if (put && put.status === 200) break
|
|
384
|
+
const status = put ? put.status : 0
|
|
385
|
+
const reason = put ? `HTTP ${put.status}` : 'network error'
|
|
386
|
+
if (isTimeoutClassFailure(status, Date.now() - startedAt)) {
|
|
387
|
+
const strikes = (timeoutStrikes.get(partNo) ?? 0) + 1
|
|
388
|
+
timeoutStrikes.set(partNo, strikes)
|
|
389
|
+
if (strikes >= 2 && partSize > MIN_PART_SIZE) {
|
|
390
|
+
out(` part ${partNo}: ${reason} again — parts too large for this connection`)
|
|
391
|
+
tooSlow = true
|
|
392
|
+
return
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
if (attempt >= 4) die(`upload part ${partNo}: ${reason} after ${attempt} attempts — re-run to resume`)
|
|
396
|
+
const wait = [1, 4, 10][attempt - 1] + Math.random()
|
|
397
|
+
out(` part ${partNo}: ${reason} — retrying in ${Math.round(wait)}s (${attempt}/3)`)
|
|
398
|
+
await sleep(wait * 1000)
|
|
399
|
+
}
|
|
400
|
+
if (put && put.status === 200) {
|
|
401
|
+
const etag = put.headers.get('etag')
|
|
402
|
+
if (!etag) die(`upload part ${partNo}: storage returned no ETag`)
|
|
403
|
+
etags.set(partNo, etag)
|
|
218
404
|
}
|
|
219
|
-
if (put.status !== 200) die(`upload part ${partNo}: HTTP ${put.status}`)
|
|
220
|
-
const etag = put.headers.get('etag')
|
|
221
|
-
if (!etag) die(`upload part ${partNo}: storage returned no ETag`)
|
|
222
|
-
etags.set(partNo, etag)
|
|
223
405
|
if (onPartDone) onPartDone()
|
|
224
406
|
}
|
|
225
407
|
})
|
|
226
408
|
await Promise.all(workers)
|
|
409
|
+
if (tooSlow) return { tooSlow: true, partSize }
|
|
227
410
|
}
|
|
228
411
|
|
|
229
|
-
|
|
412
|
+
const allParts = Array.from({ length: numParts }, (_, n) => n + 1)
|
|
413
|
+
return {
|
|
414
|
+
tooSlow: false,
|
|
415
|
+
partSize,
|
|
416
|
+
result: { upload_id: uploadId, parts: allParts.map((n) => ({ part_number: n, etag: etags.get(n) })) },
|
|
417
|
+
}
|
|
230
418
|
}
|
|
231
419
|
|
|
232
420
|
// ——— commands ———
|
|
@@ -374,8 +562,27 @@ async function cmdPush() {
|
|
|
374
562
|
if (files.length === 0) die(`No files under ${root}`)
|
|
375
563
|
const dataset = await resolveDataset(dsRef)
|
|
376
564
|
|
|
377
|
-
// 1. create or resume the draft
|
|
378
|
-
|
|
565
|
+
// 1. create or resume the draft. A 409 delivery_locked here can BE the
|
|
566
|
+
// lost-response retry case: our previous run's --submit landed, the
|
|
567
|
+
// batch left draft, and the pipeline re-ran the whole command. Before
|
|
568
|
+
// claiming success, verify the in-flight batch actually IS this delivery
|
|
569
|
+
// (same paths, same sizes) — an unrelated delivery under review must
|
|
570
|
+
// keep the honest blocked exit.
|
|
571
|
+
const createRes = await api('POST', `/datasets/${dataset.id}/batches`)
|
|
572
|
+
if (createRes.status === 409 && createRes.json?.code === 'delivery_locked' && createRes.json?.batch_id && flag('submit')) {
|
|
573
|
+
const b = await api('GET', `/batches/${createRes.json.batch_id}`)
|
|
574
|
+
const remote = b.status === 200 ? (b.json.batch?.files ?? []) : []
|
|
575
|
+
const matches =
|
|
576
|
+
remote.length === files.length &&
|
|
577
|
+
files.every((f) => remote.some((r) => r.path === f.path && r.size_bytes === f.size))
|
|
578
|
+
if (matches) {
|
|
579
|
+
const j = expectOk(await api('POST', `/batches/${createRes.json.batch_id}/submit`), 'submit')
|
|
580
|
+
emit(j)
|
|
581
|
+
out(`✓ This delivery is already submitted (${j.status}) — nothing to redo. Track: myelin status ${createRes.json.batch_id} --watch`)
|
|
582
|
+
return
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
const created = expectOk(createRes, 'create batch')
|
|
379
586
|
const batchId = created.batch_id
|
|
380
587
|
out(`Draft ${created.resumed ? 'resumed' : 'created'} (${batchId.slice(0, 8)}…).`)
|
|
381
588
|
|
|
@@ -389,6 +596,7 @@ async function cmdPush() {
|
|
|
389
596
|
replace: flag('replace'),
|
|
390
597
|
})
|
|
391
598
|
if (r.status !== 200 && r.status !== 409) expectOk(r, 'declare')
|
|
599
|
+
if (!r.json?.files) apiDie('declare', r) // 409 with a non-JSON body (a proxy page, an edge error) must not crash
|
|
392
600
|
const conflicts = r.json.files.filter((f) => f.error)
|
|
393
601
|
if (conflicts.length > 0) {
|
|
394
602
|
for (const c of conflicts) out(`✗ ${c.path}: ${c.error}`)
|
|
@@ -424,7 +632,7 @@ async function cmdPush() {
|
|
|
424
632
|
parts,
|
|
425
633
|
...(checksum ? { checksum, checksum_algorithm: 'md5' } : {}),
|
|
426
634
|
})
|
|
427
|
-
if (c.status !== 200)
|
|
635
|
+
if (c.status !== 200) apiDie(`confirm ${d.path}`, c)
|
|
428
636
|
done++
|
|
429
637
|
out(` ✓ ${d.path} (${done}/${pending.length})`)
|
|
430
638
|
}
|
|
@@ -434,15 +642,17 @@ async function cmdPush() {
|
|
|
434
642
|
|
|
435
643
|
// 4. optional submit
|
|
436
644
|
if (flag('submit')) {
|
|
437
|
-
const
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
645
|
+
const j = expectOk(await api('POST', `/batches/${batchId}/submit`), 'submit')
|
|
646
|
+
emit(j)
|
|
647
|
+
if (j.already_submitted) {
|
|
648
|
+
// A retried submit whose first attempt actually landed: the API echoes
|
|
649
|
+
// success instead of failing, and so do we — nothing to redo.
|
|
650
|
+
out(`✓ Batch already submitted (${j.status}) — nothing to redo. Track: myelin status ${batchId} --watch`)
|
|
651
|
+
} else {
|
|
652
|
+
const a = j.auto_checks
|
|
653
|
+
out(`Submitting… auto-checks: ${a.passed} passed, ${a.flagged} flagged, ${a.failed} failed.`)
|
|
654
|
+
out(`✓ Batch submitted for review. Track: myelin status ${batchId} --watch`)
|
|
441
655
|
}
|
|
442
|
-
const a = r.json.auto_checks
|
|
443
|
-
emit(r.json)
|
|
444
|
-
out(`Submitting… auto-checks: ${a.passed} passed, ${a.flagged} flagged, ${a.failed} failed.`)
|
|
445
|
-
out(`✓ Batch submitted for review. Track: myelin status ${batchId} --watch`)
|
|
446
656
|
} else {
|
|
447
657
|
emit({ batch_id: batchId, files: declared.length, submitted: false })
|
|
448
658
|
out(`Draft ready (not submitted). Submit with: myelin push ${dir} --dataset ${dsRef} --submit`)
|
|
@@ -456,7 +666,7 @@ async function cmdStatus() {
|
|
|
456
666
|
const findBatch = async () => {
|
|
457
667
|
if (/^[0-9a-f]{8}-[0-9a-f]{4}-/i.test(ref)) {
|
|
458
668
|
const r = await api('GET', `/batches/${ref}`)
|
|
459
|
-
if (r.status === 200) return r.
|
|
669
|
+
if (r.status === 200) return expectOk(r, 'batch').batch
|
|
460
670
|
die(`Batch ${ref} not found in this key's scope`)
|
|
461
671
|
}
|
|
462
672
|
const list = expectOk(await api('GET', '/batches?limit=100'), 'batches')
|
|
@@ -496,7 +706,8 @@ async function cmdSandbox() {
|
|
|
496
706
|
const file = args[1]
|
|
497
707
|
if (!file) die('Usage: myelin sandbox <file>')
|
|
498
708
|
const abs = resolve(file)
|
|
499
|
-
|
|
709
|
+
let size
|
|
710
|
+
try { size = statSync(abs).size } catch (err) { die(`Cannot read ${abs}: ${err.message}`) }
|
|
500
711
|
if (size > 4 * 1024 * 1024) die('Sandbox files should stay under ~4 MB — this verifies connectivity, not throughput.')
|
|
501
712
|
|
|
502
713
|
const me = expectOk(await api('GET', '/me'), 'auth')
|
|
@@ -556,22 +767,26 @@ Commands:
|
|
|
556
767
|
|
|
557
768
|
Client-side commands (need a CLIENT key — Organisation → API keys):
|
|
558
769
|
deliveries [--project <id>] what landed in your destination bucket
|
|
559
|
-
[--dataset <id>] [--since <iso>] [--limit <n>]
|
|
770
|
+
[--dataset <id>] [--since <iso>] [--limit <n>] [--cursor <ts>]
|
|
560
771
|
resolve <path|prefix|id> what a path in your bucket actually is —
|
|
561
772
|
project, dataset, batch, file
|
|
562
773
|
|
|
563
|
-
|
|
774
|
+
version print the CLI version
|
|
775
|
+
|
|
776
|
+
Every command accepts --json. Exit codes: 0 ok · 1 error · 2 blocked.
|
|
777
|
+
Errors print the API request id — include it when reporting a problem.`)
|
|
564
778
|
}
|
|
565
779
|
|
|
566
780
|
// ——— client-side commands (client keys only) ———
|
|
567
781
|
//
|
|
568
|
-
// These need a CLIENT key, minted by
|
|
569
|
-
//
|
|
570
|
-
// so — the two sides deliberately do not silently degrade
|
|
782
|
+
// These need a CLIENT key, minted by a tenant admin in Organisation → API keys
|
|
783
|
+
// (tenant-scoped since 2026-08-08). A partner key gets 403 wrong_key_side, and
|
|
784
|
+
// the message says so — the two sides deliberately do not silently degrade
|
|
785
|
+
// into each other.
|
|
571
786
|
|
|
572
787
|
async function cmdDeliveries() {
|
|
573
788
|
const qs = new URLSearchParams()
|
|
574
|
-
const map = { project: 'project_id', dataset: 'dataset_id', since: 'since', limit: 'limit' }
|
|
789
|
+
const map = { project: 'project_id', dataset: 'dataset_id', since: 'since', limit: 'limit', cursor: 'cursor' }
|
|
575
790
|
for (const [flagName, param] of Object.entries(map)) {
|
|
576
791
|
const v = opt(flagName)
|
|
577
792
|
if (v) qs.set(param, v)
|
|
@@ -592,7 +807,15 @@ async function cmdDeliveries() {
|
|
|
592
807
|
d.destination_root ?? '—',
|
|
593
808
|
])
|
|
594
809
|
for (const line of table(['PROJECT', 'DATASET', 'BATCH', 'FILES', 'DELIVERED', 'LOCATION'], rows)) out(line)
|
|
595
|
-
if (j.next_cursor)
|
|
810
|
+
if (j.next_cursor) {
|
|
811
|
+
// Carry the invocation's own filters into the hint — a hint that drops
|
|
812
|
+
// --dataset pages the unfiltered list and interleaves other datasets.
|
|
813
|
+
const carried = ['project', 'dataset', 'since', 'limit']
|
|
814
|
+
.filter((f) => opt(f))
|
|
815
|
+
.map((f) => `--${f} ${opt(f)}`)
|
|
816
|
+
.join(' ')
|
|
817
|
+
out(`\n… more — next page: myelin deliveries ${carried ? `${carried} ` : ''}--cursor ${j.next_cursor}`)
|
|
818
|
+
}
|
|
596
819
|
}
|
|
597
820
|
|
|
598
821
|
async function cmdResolve() {
|
|
@@ -623,6 +846,10 @@ if (!command || command === 'help' || command === '--help') {
|
|
|
623
846
|
cmdHelp()
|
|
624
847
|
process.exit(0)
|
|
625
848
|
}
|
|
849
|
+
if (command === 'version' || command === '--version' || command === '-v') {
|
|
850
|
+
console.log(JSON_MODE ? JSON.stringify({ version: VERSION }) : VERSION)
|
|
851
|
+
process.exit(0)
|
|
852
|
+
}
|
|
626
853
|
|
|
627
854
|
const commands = {
|
|
628
855
|
ping: cmdPing,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@myelinbridge/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "Myelin Partner Ingestion CLI — push R&D data deliveries from a pipeline: preflight against the client's quality rules, resumable upload, submit, track review outcomes.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|