@hope_phenom/dsh-plugin-remote-use 0.0.0-stage → 0.2.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +207 -2
- package/README.zh.md +195 -0
- package/assets/icon.svg +47 -0
- package/cordis.patch.yml +41 -0
- package/lib/bridge/acp-bridge.js +905 -0
- package/lib/bridge/peer-connection.js +730 -0
- package/lib/client.js +2842 -0
- package/lib/core/acp-model-overlay.js +121 -0
- package/lib/core/approval-queue.js +216 -0
- package/lib/core/audit-log.js +322 -0
- package/lib/core/desktop-notify.js +173 -0
- package/lib/core/environment-prompt.js +76 -0
- package/lib/core/relay-supervisor.js +278 -0
- package/lib/core/review-engine.js +679 -0
- package/lib/core/sandbox-diagnosis.js +258 -0
- package/lib/core/sandbox-probe.js +310 -0
- package/lib/core/self-acp-spec.js +117 -0
- package/lib/core/selfcheck-runner.js +150 -0
- package/lib/core/settings-store.js +226 -0
- package/lib/core/shot-gc.js +334 -0
- package/lib/core/task-cwd.js +50 -0
- package/lib/core/task-memory.js +163 -0
- package/lib/core/task-registry.js +238 -0
- package/lib/core/toolcall-cache.js +206 -0
- package/lib/core/transfer-queue.js +58 -0
- package/lib/core/transfer.js +1144 -0
- package/lib/core/worker-arming.js +334 -0
- package/lib/index.js +2207 -0
- package/lib/provider/out-of-process-helpers.js +76 -0
- package/lib/provider/remote-provider.js +261 -0
- package/lib/tools/mediator-shim.mjs +180 -0
- package/lib/tools/prepare-sandbox-mediator.mjs +529 -0
- package/lib/tools/remote-tools.js +680 -0
- package/lib/tools/selfcheck-fix-prompt.md +206 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +72 -4
- package/slave/acp-profile-approval.patch.yml +66 -0
- package/slave/file-transfer.mjs +552 -0
- package/slave/integrity-probe.ps1 +99 -0
- package/slave/make-launch-spec.mjs +184 -0
- package/slave/net-address.mjs +138 -0
- package/slave/node_modules/.package-lock.json +26 -0
- package/slave/node_modules/ws/LICENSE +20 -0
- package/slave/node_modules/ws/README.md +548 -0
- package/slave/node_modules/ws/browser.js +8 -0
- package/slave/node_modules/ws/index.js +22 -0
- package/slave/node_modules/ws/lib/buffer-util.js +131 -0
- package/slave/node_modules/ws/lib/constants.js +19 -0
- package/slave/node_modules/ws/lib/event-target.js +292 -0
- package/slave/node_modules/ws/lib/extension.js +203 -0
- package/slave/node_modules/ws/lib/limiter.js +55 -0
- package/slave/node_modules/ws/lib/permessage-deflate.js +530 -0
- package/slave/node_modules/ws/lib/receiver.js +743 -0
- package/slave/node_modules/ws/lib/sender.js +607 -0
- package/slave/node_modules/ws/lib/stream.js +161 -0
- package/slave/node_modules/ws/lib/subprotocol.js +62 -0
- package/slave/node_modules/ws/lib/validation.js +152 -0
- package/slave/node_modules/ws/lib/websocket-server.js +562 -0
- package/slave/node_modules/ws/lib/websocket.js +1421 -0
- package/slave/node_modules/ws/package.json +74 -0
- package/slave/node_modules/ws/wrapper.mjs +21 -0
- package/slave/pairing.mjs +175 -0
- package/slave/permission-pairing.mjs +240 -0
- package/slave/prune-shots.mjs +170 -0
- package/slave/relay.mjs +1351 -0
- package/slave/repair-workspace-integrity.ps1 +75 -0
- package/slave/screenshot.ps1 +114 -0
- package/slave/selfcheck.ps1 +350 -0
- package/slave/stage-app.ps1 +116 -0
- package/slave/workspace-integrity.mjs +157 -0
|
@@ -0,0 +1,1144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streaming file transfer between the host and a worker's relay.
|
|
3
|
+
*
|
|
4
|
+
* WHY HTTP AND NOT THE RPC CHANNEL: the control frames that keep a pairing alive ride the relay's
|
|
5
|
+
* WebSocket, and that socket is a single `message` handler shared with the ACP conversation pump
|
|
6
|
+
* (`relay.mjs`). A multi-hundred-megabyte file pushed through it would queue every GUI control
|
|
7
|
+
* frame behind the bytes, and `ws` would first try to hold the payload in memory (its default
|
|
8
|
+
* `maxPayload` is 100 MiB, and exceeding it closes the socket with 1009). Bytes therefore go over
|
|
9
|
+
* the relay's HTTP side -- same origin, same token -- which is exactly why `screenshot()` was moved
|
|
10
|
+
* there. This module is the file-sized version of that decision.
|
|
11
|
+
*
|
|
12
|
+
* WHY THE CORE TAKES A TRANSFER CHANNEL RATHER THAN A LIVE SOCKET: everything here is driven by
|
|
13
|
+
* plain callbacks (`get` / `put` / `head` / `rpc`) instead of a bridge or a `PeerConnection`. That
|
|
14
|
+
* is not decoration: it is what lets the tests drive a full 12 MiB upload through a fake channel and
|
|
15
|
+
* ASSERT that it arrived as three 4 MiB requests. The property that matters most in this file --
|
|
16
|
+
* "a file is never buffered whole, on either side" -- cannot be observed through a live socket
|
|
17
|
+
* without pulling the very ceiling into the test that the ceiling is supposed to prevent.
|
|
18
|
+
*
|
|
19
|
+
* THE TWO INVARIANTS EVERY PATH PRESERVES:
|
|
20
|
+
*
|
|
21
|
+
* 1. NEVER A WHOLE FILE IN MEMORY. Uploads are sliced into `TRANSFER_CHUNK_BYTES`-sized ranges of
|
|
22
|
+
* the local file and each range is handed to `fetch` AS A `ReadStream` (`duplex: 'half'`), so
|
|
23
|
+
* neither end ever holds more than a socket buffer of it; the running sha256 is computed over
|
|
24
|
+
* those same slices, in the same pass, so the digest costs no extra read. Downloads go
|
|
25
|
+
* `Readable.fromWeb(res.body)` -> an open file handle. Progress is therefore per slice, which
|
|
26
|
+
* is also why it can be rendered live.
|
|
27
|
+
* 2. THE DESTINATION PATH IS NEVER HALF-WRITTEN. Both directions stage into `<name>.part`, verify
|
|
28
|
+
* the whole-file sha256, and only then rename. A crash, a full disk or a dropped link can
|
|
29
|
+
* leave a `.part` (that is the resume point and the whole point of keeping it) but never a
|
|
30
|
+
* plausible-looking file at the real name.
|
|
31
|
+
*
|
|
32
|
+
* @module dsh-plugin-remote-use/transfer
|
|
33
|
+
*/
|
|
34
|
+
import { createHash } from 'node:crypto'
|
|
35
|
+
import { createReadStream, existsSync, mkdirSync, openSync, readSync, closeSync, renameSync, rmSync, statSync, truncateSync, writeSync } from 'node:fs'
|
|
36
|
+
import { Readable } from 'node:stream'
|
|
37
|
+
import { basename, dirname, join } from 'node:path'
|
|
38
|
+
|
|
39
|
+
import { resolveDshHome } from './settings-store.js'
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Slice size for the upload loop, per the frozen contract (4 MiB).
|
|
43
|
+
*
|
|
44
|
+
* WHY NOT SMALLER: every slice is one HTTP request, and each request costs a round trip plus a
|
|
45
|
+
* receiver-side `fsync`-cheap append. A LAN round trip is sub-millisecond, but a 4 KiB slice would
|
|
46
|
+
* mean a quarter of a million requests for one 1 GiB file and a progress callback per request;
|
|
47
|
+
* 4 MiB keeps the request count in the hundreds while staying far below any proxy's patience.
|
|
48
|
+
*/
|
|
49
|
+
export const TRANSFER_CHUNK_BYTES = 4 * 1024 * 1024
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Host-side refusal ceiling, when the caller configures nothing.
|
|
53
|
+
*
|
|
54
|
+
* WHY A DEFAULT AT ALL: an unbounded transfer is a way to fill the host's disk by accident. 8 GiB
|
|
55
|
+
* matches the worker's own hard `--max-transfer-bytes` default, so the two ends agree without
|
|
56
|
+
* either being told.
|
|
57
|
+
*/
|
|
58
|
+
export const DEFAULT_TRANSFER_MAX_BYTES = 8 * 1024 * 1024 * 1024
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Per-request wall-clock budget.
|
|
62
|
+
*
|
|
63
|
+
* WHY PER REQUEST AND NOT PER TRANSFER: a timeout on the whole transfer would abort a 6 GiB copy
|
|
64
|
+
* that is making perfect progress on a slow link. What has to be bounded is the gap -- a request
|
|
65
|
+
* that has stopped moving -- so the budget is applied per chunk and per stream, and a healthy
|
|
66
|
+
* transfer may legitimately run for hours.
|
|
67
|
+
*/
|
|
68
|
+
export const DEFAULT_TRANSFER_REQUEST_TIMEOUT_MS = 5 * 60_000
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* How many HTTP round trips one chunk may cost before the alignment loop gives up.
|
|
72
|
+
*
|
|
73
|
+
* WHY THIS EXISTS: the misalignment loop (`409` -> re-align -> retry the same chunk) terminates only
|
|
74
|
+
* because the receiver's `.part` grows. If a buggy or hostile receiver answers every `PUT` with the
|
|
75
|
+
* SAME `expectedOffset`, the loop would spin forever and the caller would see a hang rather than a
|
|
76
|
+
* failure. Measured risk is low, but "cannot happen" is not the same as "cannot loop", and a bounded
|
|
77
|
+
* number of attempts turns a hang into a named error.
|
|
78
|
+
*/
|
|
79
|
+
const MAX_ALIGN_ATTEMPTS = 5
|
|
80
|
+
|
|
81
|
+
/** Status codes this module gives names to, so the branches that act on them read as intent. */
|
|
82
|
+
const FILE_EXISTS = 409
|
|
83
|
+
const OFFSET_MISMATCH = 409
|
|
84
|
+
const TOO_LARGE = 413
|
|
85
|
+
const RANGE_NOT_SATISFIABLE = 416
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Diagnostic ceiling for error bodies. A worker's error body is JSON with a sentence in it; the
|
|
89
|
+
* screenshot path already uses 200 and there is no reason for a transfer error to carry more.
|
|
90
|
+
*/
|
|
91
|
+
const ERROR_BODY_CHARS = 200
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* An error whose `status` a caller may branch on, plus any structured fields the endpoint returned
|
|
95
|
+
* (notably `expectedOffset` from a `409`, which is what lets the upload re-align instead of guessing).
|
|
96
|
+
*/
|
|
97
|
+
export class TransferError extends Error {
|
|
98
|
+
/**
|
|
99
|
+
* @param {string} message
|
|
100
|
+
* @param {{ status?: number, detail?: object | null }} [opts]
|
|
101
|
+
*/
|
|
102
|
+
constructor(message, opts = {}) {
|
|
103
|
+
super(message)
|
|
104
|
+
this.name = 'TransferError'
|
|
105
|
+
this.status = Number.isFinite(opts.status) ? Number(opts.status) : 0
|
|
106
|
+
this.detail = opts.detail ?? null
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// ---------------------------------------------------------------------------
|
|
111
|
+
// paths
|
|
112
|
+
// ---------------------------------------------------------------------------
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Where the host keeps files pulled from workers.
|
|
116
|
+
*
|
|
117
|
+
* WHY NOT THE SCREENSHOT DIRECTORY: the screenshots live in the system temp directory because they
|
|
118
|
+
* are automatic byproducts that may be pruned without anyone noticing. A received file is a
|
|
119
|
+
* DELIBERATE product -- the operator asked for it -- so it belongs beside the plugin's other state,
|
|
120
|
+
* and it is never pruned automatically (see `docs/DESIGN.md`).
|
|
121
|
+
*
|
|
122
|
+
* @param {Record<string, string | undefined>} [env]
|
|
123
|
+
* @returns {string}
|
|
124
|
+
*/
|
|
125
|
+
export function transfersDir(env = process.env) {
|
|
126
|
+
return join(resolveDshHome(env), 'storages', 'dsh-plugin-remote-use', 'transfers')
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* The directory downloads land in. Created on demand, never pruned here.
|
|
131
|
+
* @param {Record<string, string | undefined>} [env]
|
|
132
|
+
* @returns {string}
|
|
133
|
+
*/
|
|
134
|
+
export function defaultReceiveDir(env = process.env) {
|
|
135
|
+
return transfersDir(env)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** The staging name for a destination. Always a sibling, so the rename is on one filesystem. */
|
|
139
|
+
export function partPathFor(path) {
|
|
140
|
+
return `${path}.part`
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Create a directory and everything above it. Extracted so both directions say it the same way. */
|
|
144
|
+
function ensureDir(dir) {
|
|
145
|
+
mkdirSync(dir, { recursive: true })
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// ---------------------------------------------------------------------------
|
|
149
|
+
// request plumbing
|
|
150
|
+
// ---------------------------------------------------------------------------
|
|
151
|
+
|
|
152
|
+
/** Normalize a caller-supplied channel so an HTTP error can never be reported for the wrong peer. */
|
|
153
|
+
function peerLabel(peerId) {
|
|
154
|
+
return peerId === undefined || peerId === null || String(peerId) === '' ? 'unknown' : String(peerId)
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Run an injected channel, stamping the peer into the failure.
|
|
159
|
+
*
|
|
160
|
+
* WHY THE STAMP IS MANDATORY: the host talks to several workers and every one of them speaks the
|
|
161
|
+
* same protocol on the same paths. "fetch failed" with no peer id is unactionable; the screenshot
|
|
162
|
+
* path learned this and prefixes every error with `worker "<id>"`.
|
|
163
|
+
*/
|
|
164
|
+
async function request(fn, peerId, what) {
|
|
165
|
+
try {
|
|
166
|
+
return await fn()
|
|
167
|
+
} catch (error) {
|
|
168
|
+
throw new TransferError(`worker "${peerLabel(peerId)}" ${what} failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** A JSON body of an error response, when there is one. Never throws: a body may be missing. */
|
|
173
|
+
async function readErrorDetail(res) {
|
|
174
|
+
try {
|
|
175
|
+
const text = await res.text()
|
|
176
|
+
try {
|
|
177
|
+
const parsed = JSON.parse(text)
|
|
178
|
+
if (parsed !== null && typeof parsed === 'object') return parsed
|
|
179
|
+
} catch { /* not JSON: the text is still the best detail available */ }
|
|
180
|
+
return text === '' ? null : text.slice(0, ERROR_BODY_CHARS)
|
|
181
|
+
} catch {
|
|
182
|
+
return null
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Render a detail for a human. Accepts the two shapes `readErrorDetail` may return. */
|
|
187
|
+
function detailText(detail) {
|
|
188
|
+
if (detail === null || detail === undefined) return ''
|
|
189
|
+
if (typeof detail === 'string') return detail
|
|
190
|
+
const message = detail.error ?? detail.message
|
|
191
|
+
return typeof message === 'string' ? message : JSON.stringify(detail).slice(0, ERROR_BODY_CHARS)
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* `409`/`400` responses carry the server's own `error` string; lifting it out of the JSON matters
|
|
196
|
+
* because otherwise a misalignment is reported as the bare status and the operator cannot tell it
|
|
197
|
+
* apart from a missing file or a rejected token.
|
|
198
|
+
*/
|
|
199
|
+
function httpError(peerId, verb, status, detail) {
|
|
200
|
+
const text = detailText(detail)
|
|
201
|
+
return new TransferError(
|
|
202
|
+
`worker "${peerLabel(peerId)}" ${verb} returned HTTP ${status}${text === '' ? '' : `: ${text}`}`,
|
|
203
|
+
{ status, detail },
|
|
204
|
+
)
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** The offset a `409` asked for, or NaN when the response did not name one. */
|
|
208
|
+
function expectedOffsetOf(detail) {
|
|
209
|
+
const value = Number(detail?.expectedOffset)
|
|
210
|
+
return Number.isFinite(value) && value >= 0 ? value : NaN
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Encode one query value.
|
|
215
|
+
*
|
|
216
|
+
* WHY NOT `URLSearchParams`: it encodes a space as `+`, and a path is not a form field -- `+` is a
|
|
217
|
+
* legal filename character, so `a+b.txt` would come back as `a b.txt` on the worker. `encodeURIComponent`
|
|
218
|
+
* is the top-level `set` of the same table without the form encoding.
|
|
219
|
+
*/
|
|
220
|
+
function queryValue(value) {
|
|
221
|
+
return encodeURIComponent(String(value))
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** A short, peer-scoped label for a transfer, used in the error text the operator reads. */
|
|
225
|
+
function describe(direction, peerId, from, to) {
|
|
226
|
+
return direction === 'upload'
|
|
227
|
+
? `upload to worker "${peerLabel(peerId)}" (${from} -> ${to})`
|
|
228
|
+
: `download from worker "${peerLabel(peerId)}" (${to} <- ${from})`
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
// ---------------------------------------------------------------------------
|
|
232
|
+
// hashing
|
|
233
|
+
// ---------------------------------------------------------------------------
|
|
234
|
+
|
|
235
|
+
/** Cheap commit-then-digest handle, so a hasher can never be reused after it has produced a value. */
|
|
236
|
+
export function createHasher() {
|
|
237
|
+
const hash = createHash('sha256')
|
|
238
|
+
let taken = false
|
|
239
|
+
return {
|
|
240
|
+
update(chunk) { hash.update(chunk) },
|
|
241
|
+
digest() {
|
|
242
|
+
if (taken) throw new Error('createHasher: digest() was already taken')
|
|
243
|
+
taken = true
|
|
244
|
+
return hash.digest('hex')
|
|
245
|
+
},
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** sha256 of an existing file, read once in slices. Used to trust a resume point. */
|
|
250
|
+
function hashFileSync(path, chunk = TRANSFER_CHUNK_BYTES) {
|
|
251
|
+
const hasher = createHasher()
|
|
252
|
+
const fd = openSync(path, 'r')
|
|
253
|
+
try {
|
|
254
|
+
const buf = Buffer.allocUnsafe(chunk)
|
|
255
|
+
for (;;) {
|
|
256
|
+
const n = readSync(fd, buf, 0, buf.length, null)
|
|
257
|
+
if (n <= 0) break
|
|
258
|
+
hasher.update(buf.subarray(0, n))
|
|
259
|
+
}
|
|
260
|
+
} finally {
|
|
261
|
+
closeSync(fd)
|
|
262
|
+
}
|
|
263
|
+
return hasher.digest()
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** sha256 of a buffer or string. The commit comparison is the only caller. */
|
|
267
|
+
function sha256Of(bytes) {
|
|
268
|
+
return createHash('sha256').update(bytes).digest('hex')
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* WHERE TWO DIGESTS FIRST DISAGREE, as a character index (2 hex characters per byte).
|
|
273
|
+
*
|
|
274
|
+
* ⚠️ THIS IS A PROGRESS MARKER, NOT A LOCALIZATION. A hash diffuses: changing ONE byte near the end
|
|
275
|
+
* changes the FIRST character with probability 15/16, so in practice `divergeAtByte` is almost always
|
|
276
|
+
* 0 and a download that fails verification is re-pulled from the start. The comparison is kept because
|
|
277
|
+
* it costs nothing and is correct when it does bite (two files that share a long prefix, e.g. a remote
|
|
278
|
+
* that only had bytes APPENDED, produce a late divergence), but nothing may depend on it localizing
|
|
279
|
+
* damage -- the honest description of a digest mismatch is "this content is not that file".
|
|
280
|
+
*/
|
|
281
|
+
function digestDivergence(expected, actual) {
|
|
282
|
+
const n = Math.min(expected.length, actual.length)
|
|
283
|
+
for (let i = 0; i < n; i += 1) {
|
|
284
|
+
if (expected[i] !== actual[i]) return i
|
|
285
|
+
}
|
|
286
|
+
return n
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/** Replace a staged file with a fresh one of exactly `size` bytes (0 truncates). */
|
|
290
|
+
function resetPart(path, size) {
|
|
291
|
+
mkdirSync(dirname(path), { recursive: true })
|
|
292
|
+
const fd = openSync(path, 'w')
|
|
293
|
+
try {
|
|
294
|
+
if (size > 0) truncateSync(fd, size)
|
|
295
|
+
} finally {
|
|
296
|
+
closeSync(fd)
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* Write one buffer at an absolute offset, looping over short writes.
|
|
302
|
+
*
|
|
303
|
+
* `writeSync` is documented to return the number of bytes written and may return fewer than asked
|
|
304
|
+
* (a signal, a nearly-full disk edge). Treating the return value as "all of it" is how a file ends
|
|
305
|
+
* up silently short, and a short file would then fail the commit check at the far end with a hash
|
|
306
|
+
* mismatch that points nowhere near the real bug.
|
|
307
|
+
*/
|
|
308
|
+
function writeAll(fd, chunk, position) {
|
|
309
|
+
let written = 0
|
|
310
|
+
while (written < chunk.length) {
|
|
311
|
+
written += writeSync(fd, chunk, written, chunk.length - written, position + written)
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Digest + copy a prefix of a file without reading it twice, so the upload hash and the upload bytes
|
|
317
|
+
* are literally the same pass over disk. See `uploadFile` for why the prefix needs re-hashing at all.
|
|
318
|
+
*/
|
|
319
|
+
function seedPrefix(hasher, path, bytes, chunk = TRANSFER_CHUNK_BYTES) {
|
|
320
|
+
const fd = openSync(path, 'r')
|
|
321
|
+
try {
|
|
322
|
+
const buf = Buffer.allocUnsafe(Math.min(chunk, Math.max(1, bytes)))
|
|
323
|
+
let read = 0
|
|
324
|
+
while (read < bytes) {
|
|
325
|
+
const want = Math.min(buf.length, bytes - read)
|
|
326
|
+
const n = readSync(fd, buf, 0, want, read)
|
|
327
|
+
if (n <= 0) throw new TransferError(`source file shrank while reading ${path} (${read} of ${bytes} bytes)`)
|
|
328
|
+
hasher.update(buf.subarray(0, n))
|
|
329
|
+
read += n
|
|
330
|
+
}
|
|
331
|
+
} finally {
|
|
332
|
+
closeSync(fd)
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// ---------------------------------------------------------------------------
|
|
337
|
+
// upload
|
|
338
|
+
// ---------------------------------------------------------------------------
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Open the slice that will be a request body, with the cleanup a failed request needs.
|
|
342
|
+
*
|
|
343
|
+
* WHY THE STREAM MUST BE TORN DOWN BY HAND: when the receiver answers `409` (or anything else) without
|
|
344
|
+
* draining the body, `fetch` rejects and the read stream we handed it is left open and unread. An
|
|
345
|
+
* abandoned `ReadStream` keeps its file descriptor and fires an `error` on some later tick -- and the
|
|
346
|
+
* listenerless version of that error is an uncaught exception, i.e. the process dies from a bad
|
|
347
|
+
* response to one chunk. `.destroy()` releases the descriptor and is idempotent, so calling it on a
|
|
348
|
+
* fully-consumed stream is a no-op the success path does not have to think about.
|
|
349
|
+
*/
|
|
350
|
+
function openSlice(path, offset, length) {
|
|
351
|
+
const stream = createReadStream(path, { start: offset, end: offset + length - 1 })
|
|
352
|
+
stream.on('error', () => { /* the failure is reported by the request, not by a second handler */ })
|
|
353
|
+
return {
|
|
354
|
+
stream,
|
|
355
|
+
release() {
|
|
356
|
+
if (!stream.destroyed) stream.destroy()
|
|
357
|
+
},
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/**
|
|
362
|
+
* Push one slice as a `PUT`, as a STREAM rather than a buffer.
|
|
363
|
+
*
|
|
364
|
+
* WHY A STREAM BODY: the whole constraint this module exists for is not holding the file in memory.
|
|
365
|
+
* A `Buffer` of one 4 MiB slice would technically satisfy the contract's chunking, but it would also
|
|
366
|
+
* make the memory ceiling a function of the slice size, and the next person to raise
|
|
367
|
+
* `TRANSFER_CHUNK_BYTES` would raise the ceiling with it. `duplex: 'half'` is what tells `fetch`
|
|
368
|
+
* that the body is a stream it may write out and not expect a response on; without it, undici
|
|
369
|
+
* refuses a stream body outright.
|
|
370
|
+
*
|
|
371
|
+
* @returns {Promise<{ status: number, body: object | null }>}
|
|
372
|
+
*/
|
|
373
|
+
async function putChunk(channel, peerId, remotePath, offset, length, bodyPath, timeoutMs, signal) {
|
|
374
|
+
const url = `${channel.base}/file?path=${queryValue(remotePath)}&offset=${offset}`
|
|
375
|
+
const timeout = armTimeout(timeoutMs, signal)
|
|
376
|
+
const slice = openSlice(bodyPath, offset, length)
|
|
377
|
+
let res
|
|
378
|
+
try {
|
|
379
|
+
res = await request(() => channel.put(url, {
|
|
380
|
+
method: 'PUT',
|
|
381
|
+
// content-length is mandatory per the contract: a chunked PUT is refused with 411, because
|
|
382
|
+
// without a declared length the receiver cannot tell a complete slice from a truncated one.
|
|
383
|
+
headers: { ...channel.authHeaders(), 'content-length': String(length) },
|
|
384
|
+
body: slice.stream,
|
|
385
|
+
duplex: 'half',
|
|
386
|
+
signal: timeout.signal,
|
|
387
|
+
}), peerId, `PUT /file (offset ${offset})`)
|
|
388
|
+
} catch (error) {
|
|
389
|
+
slice.release()
|
|
390
|
+
throw error
|
|
391
|
+
} finally {
|
|
392
|
+
timeout.disarm()
|
|
393
|
+
}
|
|
394
|
+
if (res.status === 200) {
|
|
395
|
+
// The JSON body is advisory only; the loop never trusts it over the commit check.
|
|
396
|
+
let body = null
|
|
397
|
+
try { body = await res.json() } catch { /* a 200 with an unreadable body still moved the bytes */ }
|
|
398
|
+
slice.release()
|
|
399
|
+
return { status: 200, body }
|
|
400
|
+
}
|
|
401
|
+
slice.release()
|
|
402
|
+
const detail = await readErrorDetail(res)
|
|
403
|
+
if (res.status === OFFSET_MISMATCH) return { status: res.status, body: detail }
|
|
404
|
+
throw httpError(peerId, `PUT /file (offset ${offset})`, res.status, detail)
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Upload a local file and leave it at `remotePath` on the worker.
|
|
409
|
+
*
|
|
410
|
+
* THE ORDER IS THE PROOF: bytes first, name second. Every slice lands in `<remote>.part`; only after
|
|
411
|
+
* the receiver has recomputed the whole-file sha256 of that `.part` and found it equal to the digest
|
|
412
|
+
* computed here DURING THE SAME READ does the receiver rename it. Either side can therefore crash at
|
|
413
|
+
* any instant without leaving a plausible file at the real name.
|
|
414
|
+
*
|
|
415
|
+
* @param {object} opts
|
|
416
|
+
* @param {string} opts.peerId
|
|
417
|
+
* @param {string} opts.localPath file to read
|
|
418
|
+
* @param {string} opts.remotePath path ON THE WORKER (relative = task directory)
|
|
419
|
+
* @param {object} opts.channel
|
|
420
|
+
* @param {string} opts.channel.base
|
|
421
|
+
* @param {() => Record<string, string>} opts.channel.authHeaders
|
|
422
|
+
* @param {(url: string, init: object) => Promise<object>} opts.channel.put
|
|
423
|
+
* @param {(url: string, init: object) => Promise<object>} [opts.channel.head] cheap pre-flight probe
|
|
424
|
+
* @param {boolean} [opts.overwrite] false/absent refuses an existing remote target
|
|
425
|
+
* @param {number} [opts.maxBytes] host-side ceiling; refuse BEFORE any request
|
|
426
|
+
* @param {number} [opts.timeoutMs] per-request budget
|
|
427
|
+
* @param {AbortSignal} [opts.signal]
|
|
428
|
+
* @param {(p: { direction: string, peerId: string, path: string, offset: number, bytes: number,
|
|
429
|
+
* total: number, percent: number, attempts: number, phase: string }) => void} [opts.onProgress]
|
|
430
|
+
* @returns {Promise<{ ok: true, direction: 'upload', peerId: string, localPath: string,
|
|
431
|
+
* remotePath: string, size: number, sha256: string, bytes: number, offset: number,
|
|
432
|
+
* attempts: number, aligned: boolean, commit: object | null, elapsedMs: number }>}
|
|
433
|
+
*/
|
|
434
|
+
export async function uploadFile(opts) {
|
|
435
|
+
const started = Date.now()
|
|
436
|
+
const peerId = opts.peerId
|
|
437
|
+
const localPath = String(opts.localPath ?? '')
|
|
438
|
+
const remotePath = String(opts.remotePath ?? '')
|
|
439
|
+
const channel = opts.channel
|
|
440
|
+
if (channel === undefined || channel === null || typeof channel.put !== 'function') {
|
|
441
|
+
throw new TransferError(`upload to worker "${peerLabel(peerId)}": a transfer channel with put() is required`)
|
|
442
|
+
}
|
|
443
|
+
assertUsablePath(localPath, 'local')
|
|
444
|
+
assertUsablePath(remotePath, 'remote')
|
|
445
|
+
|
|
446
|
+
let size
|
|
447
|
+
try {
|
|
448
|
+
size = statSync(localPath).size
|
|
449
|
+
} catch (error) {
|
|
450
|
+
throw new TransferError(`upload to worker "${peerLabel(peerId)}": cannot read ${localPath}: ${error instanceof Error ? error.message : String(error)}`)
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
// PRE-FLIGHT, BEFORE ANY REQUEST. Finding out at 99% that the far end will refuse a 9 GiB file
|
|
454
|
+
// costs the operator the whole upload; the same answer is available from two numbers on disk, and
|
|
455
|
+
// the worker enforces its own ceiling anyway (defence in depth: this end cannot know a token
|
|
456
|
+
// someone else holds).
|
|
457
|
+
const maxBytes = Number.isFinite(opts.maxBytes) ? Number(opts.maxBytes) : DEFAULT_TRANSFER_MAX_BYTES
|
|
458
|
+
if (maxBytes > 0 && size > maxBytes) {
|
|
459
|
+
throw new TransferError(
|
|
460
|
+
describe('upload', peerId, localPath, remotePath) + ` refused: ${size} bytes exceeds the ${maxBytes}-byte limit`,
|
|
461
|
+
{ status: TOO_LARGE },
|
|
462
|
+
)
|
|
463
|
+
}
|
|
464
|
+
await probeRemoteSize(channel, peerId, remotePath, maxBytes)
|
|
465
|
+
|
|
466
|
+
const onProgress = typeof opts.onProgress === 'function' ? opts.onProgress : () => {}
|
|
467
|
+
const chunkBytes = Number.isFinite(opts.chunkBytes) && Number(opts.chunkBytes) > 0
|
|
468
|
+
? Math.floor(Number(opts.chunkBytes))
|
|
469
|
+
: TRANSFER_CHUNK_BYTES
|
|
470
|
+
|
|
471
|
+
let offset = 0
|
|
472
|
+
let attempts = 0
|
|
473
|
+
let aligned = false
|
|
474
|
+
const maxAttempts = Math.ceil(size / chunkBytes) + MAX_ALIGN_ATTEMPTS
|
|
475
|
+
|
|
476
|
+
/**
|
|
477
|
+
* The hash covers the bytes the WORKER will hold, and nothing else.
|
|
478
|
+
*
|
|
479
|
+
* It is seeded from the bytes the receiver already has, then extended by every slice we send. On a
|
|
480
|
+
* fresh transfer the seed is empty, so the digest is exactly one pass over the file and no second
|
|
481
|
+
* read is needed to compute it. Re-seeding happens only when a `409` proves the receiver is ahead
|
|
482
|
+
* of us (a previous transfer died after landing slices), and that re-read is the price of being
|
|
483
|
+
* able to CONTINUE: restarting would re-send gigabytes purely to justify a hash.
|
|
484
|
+
*
|
|
485
|
+
* `seededFor` is what keeps this honest. The digest is only the file's digest while the hasher has
|
|
486
|
+
* consumed exactly the prefix `[0, offset)`; if a resync moves the offset to a prefix the hasher
|
|
487
|
+
* has not seen, the numbers would silently describe different byte ranges and the commit endpoint
|
|
488
|
+
* would reject a transfer that actually succeeded. Rebuilding in one place removes that class.
|
|
489
|
+
*/
|
|
490
|
+
let hasher = createHasher()
|
|
491
|
+
let seededFor = 0
|
|
492
|
+
const reseedTo = (target) => {
|
|
493
|
+
const fresh = createHasher()
|
|
494
|
+
if (target > 0) seedPrefix(fresh, localPath, target)
|
|
495
|
+
hasher = fresh
|
|
496
|
+
seededFor = target
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
const buf = Buffer.allocUnsafe(chunkBytes)
|
|
500
|
+
const fd = openSync(localPath, 'r')
|
|
501
|
+
try {
|
|
502
|
+
while (offset < size) {
|
|
503
|
+
if (opts.signal?.aborted === true) throw new TransferError(`${describe('upload', peerId, localPath, remotePath)} aborted at offset ${offset}`)
|
|
504
|
+
const want = Math.min(chunkBytes, size - offset)
|
|
505
|
+
const got = readSync(fd, buf, 0, want, offset)
|
|
506
|
+
if (got <= 0) throw new TransferError(`${describe('upload', peerId, localPath, remotePath)}: source file shrank at offset ${offset}`)
|
|
507
|
+
const slice = buf.subarray(0, got)
|
|
508
|
+
|
|
509
|
+
attempts += 1
|
|
510
|
+
if (attempts > maxAttempts) {
|
|
511
|
+
throw new TransferError(`${describe('upload', peerId, localPath, remotePath)}: gave up after ${attempts} requests without making progress past offset ${offset}`)
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
const res = await putChunk(channel, peerId, remotePath, offset, got, localPath, requestTimeout(opts), opts.signal)
|
|
515
|
+
if (res.status === OFFSET_MISMATCH) {
|
|
516
|
+
const expected = expectedOffsetOf(res.body)
|
|
517
|
+
if (!Number.isFinite(expected)) {
|
|
518
|
+
throw httpError(peerId, `PUT /file (offset ${offset})`, OFFSET_MISMATCH, res.body)
|
|
519
|
+
}
|
|
520
|
+
// Re-align onto what the receiver ACTUALLY holds. `expected === 0` is the ordinary case -- a
|
|
521
|
+
// fresh upload whose slice was refused -- and needs only an empty hasher.
|
|
522
|
+
aligned = true
|
|
523
|
+
offset = expected
|
|
524
|
+
seededFor = -1
|
|
525
|
+
continue
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
if (seededFor !== offset) reseedTo(offset)
|
|
529
|
+
hasher.update(slice)
|
|
530
|
+
seededFor = offset + got
|
|
531
|
+
offset += got
|
|
532
|
+
onProgress(progress('upload', peerId, remotePath, offset, size, attempts, 'sending'))
|
|
533
|
+
}
|
|
534
|
+
} finally {
|
|
535
|
+
closeSync(fd)
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
/**
|
|
539
|
+
* ⚠️ THE RE-ALIGN CAN LAND EXACTLY ON `size`, SO THE GUARD MUST FINISH THE HASH RATHER THAN REFUSE.
|
|
540
|
+
*
|
|
541
|
+
* A `409` whose `expectedOffset` is already the whole file is a real state, not a corner invented by
|
|
542
|
+
* a test: the receiver holds every byte of `<remote>.part` (a previous attempt landed them all and
|
|
543
|
+
* then died before the commit, or its commit was lost), the master retries from 0, and the relay
|
|
544
|
+
* answers `409 expectedOffset=<size>`. The `409` handler sets `offset = size` and `seededFor = -1`,
|
|
545
|
+
* the loop condition `offset < size` is then FALSE, and the loop exits without ever hashing a byte.
|
|
546
|
+
*
|
|
547
|
+
* This used to be a hard failure -- "sent 5242880 of 5242880 bytes", which reads like a wire fault
|
|
548
|
+
* and is not one. There is nothing to recover on the wire: the SOURCE file is local and intact, so
|
|
549
|
+
* the digest of the range `[0, offset)` is always available by re-reading it. That re-read is the
|
|
550
|
+
* whole cost of accepting the re-align, and it is bounded by the offset, not by the transfer.
|
|
551
|
+
*
|
|
552
|
+
* WHY FINISHING HERE CANNOT MASK A WRONG FILE: `offset !== size` is still fatal (the loop only stops
|
|
553
|
+
* when it has sent everything), and the digest this produces is checked by the receiver against the
|
|
554
|
+
* `<remote>.part` it actually holds. A hasher that described the wrong range therefore fails LOUDLY
|
|
555
|
+
* at commit -- it cannot turn into a file that is quietly wrong.
|
|
556
|
+
*/
|
|
557
|
+
if (offset !== size) {
|
|
558
|
+
throw new TransferError(`${describe('upload', peerId, localPath, remotePath)}: sent ${offset} of ${size} bytes`)
|
|
559
|
+
}
|
|
560
|
+
if (seededFor !== size) reseedTo(size)
|
|
561
|
+
|
|
562
|
+
// The commit response is the only body a caller needs in memory -- a fixed single-digit number of
|
|
563
|
+
// bytes, read after every file byte has left, not the file.
|
|
564
|
+
const commit = await commitUpload(channel, peerId, remotePath, hasher.digest(), size, opts)
|
|
565
|
+
onProgress(progress('upload', peerId, remotePath, size, size, attempts, 'committed'))
|
|
566
|
+
return {
|
|
567
|
+
ok: true,
|
|
568
|
+
direction: 'upload',
|
|
569
|
+
peerId: peerLabel(peerId),
|
|
570
|
+
localPath,
|
|
571
|
+
remotePath,
|
|
572
|
+
size,
|
|
573
|
+
sha256: commit?.sha256 ?? '',
|
|
574
|
+
bytes: size,
|
|
575
|
+
offset,
|
|
576
|
+
attempts,
|
|
577
|
+
aligned,
|
|
578
|
+
commit,
|
|
579
|
+
// The WORKER's view of what it wrote, in the same shape `downloadFile` already reports (`remote`).
|
|
580
|
+
// The worker is the side that resolves a relative path, so the caller must be able to see the
|
|
581
|
+
// result and not only its own input. Its absence on this path is precisely why a relative upload
|
|
582
|
+
// could land in the wrong directory (the relay's inherited cwd, not the armed task dir) while
|
|
583
|
+
// every layer above reported success.
|
|
584
|
+
remote: {
|
|
585
|
+
path: String(commit?.path ?? remotePath),
|
|
586
|
+
mtime: String(commit?.mtime ?? ''),
|
|
587
|
+
bytes: Number.isFinite(commit?.bytes) ? Number(commit.bytes) : size,
|
|
588
|
+
},
|
|
589
|
+
elapsedMs: Date.now() - started,
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
/**
|
|
594
|
+
* A cheap "will the far end take this?" probe.
|
|
595
|
+
*
|
|
596
|
+
* WHY HEAD AND NOT NOTHING: the host ceiling and the worker ceiling can disagree (a token someone
|
|
597
|
+
* else holds, a worker configured tighter). `HEAD /file` answers the worker's view of the size for
|
|
598
|
+
* the price of one round trip, so a refusal arrives before the first byte. It is best-effort: a
|
|
599
|
+
* relay older than the endpoint, or one that answers with no length, must not block a transfer that
|
|
600
|
+
* would have worked.
|
|
601
|
+
*/
|
|
602
|
+
async function probeRemoteSize(channel, peerId, remotePath, maxBytes) {
|
|
603
|
+
if (typeof channel.head !== 'function') return
|
|
604
|
+
let res
|
|
605
|
+
try {
|
|
606
|
+
res = await channel.head(`${channel.base}/file?path=${queryValue(remotePath)}`, { headers: channel.authHeaders() })
|
|
607
|
+
} catch {
|
|
608
|
+
return
|
|
609
|
+
}
|
|
610
|
+
if (res === null || res === undefined) return
|
|
611
|
+
if (res.status === 413) {
|
|
612
|
+
throw new TransferError(`${describe('upload', peerId, '', remotePath)} refused by the worker: too large`, { status: TOO_LARGE })
|
|
613
|
+
}
|
|
614
|
+
const remoteSize = Number(res.headers?.get?.('x-file-size'))
|
|
615
|
+
if (Number.isFinite(remoteSize) && maxBytes > 0 && remoteSize > maxBytes) {
|
|
616
|
+
throw new TransferError(`${describe('upload', peerId, '', remotePath)} refused: the worker's copy is ${remoteSize} bytes, over the ${maxBytes}-byte limit`, { status: TOO_LARGE })
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
/**
|
|
621
|
+
* `POST /file/commit`: hand over the digest and the size and let the receiver verify its own `.part`.
|
|
622
|
+
*
|
|
623
|
+
* This is the single point at which a transfer becomes real. Nothing before it can produce a file at
|
|
624
|
+
* the destination name, so a mismatch here is a normal, recoverable outcome -- it is reported with
|
|
625
|
+
* both digests and the `.part` is left where it is.
|
|
626
|
+
*/
|
|
627
|
+
async function commitUpload(channel, peerId, remotePath, sha256, size, opts) {
|
|
628
|
+
if (typeof channel.post !== 'function') {
|
|
629
|
+
throw new TransferError(`upload to worker "${peerLabel(peerId)}": a transfer channel with post() is required to commit`)
|
|
630
|
+
}
|
|
631
|
+
const body = JSON.stringify({ path: remotePath, sha256, size, overwrite: opts.overwrite === true })
|
|
632
|
+
const timeout = armTimeout(requestTimeout(opts), opts.signal)
|
|
633
|
+
let res
|
|
634
|
+
try {
|
|
635
|
+
res = await request(() => channel.post(`${channel.base}/file/commit`, {
|
|
636
|
+
method: 'POST',
|
|
637
|
+
headers: { ...channel.authHeaders(), 'content-type': 'application/json', 'content-length': String(Buffer.byteLength(body)) },
|
|
638
|
+
body,
|
|
639
|
+
signal: timeout.signal,
|
|
640
|
+
}), peerId, 'POST /file/commit')
|
|
641
|
+
} finally {
|
|
642
|
+
timeout.disarm()
|
|
643
|
+
}
|
|
644
|
+
let detail = null
|
|
645
|
+
try { detail = await res.json() } catch { /* handled below per status */ }
|
|
646
|
+
if (res.status !== 200) {
|
|
647
|
+
const err = httpError(peerId, 'POST /file/commit', res.status, detail)
|
|
648
|
+
if (res.status === FILE_EXISTS) {
|
|
649
|
+
throw new TransferError(`${describe('upload', peerId, '', remotePath)} refused: the target exists on the worker (pass overwrite to replace it)`, { status: FILE_EXISTS, detail })
|
|
650
|
+
}
|
|
651
|
+
// A 400 here is the integrity check failing. Saying which digest the receiver computed is what
|
|
652
|
+
// makes it debuggable at all -- the two values differ, and the difference IS the diagnosis.
|
|
653
|
+
if (err.status === 400) {
|
|
654
|
+
const expected = typeof detail?.expected === 'string' ? detail.expected : sha256
|
|
655
|
+
const actual = typeof detail?.actual === 'string' ? detail.actual : ''
|
|
656
|
+
throw new TransferError(`${describe('upload', peerId, '', remotePath)} failed verification: worker sees sha256 ${actual || '(unknown)'} for its staged copy, host computed ${expected}`, { status: 400, detail })
|
|
657
|
+
}
|
|
658
|
+
throw err
|
|
659
|
+
}
|
|
660
|
+
return detail
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
// ---------------------------------------------------------------------------
|
|
664
|
+
// download
|
|
665
|
+
// ---------------------------------------------------------------------------
|
|
666
|
+
|
|
667
|
+
/**
|
|
668
|
+
* Pull a file from the worker by streaming its body straight to disk.
|
|
669
|
+
*
|
|
670
|
+
* WHY THE BODY IS CONSUMED AS AN ASYNC ITERATOR AND NOT VIA `pipeline()`: `pipeline` is the tidier
|
|
671
|
+
* construct when the destination is a path, but this destination is a handle that must stay open
|
|
672
|
+
* across a resume (closing it per attempt would turn a resume into a truncate). Iterating gives the
|
|
673
|
+
* same backpressure -- nothing is read until the previous slice has been written -- while keeping one
|
|
674
|
+
* handle, and the digest is folded into the same loop rather than a second pass over the bytes.
|
|
675
|
+
*
|
|
676
|
+
* @param {object} opts
|
|
677
|
+
* @param {string} opts.peerId
|
|
678
|
+
* @param {string} opts.remotePath
|
|
679
|
+
* @param {string} opts.destPath final local path; written only by an atomic rename
|
|
680
|
+
* @param {object} opts.channel
|
|
681
|
+
* @param {string} opts.channel.base
|
|
682
|
+
* @param {() => Record<string, string>} opts.channel.authHeaders
|
|
683
|
+
* @param {(url: string, init: object) => Promise<object>} opts.channel.get
|
|
684
|
+
* @param {boolean} [opts.overwrite]
|
|
685
|
+
* @param {number} [opts.maxBytes]
|
|
686
|
+
* @param {number} [opts.timeoutMs]
|
|
687
|
+
* @param {AbortSignal} [opts.signal]
|
|
688
|
+
* @param {Function} [opts.onProgress]
|
|
689
|
+
* @returns {Promise<{ ok: true, direction: 'download', peerId: string, remotePath: string,
|
|
690
|
+
* localPath: string, size: number, sha256: string, bytes: number,
|
|
691
|
+
* resumedFrom: number, alreadyComplete: boolean, remote: object,
|
|
692
|
+
* replacedNonAtomically: boolean, warnings: string[], elapsedMs: number }>}
|
|
693
|
+
*/
|
|
694
|
+
export async function downloadFile(opts) {
|
|
695
|
+
const started = Date.now()
|
|
696
|
+
const peerId = opts.peerId
|
|
697
|
+
const remotePath = String(opts.remotePath ?? '')
|
|
698
|
+
const destPath = String(opts.destPath ?? '')
|
|
699
|
+
const channel = opts.channel
|
|
700
|
+
if (channel === undefined || channel === null || typeof channel.get !== 'function') {
|
|
701
|
+
throw new TransferError(`download from worker "${peerLabel(peerId)}": a transfer channel with get() is required`)
|
|
702
|
+
}
|
|
703
|
+
assertUsablePath(remotePath, 'remote')
|
|
704
|
+
assertUsablePath(destPath, 'local')
|
|
705
|
+
|
|
706
|
+
const maxBytes = Number.isFinite(opts.maxBytes) ? Number(opts.maxBytes) : DEFAULT_TRANSFER_MAX_BYTES
|
|
707
|
+
const onProgress = typeof opts.onProgress === 'function' ? opts.onProgress : () => {}
|
|
708
|
+
const partPath = opts.partPath === undefined ? partPathFor(destPath) : String(opts.partPath)
|
|
709
|
+
const state = pathState(destPath)
|
|
710
|
+
|
|
711
|
+
if (existsSync(destPath) && opts.overwrite !== true) {
|
|
712
|
+
// Refusing is the data-safe default: a silent overwrite is the failure class this whole option
|
|
713
|
+
// exists for. Reported with the path so the caller can retry deliberately.
|
|
714
|
+
throw new TransferError(`${describe('download', peerId, remotePath, destPath)} refused: ${destPath} already exists (pass overwrite to replace it)`, { status: FILE_EXISTS })
|
|
715
|
+
}
|
|
716
|
+
ensureDir(dirname(destPath))
|
|
717
|
+
|
|
718
|
+
let attempts = 0
|
|
719
|
+
let restarted = 0
|
|
720
|
+
let staleDropped = false
|
|
721
|
+
for (;;) {
|
|
722
|
+
attempts += 1
|
|
723
|
+
const startOffset = existsSync(partPath) ? statSync(partPath).size : 0
|
|
724
|
+
|
|
725
|
+
let res
|
|
726
|
+
try {
|
|
727
|
+
res = await getFile(channel, peerId, remotePath, startOffset, opts)
|
|
728
|
+
} catch (error) {
|
|
729
|
+
/**
|
|
730
|
+
* 416 is the receiver saying our resume point is past the end of the remote file, which happens
|
|
731
|
+
* when something OUTSIDE this transfer shrank or replaced the file. Retrying that offset is a
|
|
732
|
+
* guaranteed infinite loop and resuming is impossible, so the stale staging file is dropped and
|
|
733
|
+
* the pull starts over -- once. A second 416 means the remote is moving faster than we can read
|
|
734
|
+
* it, and saying so is more useful than another attempt.
|
|
735
|
+
*/
|
|
736
|
+
if (error?.status !== RANGE_NOT_SATISFIABLE || staleDropped) throw error
|
|
737
|
+
staleDropped = true
|
|
738
|
+
rmSync(partPath, { force: true })
|
|
739
|
+
continue
|
|
740
|
+
}
|
|
741
|
+
const remote = readRemoteMeta(res)
|
|
742
|
+
if (maxBytes > 0 && remote.size > maxBytes) {
|
|
743
|
+
throw new TransferError(`${describe('download', peerId, remotePath, destPath)} refused: ${remote.size} bytes exceeds the ${maxBytes}-byte limit`, { status: TOO_LARGE })
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
if (startOffset >= remote.size && remote.size > 0) {
|
|
747
|
+
// The staging file is at least as long as the remote claims to be: either a previous attempt
|
|
748
|
+
// finished the bytes, or the remote got SHORTER since. The digest decides, and re-hashing a
|
|
749
|
+
// local file is far cheaper than re-downloading one.
|
|
750
|
+
const localHash = hashFileSync(partPath)
|
|
751
|
+
if (localHash === remote.sha256) {
|
|
752
|
+
truncateSync(partPath, remote.size)
|
|
753
|
+
finishDownload(partPath, destPath, state)
|
|
754
|
+
onProgress(progress('download', peerId, remotePath, remote.size, remote.size, attempts, 'renamed'))
|
|
755
|
+
return {
|
|
756
|
+
ok: true, direction: 'download', peerId: peerLabel(peerId), remotePath, localPath: destPath,
|
|
757
|
+
size: remote.size, sha256: remote.sha256, bytes: remote.size, resumedFrom: startOffset,
|
|
758
|
+
alreadyComplete: true, remote, replacedNonAtomically: state.replacedNonAtomically,
|
|
759
|
+
warnings: state.warnings, elapsedMs: Date.now() - started,
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
// The bytes on disk are not this file. Keeping a staging file we already know to be wrong
|
|
763
|
+
// would make every following attempt fail the same way.
|
|
764
|
+
rmSync(partPath, { force: true })
|
|
765
|
+
continue
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
const hasher = createHasher()
|
|
769
|
+
if (startOffset > 0) {
|
|
770
|
+
// Seed from what is already on disk so the digest covers the whole file, not just this tail.
|
|
771
|
+
seedPrefix(hasher, partPath, startOffset)
|
|
772
|
+
}
|
|
773
|
+
let seen = startOffset
|
|
774
|
+
const fd = openSync(partPath, 'a')
|
|
775
|
+
try {
|
|
776
|
+
await streamBody(res, fd, peerId, (chunk) => {
|
|
777
|
+
// One pass, two products: the digest of the whole file and the live progress figure. Reading
|
|
778
|
+
// the staged prefix again purely to hash it would double the cost of every resume.
|
|
779
|
+
hasher.update(chunk)
|
|
780
|
+
seen += chunk.length
|
|
781
|
+
onProgress(progress('download', peerId, remotePath, Math.min(remote.size || seen, seen), remote.size, attempts, 'receiving'))
|
|
782
|
+
}, opts)
|
|
783
|
+
} finally {
|
|
784
|
+
closeSync(fd)
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
const have = statSync(partPath).size
|
|
788
|
+
if (have !== remote.size) {
|
|
789
|
+
throw new TransferError(`${describe('download', peerId, remotePath, destPath)}: received ${have} of ${remote.size} bytes, staged copy kept at ${partPath} for resume`)
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
const actual = hasher.digest()
|
|
793
|
+
if (remote.sha256 === '' || actual === remote.sha256) {
|
|
794
|
+
// An empty `x-file-sha256` means the worker could not tell us; the size is then the only
|
|
795
|
+
// evidence there is, and pretending otherwise would refuse every download from an older relay.
|
|
796
|
+
finishDownload(partPath, destPath, state)
|
|
797
|
+
onProgress(progress('download', peerId, remotePath, remote.size, remote.size, attempts, 'renamed'))
|
|
798
|
+
return {
|
|
799
|
+
ok: true, direction: 'download', peerId: peerLabel(peerId), remotePath, localPath: destPath,
|
|
800
|
+
size: remote.size, sha256: actual, bytes: actual === remote.sha256 ? remote.size : have,
|
|
801
|
+
resumedFrom: startOffset, alreadyComplete: false, remote,
|
|
802
|
+
replacedNonAtomically: state.replacedNonAtomically, warnings: state.warnings,
|
|
803
|
+
elapsedMs: Date.now() - started,
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
if (restarted >= 1) {
|
|
808
|
+
// Two attempts have disagreed with the advertised digest. Repeating the download cannot fix a
|
|
809
|
+
// remote that is changing underneath us, so it is reported instead of retried forever.
|
|
810
|
+
throw new TransferError(`${describe('download', peerId, remotePath, destPath)} failed verification after restart: host computed sha256 ${actual}, worker advertised ${remote.sha256} (staged copy kept at ${partPath})`, { status: 400 })
|
|
811
|
+
}
|
|
812
|
+
restarted += 1
|
|
813
|
+
|
|
814
|
+
// Keeping the longest prefix the two digests agree on costs one comparison and sometimes saves a
|
|
815
|
+
// whole re-send (see `digestDivergence`); it must never be relied on to localize the damage.
|
|
816
|
+
const divergeAtByte = Math.floor(digestDivergence(remote.sha256, actual) / 2)
|
|
817
|
+
if (divergeAtByte >= 1) {
|
|
818
|
+
truncateSync(partPath, Math.min(divergeAtByte, statSync(partPath).size))
|
|
819
|
+
continue
|
|
820
|
+
}
|
|
821
|
+
rmSync(partPath, { force: true })
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
/**
|
|
826
|
+
* The post-transfer state of one destination file, with a warning channel for the rare cases.
|
|
827
|
+
*
|
|
828
|
+
* WHY A LITTLE STATE OBJECT AND NOT A MODULE-LEVEL ARRAY: the one fact worth reporting about a rename
|
|
829
|
+
* is "it still cost more than an atomic rename", and that fact belongs to ONE transfer. A shared
|
|
830
|
+
* buffer would have to be drained by whichever call happened to run next, which is how a warning ends
|
|
831
|
+
* up attached to the wrong file -- or to none.
|
|
832
|
+
*/
|
|
833
|
+
function pathState(path) {
|
|
834
|
+
return {
|
|
835
|
+
path,
|
|
836
|
+
/** True for the whole transfer after a non-atomic replace was needed once. */
|
|
837
|
+
replacedNonAtomically: false,
|
|
838
|
+
/** @type {string[]} */
|
|
839
|
+
warnings: [],
|
|
840
|
+
note(message) {
|
|
841
|
+
this.warnings.push(message)
|
|
842
|
+
},
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
/**
|
|
847
|
+
* Move the verified staging file into place.
|
|
848
|
+
*
|
|
849
|
+
* WHY THE UNLINK IS A FALLBACK AND NOT THE PATH: `rename` is atomic, unlink-then-rename is not -- it
|
|
850
|
+
* has a window in which the destination does not exist at all. On every platform this plugin targets,
|
|
851
|
+
* a rename over an existing file succeeds (POSIX `rename(2)`, Windows via `MoveFileEx` with replace),
|
|
852
|
+
* so the fallback should never run; it exists because a filesystem that refuses to replace (a share,
|
|
853
|
+
* an antivirus holding a handle) must degrade to "worked, with a brief gap" rather than "failed".
|
|
854
|
+
* Every fallback is recorded, so a silent degradation cannot hide. See also the header's invariant 2.
|
|
855
|
+
*/
|
|
856
|
+
function replaceFile(from, to, state) {
|
|
857
|
+
try {
|
|
858
|
+
renameSync(from, to)
|
|
859
|
+
return true
|
|
860
|
+
} catch (error) {
|
|
861
|
+
const code = error?.code
|
|
862
|
+
if (code !== 'EEXIST' && code !== 'EPERM' && code !== 'EACCES') {
|
|
863
|
+
throw new TransferError(`cannot move ${from} into place at ${to}: ${error instanceof Error ? error.message : String(error)}`)
|
|
864
|
+
}
|
|
865
|
+
state.replacedNonAtomically = true
|
|
866
|
+
state.note(`had to replace ${to} in two steps (${code}); the destination path was briefly absent`)
|
|
867
|
+
}
|
|
868
|
+
try {
|
|
869
|
+
rmSync(to, { force: true })
|
|
870
|
+
renameSync(from, to)
|
|
871
|
+
return true
|
|
872
|
+
} catch (error) {
|
|
873
|
+
throw new TransferError(`cannot move ${from} into place at ${to}: ${error instanceof Error ? error.message : String(error)}`)
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
/**
|
|
878
|
+
* Atomic-move the staged file into place, carrying the destination's state so a non-atomic fallback
|
|
879
|
+
* reaches the caller instead of vanishing. `overwrite` is not consulted: the rename replaces whatever
|
|
880
|
+
* is at the destination in one step, and the refusal for an existing file happens BEFORE any bytes are
|
|
881
|
+
* staged, so a refused download costs nothing.
|
|
882
|
+
*/
|
|
883
|
+
function finishDownload(partPath, destPath, state) {
|
|
884
|
+
replaceFile(partPath, destPath, state)
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
/** Everything the response headers say about the remote file, normalized and typed. */
|
|
888
|
+
function readRemoteMeta(res) {
|
|
889
|
+
const header = (name) => {
|
|
890
|
+
const value = res?.headers?.get?.(name)
|
|
891
|
+
return value === undefined || value === null ? '' : String(value)
|
|
892
|
+
}
|
|
893
|
+
const size = Number(header('x-file-size'))
|
|
894
|
+
return {
|
|
895
|
+
size: Number.isFinite(size) && size >= 0 ? size : 0,
|
|
896
|
+
sha256: header('x-file-sha256').toLowerCase(),
|
|
897
|
+
// The worker echoes the RESOLVED absolute path; keeping it is what makes the audit row say where
|
|
898
|
+
// the bytes really went rather than where we guessed.
|
|
899
|
+
path: header('x-file-path'),
|
|
900
|
+
mtime: header('x-file-mtime'),
|
|
901
|
+
offset: Number(header('x-file-offset')) || 0,
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
/**
|
|
906
|
+
* One `GET /file`, with the two statuses that mean "your resume point is stale" handled where they
|
|
907
|
+
* occur rather than by the caller.
|
|
908
|
+
*
|
|
909
|
+
* WHY 416 GETS ITS OWN PATH: `offset >= size` on the receiver means the staging file is LONGER than
|
|
910
|
+
* the remote. Something outside this transfer shrank the remote file. Resuming is impossible and
|
|
911
|
+
* retrying the same offset is a guaranteed infinite loop, so the stale staging file is dropped and
|
|
912
|
+
* the download starts over -- once, loudly, because it costs a full re-send the operator may want to
|
|
913
|
+
* know about.
|
|
914
|
+
*/
|
|
915
|
+
async function getFile(channel, peerId, remotePath, offset, opts) {
|
|
916
|
+
const suffix = offset > 0 ? `&offset=${offset}` : ''
|
|
917
|
+
const url = `${channel.base}/file?path=${queryValue(remotePath)}${suffix}`
|
|
918
|
+
const timeout = armTimeout(requestTimeout(opts), opts.signal)
|
|
919
|
+
let res
|
|
920
|
+
try {
|
|
921
|
+
res = await request(() => channel.get(url, {
|
|
922
|
+
headers: channel.authHeaders(),
|
|
923
|
+
signal: timeout.signal,
|
|
924
|
+
}), peerId, `GET /file (offset ${offset})`)
|
|
925
|
+
} finally {
|
|
926
|
+
timeout.disarm()
|
|
927
|
+
}
|
|
928
|
+
if (res.status === 200 || res.status === 206) return res
|
|
929
|
+
const detail = await readErrorDetail(res)
|
|
930
|
+
throw httpError(peerId, `GET /file (offset ${offset})`, res.status, detail)
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
/**
|
|
934
|
+
* Stream a response body into an already-open, already-positioned file handle.
|
|
935
|
+
*
|
|
936
|
+
* The handle stays open across a resume precisely so a retry can continue at the byte it reached;
|
|
937
|
+
* reopening it per attempt is what would make a resume silently start over.
|
|
938
|
+
*/
|
|
939
|
+
async function streamBody(res, fd, peerId, onChunk, opts) {
|
|
940
|
+
const body = res.body
|
|
941
|
+
if (body === null || body === undefined) {
|
|
942
|
+
throw new TransferError(`worker "${peerLabel(peerId)}" returned HTTP ${res.status} with no body`)
|
|
943
|
+
}
|
|
944
|
+
let written = 0
|
|
945
|
+
try {
|
|
946
|
+
const stream = Readable.fromWeb(body)
|
|
947
|
+
for await (const chunk of stream) {
|
|
948
|
+
if (opts.signal?.aborted === true) {
|
|
949
|
+
stream.destroy()
|
|
950
|
+
throw new TransferError(`worker "${peerLabel(peerId)}" download aborted after ${written} bytes`)
|
|
951
|
+
}
|
|
952
|
+
writeAll(fd, chunk, null)
|
|
953
|
+
written += chunk.length
|
|
954
|
+
onChunk(chunk)
|
|
955
|
+
}
|
|
956
|
+
} catch (error) {
|
|
957
|
+
if (error instanceof TransferError) throw error
|
|
958
|
+
// The staged bytes are deliberately NOT removed: they are the resume point the next attempt
|
|
959
|
+
// starts from, and deleting them would turn an interrupted link into a full re-transfer.
|
|
960
|
+
throw new TransferError(`worker "${peerLabel(peerId)}" download failed after ${written} bytes: ${error instanceof Error ? error.message : String(error)}`)
|
|
961
|
+
}
|
|
962
|
+
return written
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
// ---------------------------------------------------------------------------
|
|
966
|
+
// small RPC helpers
|
|
967
|
+
// ---------------------------------------------------------------------------
|
|
968
|
+
|
|
969
|
+
/**
|
|
970
|
+
* The transport options of an RPC call, separated from the call's own arguments.
|
|
971
|
+
*
|
|
972
|
+
* WHY THEY MUST NOT BE FORWARDED TOGETHER: `listFiles` takes a `limit` and the channel takes a
|
|
973
|
+
* `timeoutMs`. Passing the caller's whole options object as the transport options hands `limit` to the
|
|
974
|
+
* bridge, which is one typo away from a real argument (or a real timeout) travelling in the wrong
|
|
975
|
+
* field. The split is stated here once so both helpers cannot disagree about it.
|
|
976
|
+
*/
|
|
977
|
+
function rpcOptions(opts) {
|
|
978
|
+
return Number.isFinite(opts?.timeoutMs) ? { timeoutMs: Number(opts.timeoutMs) } : {}
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
/**
|
|
982
|
+
* `statFile` -- "does this exist, how big is it, is it a directory" over the WS channel.
|
|
983
|
+
*
|
|
984
|
+
* WHY THE RPC AND NOT `HEAD /file`: existence and kind are a handful of bytes, and the JSON-RPC
|
|
985
|
+
* channel is already authenticated and already connected. `HEAD` is kept for the pre-flight size
|
|
986
|
+
* check, where the headers are the point.
|
|
987
|
+
*
|
|
988
|
+
* @param {{ rpc: (method: string, params: object, opts?: object) => Promise<object> }} client
|
|
989
|
+
* @param {string} path
|
|
990
|
+
* @returns {Promise<{ ok: boolean, path: string, exists: boolean, isDir: boolean, size: number, mtime: string, sha256: string }>}
|
|
991
|
+
*/
|
|
992
|
+
export async function statFile(client, path, opts = {}) {
|
|
993
|
+
if (typeof path !== 'string' || path === '' || path.includes('\u0000')) {
|
|
994
|
+
throw new TransferError('stat requires a non-empty path with no NUL bytes')
|
|
995
|
+
}
|
|
996
|
+
if (client === undefined || typeof client.rpc !== 'function') {
|
|
997
|
+
throw new TransferError('stat requires a connection to the worker (not connected)')
|
|
998
|
+
}
|
|
999
|
+
const result = await client.rpc('dsh-remote/stat', { path }, rpcOptions(opts))
|
|
1000
|
+
return {
|
|
1001
|
+
ok: result?.ok !== false,
|
|
1002
|
+
path: String(result?.path ?? path),
|
|
1003
|
+
exists: result?.exists === true,
|
|
1004
|
+
isDir: result?.isDir === true,
|
|
1005
|
+
// A directory has no meaningful size, and reporting a filesystem block count as one reads as a
|
|
1006
|
+
// real number the caller then compares against a budget.
|
|
1007
|
+
size: result?.isDir === true ? 0 : (Number(result?.size) || 0),
|
|
1008
|
+
mtime: typeof result?.mtime === 'string' ? result.mtime : '',
|
|
1009
|
+
sha256: typeof result?.sha256 === 'string' ? result.sha256 : '',
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
/**
|
|
1014
|
+
* `listFiles` -- one directory's entries, over the same channel.
|
|
1015
|
+
*
|
|
1016
|
+
* @param {{ rpc: Function }} client
|
|
1017
|
+
* @param {string} path
|
|
1018
|
+
* @param {{ limit?: number, timeoutMs?: number }} [opts]
|
|
1019
|
+
* @returns {Promise<{ ok: boolean, path: string, entries: Array<{name: string, isDir: boolean, size: number, mtime: string}>, truncated: boolean }>}
|
|
1020
|
+
*/
|
|
1021
|
+
export async function listFiles(client, path, opts = {}) {
|
|
1022
|
+
if (typeof path !== 'string' || path === '' || path.includes('\u0000')) {
|
|
1023
|
+
throw new TransferError('list requires a non-empty path with no NUL bytes')
|
|
1024
|
+
}
|
|
1025
|
+
if (client === undefined || typeof client.rpc !== 'function') {
|
|
1026
|
+
throw new TransferError('list requires a connection to the worker (not connected)')
|
|
1027
|
+
}
|
|
1028
|
+
const params = { path }
|
|
1029
|
+
if (Number.isFinite(opts.limit)) params.limit = Math.max(0, Math.floor(Number(opts.limit)))
|
|
1030
|
+
const result = await client.rpc('dsh-remote/list', params, rpcOptions(opts))
|
|
1031
|
+
const entries = Array.isArray(result?.entries) ? result.entries : []
|
|
1032
|
+
return {
|
|
1033
|
+
ok: result?.ok !== false,
|
|
1034
|
+
path: String(result?.path ?? path),
|
|
1035
|
+
entries: entries.map((entry) => ({
|
|
1036
|
+
name: String(entry?.name ?? ''),
|
|
1037
|
+
isDir: entry?.isDir === true,
|
|
1038
|
+
size: entry?.isDir === true ? 0 : (Number(entry?.size) || 0),
|
|
1039
|
+
mtime: typeof entry?.mtime === 'string' ? entry.mtime : '',
|
|
1040
|
+
})),
|
|
1041
|
+
truncated: result?.truncated === true,
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
/**
|
|
1046
|
+
* Give up on a transfer by removing the WORKER's staging file.
|
|
1047
|
+
*
|
|
1048
|
+
* WHY THIS CANNOT TOUCH THE TARGET: the contract says `DELETE /file` removes `<path>.part` and
|
|
1049
|
+
* nothing else. That is what makes "cancel" safe to offer at all -- the worst outcome of giving up
|
|
1050
|
+
* is losing resume progress, never a file.
|
|
1051
|
+
*
|
|
1052
|
+
* @param {object} opts
|
|
1053
|
+
* @param {string} opts.peerId
|
|
1054
|
+
* @param {string} opts.remotePath
|
|
1055
|
+
* @param {object} opts.channel
|
|
1056
|
+
* @returns {Promise<{ ok: boolean, removed: boolean, part: string }>}
|
|
1057
|
+
*/
|
|
1058
|
+
export async function abortTransfer(opts) {
|
|
1059
|
+
const peerId = opts.peerId
|
|
1060
|
+
const remotePath = String(opts.remotePath ?? '')
|
|
1061
|
+
const channel = opts.channel
|
|
1062
|
+
assertUsablePath(remotePath, 'remote')
|
|
1063
|
+
if (channel === undefined || typeof channel.del !== 'function') {
|
|
1064
|
+
throw new TransferError(`abort on worker "${peerLabel(peerId)}": a transfer channel with del() is required`)
|
|
1065
|
+
}
|
|
1066
|
+
const timeout = armTimeout(requestTimeout(opts), opts.signal)
|
|
1067
|
+
let res
|
|
1068
|
+
try {
|
|
1069
|
+
res = await request(() => channel.del(`${channel.base}/file?path=${queryValue(remotePath)}`, {
|
|
1070
|
+
method: 'DELETE',
|
|
1071
|
+
headers: channel.authHeaders(),
|
|
1072
|
+
signal: timeout.signal,
|
|
1073
|
+
}), peerId, 'DELETE /file')
|
|
1074
|
+
} finally {
|
|
1075
|
+
timeout.disarm()
|
|
1076
|
+
}
|
|
1077
|
+
if (res.status !== 200) throw httpError(peerId, 'DELETE /file', res.status, await readErrorDetail(res))
|
|
1078
|
+
let body = null
|
|
1079
|
+
try { body = await res.json() } catch { /* a 200 with no body still removed nothing or something */ }
|
|
1080
|
+
return { ok: true, removed: body?.removed === true, part: String(body?.part ?? partPathFor(remotePath)) }
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
// ---------------------------------------------------------------------------
|
|
1084
|
+
// shared bits
|
|
1085
|
+
// ---------------------------------------------------------------------------
|
|
1086
|
+
|
|
1087
|
+
/** The contract's 400 cases, checked here so no request is spent on a path the worker must reject. */
|
|
1088
|
+
function assertUsablePath(value, which) {
|
|
1089
|
+
if (typeof value !== 'string' || value === '') {
|
|
1090
|
+
throw new TransferError(`${which} path is required`)
|
|
1091
|
+
}
|
|
1092
|
+
if (value.includes('\u0000')) {
|
|
1093
|
+
throw new TransferError(`${which} path contains a NUL byte`)
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
function requestTimeout(opts) {
|
|
1098
|
+
return Number.isFinite(opts.timeoutMs) ? Number(opts.timeoutMs) : DEFAULT_TRANSFER_REQUEST_TIMEOUT_MS
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
/** One progress shape for both directions, so the renderer has one thing to understand. */
|
|
1102
|
+
function progress(direction, peerId, path, offset, total, attempts, phase) {
|
|
1103
|
+
const size = Number.isFinite(total) ? total : 0
|
|
1104
|
+
return {
|
|
1105
|
+
direction,
|
|
1106
|
+
peerId: peerLabel(peerId),
|
|
1107
|
+
path,
|
|
1108
|
+
offset,
|
|
1109
|
+
bytes: offset,
|
|
1110
|
+
total: size,
|
|
1111
|
+
percent: size > 0 ? Math.min(100, Math.round((offset / size) * 1000) / 10) : 0,
|
|
1112
|
+
attempts,
|
|
1113
|
+
phase,
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
/**
|
|
1118
|
+
* Combine the caller's cancellation with a wall-clock budget.
|
|
1119
|
+
*
|
|
1120
|
+
* WHY BOTH: `AbortSignal.any` in principle, but Node 20 support for it arrived late and a plugin
|
|
1121
|
+
* that only works on the newest runtime is a support burden. Two listeners plus a timer is four
|
|
1122
|
+
* lines and behaves identically. `unref` matters: a pending transfer must not keep the host process
|
|
1123
|
+
* alive after the operator quits DSH.
|
|
1124
|
+
*/
|
|
1125
|
+
function armTimeout(timeoutMs, outer) {
|
|
1126
|
+
const controller = new AbortController()
|
|
1127
|
+
const timer = setTimeout(() => controller.abort(new TransferError(`transfer request exceeded ${timeoutMs} ms`)), Math.max(1, timeoutMs))
|
|
1128
|
+
if (typeof timer.unref === 'function') timer.unref()
|
|
1129
|
+
const relay = () => controller.abort(outer?.reason)
|
|
1130
|
+
if (outer !== undefined && outer !== null) {
|
|
1131
|
+
if (outer.aborted === true) controller.abort(outer.reason)
|
|
1132
|
+
else outer.addEventListener('abort', relay, { once: true })
|
|
1133
|
+
}
|
|
1134
|
+
return {
|
|
1135
|
+
signal: controller.signal,
|
|
1136
|
+
disarm() {
|
|
1137
|
+
clearTimeout(timer)
|
|
1138
|
+
outer?.removeEventListener?.('abort', relay)
|
|
1139
|
+
},
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
/** Re-exported so a caller that needs one file's digest does not have to reimplement it. */
|
|
1144
|
+
export { sha256Of }
|