@hope_phenom/dsh-plugin-remote-use 0.0.0-stage → 0.2.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +207 -2
- package/README.zh.md +195 -0
- package/assets/icon.svg +47 -0
- package/cordis.patch.yml +41 -0
- package/lib/bridge/acp-bridge.js +905 -0
- package/lib/bridge/peer-connection.js +730 -0
- package/lib/client.js +2842 -0
- package/lib/core/acp-model-overlay.js +121 -0
- package/lib/core/approval-queue.js +216 -0
- package/lib/core/audit-log.js +322 -0
- package/lib/core/desktop-notify.js +173 -0
- package/lib/core/environment-prompt.js +76 -0
- package/lib/core/relay-supervisor.js +278 -0
- package/lib/core/review-engine.js +679 -0
- package/lib/core/sandbox-diagnosis.js +258 -0
- package/lib/core/sandbox-probe.js +310 -0
- package/lib/core/self-acp-spec.js +117 -0
- package/lib/core/selfcheck-runner.js +150 -0
- package/lib/core/settings-store.js +226 -0
- package/lib/core/shot-gc.js +334 -0
- package/lib/core/task-cwd.js +50 -0
- package/lib/core/task-memory.js +163 -0
- package/lib/core/task-registry.js +238 -0
- package/lib/core/toolcall-cache.js +206 -0
- package/lib/core/transfer-queue.js +58 -0
- package/lib/core/transfer.js +1144 -0
- package/lib/core/worker-arming.js +334 -0
- package/lib/index.js +2207 -0
- package/lib/provider/out-of-process-helpers.js +76 -0
- package/lib/provider/remote-provider.js +261 -0
- package/lib/tools/mediator-shim.mjs +180 -0
- package/lib/tools/prepare-sandbox-mediator.mjs +529 -0
- package/lib/tools/remote-tools.js +680 -0
- package/lib/tools/selfcheck-fix-prompt.md +206 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +72 -4
- package/slave/acp-profile-approval.patch.yml +66 -0
- package/slave/file-transfer.mjs +552 -0
- package/slave/integrity-probe.ps1 +99 -0
- package/slave/make-launch-spec.mjs +184 -0
- package/slave/net-address.mjs +138 -0
- package/slave/node_modules/.package-lock.json +26 -0
- package/slave/node_modules/ws/LICENSE +20 -0
- package/slave/node_modules/ws/README.md +548 -0
- package/slave/node_modules/ws/browser.js +8 -0
- package/slave/node_modules/ws/index.js +22 -0
- package/slave/node_modules/ws/lib/buffer-util.js +131 -0
- package/slave/node_modules/ws/lib/constants.js +19 -0
- package/slave/node_modules/ws/lib/event-target.js +292 -0
- package/slave/node_modules/ws/lib/extension.js +203 -0
- package/slave/node_modules/ws/lib/limiter.js +55 -0
- package/slave/node_modules/ws/lib/permessage-deflate.js +530 -0
- package/slave/node_modules/ws/lib/receiver.js +743 -0
- package/slave/node_modules/ws/lib/sender.js +607 -0
- package/slave/node_modules/ws/lib/stream.js +161 -0
- package/slave/node_modules/ws/lib/subprotocol.js +62 -0
- package/slave/node_modules/ws/lib/validation.js +152 -0
- package/slave/node_modules/ws/lib/websocket-server.js +562 -0
- package/slave/node_modules/ws/lib/websocket.js +1421 -0
- package/slave/node_modules/ws/package.json +74 -0
- package/slave/node_modules/ws/wrapper.mjs +21 -0
- package/slave/pairing.mjs +175 -0
- package/slave/permission-pairing.mjs +240 -0
- package/slave/prune-shots.mjs +170 -0
- package/slave/relay.mjs +1351 -0
- package/slave/repair-workspace-integrity.ps1 +75 -0
- package/slave/screenshot.ps1 +114 -0
- package/slave/selfcheck.ps1 +350 -0
- package/slave/stage-app.ps1 +116 -0
- package/slave/workspace-integrity.mjs +157 -0
|
@@ -0,0 +1,552 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The worker half of DSH's file transfer: path resolution, ranges, hashing and the `.part` dance.
|
|
3
|
+
*
|
|
4
|
+
* Kept in its own module for the same reason `prune-shots.mjs` is -- `relay.mjs` starts listening on
|
|
5
|
+
* a port the moment it is imported, so nothing inside it can be unit-tested. Every decision that can
|
|
6
|
+
* be wrong lives here, where a test can reach it.
|
|
7
|
+
*
|
|
8
|
+
* Node builtins only, like its siblings: this file is copied to a worker next to `relay.mjs` and must
|
|
9
|
+
* not drag anything else along.
|
|
10
|
+
*
|
|
11
|
+
* STREAMING IS NOT A STYLE PREFERENCE HERE. The bytes have no upper bound -- `--max-transfer-bytes`
|
|
12
|
+
* defaults to 8 GiB, and the whole point of the feature is moving files that are too large to be
|
|
13
|
+
* convenient. A single `readFileSync` on a request path therefore turns one 8 GiB copy into 8 GiB of
|
|
14
|
+
* resident memory plus an 8 GiB Buffer concatenation, which on a worker is an out-of-memory kill and
|
|
15
|
+
* a lost transfer. `GET /screenshot` reads a ~9.5 MB PNG into memory and that is precisely the pattern
|
|
16
|
+
* NOT repeated here: every read in this file is a stream, and every hash is fed chunk by chunk.
|
|
17
|
+
*
|
|
18
|
+
* The protocol this implements is frozen in `.research/transfer-protocol.md`; the relay owns the HTTP
|
|
19
|
+
* shape and this module owns the rules.
|
|
20
|
+
*/
|
|
21
|
+
import { createHash } from 'node:crypto'
|
|
22
|
+
import { createReadStream, createWriteStream } from 'node:fs'
|
|
23
|
+
import { open, readdir, rename, stat, unlink } from 'node:fs/promises'
|
|
24
|
+
import { dirname, isAbsolute, join, resolve } from 'node:path'
|
|
25
|
+
import { Transform } from 'node:stream'
|
|
26
|
+
import { pipeline } from 'node:stream/promises'
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* The worker's hard ceiling, in bytes.
|
|
30
|
+
*
|
|
31
|
+
* 8 GiB because that is also the master's default: the two defaults agreeing is what keeps a transfer
|
|
32
|
+
* that never touched either flag from failing at the last chunk. This side is the one that matters --
|
|
33
|
+
* a master's budget is a politeness, an operator's disk is not.
|
|
34
|
+
*/
|
|
35
|
+
export const DEFAULT_MAX_TRANSFER_BYTES = 8 * 1024 * 1024 * 1024
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Chunk size the master is expected to use, exported so both sides can name one number.
|
|
39
|
+
* 4 MiB balances syscall count against the size of the re-send when a chunk fails.
|
|
40
|
+
*/
|
|
41
|
+
export const TRANSFER_CHUNK_BYTES = 4 * 1024 * 1024
|
|
42
|
+
|
|
43
|
+
/** How many directory entries `list` returns when the caller does not say. */
|
|
44
|
+
export const DEFAULT_LIST_LIMIT = 1000
|
|
45
|
+
|
|
46
|
+
/** A commit request is a path plus a 64-character hash; anything near this is not one. */
|
|
47
|
+
export const DEFAULT_JSON_BODY_BYTES = 64 * 1024
|
|
48
|
+
|
|
49
|
+
/** The suffix a transfer lands under before it is allowed to take its real name. */
|
|
50
|
+
export const PART_SUFFIX = '.part'
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Read a byte ceiling, falling back to the default for anything unusable.
|
|
54
|
+
*
|
|
55
|
+
* WHY A NONSENSE VALUE KEEPS THE CEILING rather than removing it: `--max-transfer-bytes 0` reads as
|
|
56
|
+
* "no limit" to the person typing it and as "reject every byte" to the code, and both readings are
|
|
57
|
+
* worse than the default. `prune-shots.mjs` solved the same ambiguity for the shot cache by treating
|
|
58
|
+
* 0 as "no budget" -- that is the safe direction there (the failure mode is disk usage) and the
|
|
59
|
+
* UNSAFE direction here (the failure mode is a full disk on a machine nobody is sitting at). So an
|
|
60
|
+
* unusable value silently means 8 GiB, and the flag cannot be used to remove the ceiling by accident.
|
|
61
|
+
*
|
|
62
|
+
* @param {unknown} raw the flag's value, or a non-number for "not given"
|
|
63
|
+
* @returns {number} a positive byte count
|
|
64
|
+
*/
|
|
65
|
+
export function maxTransferBytesOf(raw) {
|
|
66
|
+
// Deliberately not `Number(raw)`: `'64'` is not a number anyone clearly meant, and `prune-shots.mjs`
|
|
67
|
+
// refuses `'20'` for the count budget for exactly this reason.
|
|
68
|
+
return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : DEFAULT_MAX_TRANSFER_BYTES
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Turn a request's `path` into the absolute path this worker will actually touch.
|
|
73
|
+
*
|
|
74
|
+
* Relative means "relative to the task directory" -- the relay's working directory, the one `/healthz`
|
|
75
|
+
* reports, which is the directory the operator started the relay in. Absolute is used as it stands.
|
|
76
|
+
*
|
|
77
|
+
* THERE IS NO CAGE, deliberately. A relay is not a sandbox: the same token that asks for a file can
|
|
78
|
+
* ask this process to run an arbitrary command through ACP, so refusing `D:\build` would block a real
|
|
79
|
+
* deployment while stopping nothing. What replaces the cage is that the resolved path is ECHOED in
|
|
80
|
+
* every response and every log line, so "which file was that, really" is always answerable.
|
|
81
|
+
*
|
|
82
|
+
* @param {unknown} rawPath
|
|
83
|
+
* @param {string} baseDir the task directory
|
|
84
|
+
* @returns {{ ok: true, path: string } | { ok: false, error: string }}
|
|
85
|
+
*/
|
|
86
|
+
export function resolveTransferPath(rawPath, baseDir) {
|
|
87
|
+
if (typeof rawPath !== 'string') return { ok: false, error: 'path is required' }
|
|
88
|
+
// A whitespace-only path is a truncated request, not a filename: Windows strips trailing spaces in
|
|
89
|
+
// some APIs and the result would be a file nobody asked for.
|
|
90
|
+
if (rawPath.trim() === '') return { ok: false, error: 'path is required' }
|
|
91
|
+
// NUL is the one byte that cannot survive the trip: it must be refused before it reaches the
|
|
92
|
+
// filesystem, where some APIs silently truncate at it and others throw something unhelpful.
|
|
93
|
+
if (rawPath.includes('\0')) return { ok: false, error: 'path contains a NUL byte' }
|
|
94
|
+
|
|
95
|
+
// A RELATIVE PATH REQUIRES A *DECLARED* TASK DIRECTORY -- never an inferred one.
|
|
96
|
+
//
|
|
97
|
+
// `null` means the relay was not told one. The old fallback was `process.cwd()`, and that is how a
|
|
98
|
+
// relative upload wrote into `...\.dsh\profiles\desktop\` on a supervised worker: the plugin starts
|
|
99
|
+
// the relay without a `cwd`, so the inherited directory is the DSH profile directory -- a place
|
|
100
|
+
// nothing should ever be written -- and the transfer reported success while doing it.
|
|
101
|
+
//
|
|
102
|
+
// An absolute path is unaffected: it names its own destination, so no base is needed. A relative one
|
|
103
|
+
// now fails loudly with both remedies named, which is the only honest answer when the alternative is
|
|
104
|
+
// guessing at a directory the operator never chose. A hand-started relay that WANTS relative paths
|
|
105
|
+
// says so once: `--task-dir .`.
|
|
106
|
+
const base = typeof baseDir === 'string' && baseDir !== '' ? baseDir : null
|
|
107
|
+
if (base === null && !isAbsolute(rawPath)) {
|
|
108
|
+
return {
|
|
109
|
+
ok: false,
|
|
110
|
+
error: `"${rawPath}" is a relative path and this relay has no task directory; `
|
|
111
|
+
+ 'send an absolute path, or start the relay with --task-dir <dir>',
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
// One `resolve` covers both cases: it ignores `base` when the input is absolute and normalises
|
|
115
|
+
// `.`/`..` either way, which is what makes the echoed path the path that is really used.
|
|
116
|
+
return { ok: true, path: resolve(base ?? process.cwd(), rawPath) }
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** The temporary name a transfer to `targetPath` lands under. */
|
|
120
|
+
export function partPathFor(targetPath) {
|
|
121
|
+
return `${targetPath}${PART_SUFFIX}`
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* A byte offset, from a URL query value or from a caller.
|
|
126
|
+
*
|
|
127
|
+
* The shape is checked rather than coerced, and that is not pedantry: `Number([])` is 0, `Number(true)`
|
|
128
|
+
* is 1 and `Number(' ')` is 0, so a coerced offset turns "that is not a byte position" into "resume
|
|
129
|
+
* from the beginning" -- a silent restart of a multi-gigabyte transfer instead of a 400.
|
|
130
|
+
*
|
|
131
|
+
* @param {unknown} raw
|
|
132
|
+
* @returns {{ ok: true, offset: number } | { ok: false, error: string }}
|
|
133
|
+
*/
|
|
134
|
+
function offsetOf(raw) {
|
|
135
|
+
if (raw === undefined || raw === null || raw === '') return { ok: true, offset: 0 }
|
|
136
|
+
const error = { ok: false, error: `offset must be a non-negative integer (got ${JSON.stringify(raw)})` }
|
|
137
|
+
if (typeof raw === 'string') {
|
|
138
|
+
// Only digits: `'1e3'` and `'0x10'` are not offsets anyone typed on purpose.
|
|
139
|
+
if (!/^[0-9]+$/u.test(raw)) return error
|
|
140
|
+
return { ok: true, offset: Number(raw) }
|
|
141
|
+
}
|
|
142
|
+
if (typeof raw === 'number' && Number.isInteger(raw) && raw >= 0) return { ok: true, offset: raw }
|
|
143
|
+
return error
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Decide what a `GET /file` for a `size`-byte file at `offset` must answer.
|
|
148
|
+
*
|
|
149
|
+
* The empty-file case is the one worth stating: `offset=0` on a 0-byte file is `200` with an empty
|
|
150
|
+
* body, not `416`. The protocol says "offset >= size -> 416", which read literally would make an
|
|
151
|
+
* empty file undownloadable -- and an empty file is a perfectly ordinary thing to transfer.
|
|
152
|
+
*
|
|
153
|
+
* @param {number} size
|
|
154
|
+
* @param {unknown} offset raw query value; absent/'' means 0
|
|
155
|
+
* @returns {{ ok: true, status: 200 | 206, offset: number, length: number, contentRange?: string }
|
|
156
|
+
* | { ok: false, status: 400 | 416, error: string }}
|
|
157
|
+
*/
|
|
158
|
+
export function rangeOf(size, offset = 0) {
|
|
159
|
+
const total = Number.isFinite(size) && size > 0 ? Math.floor(size) : 0
|
|
160
|
+
const start = offsetOf(offset)
|
|
161
|
+
// A fractional or negative offset is a caller bug; 416 would say "past the end", which is a
|
|
162
|
+
// different and misleading answer.
|
|
163
|
+
if (!start.ok) return { ok: false, status: 400, error: start.error }
|
|
164
|
+
if (start.offset === 0) return { ok: true, status: 200, offset: 0, length: total }
|
|
165
|
+
if (start.offset >= total) {
|
|
166
|
+
return { ok: false, status: 416, error: `offset ${start.offset} is at or past the end of a ${total}-byte file` }
|
|
167
|
+
}
|
|
168
|
+
return {
|
|
169
|
+
ok: true,
|
|
170
|
+
status: 206,
|
|
171
|
+
offset: start.offset,
|
|
172
|
+
length: total - start.offset,
|
|
173
|
+
// The protocol's exact shape, inclusive last byte.
|
|
174
|
+
contentRange: `bytes ${start.offset}-${total - 1}/${total}`,
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Hash a file in one streaming pass.
|
|
180
|
+
*
|
|
181
|
+
* The size comes back with the hash because `commit` has to compare both and a second `stat` on a
|
|
182
|
+
* file that is being written would be a race for no gain.
|
|
183
|
+
*
|
|
184
|
+
* @param {string} path
|
|
185
|
+
* @returns {Promise<{ sha256: string, bytes: number }>}
|
|
186
|
+
*/
|
|
187
|
+
export async function sha256File(path) {
|
|
188
|
+
const hash = createHash('sha256')
|
|
189
|
+
let bytes = 0
|
|
190
|
+
// `for await` pulls one chunk at a time and lets the stream apply backpressure, which is the whole
|
|
191
|
+
// difference between this and `readFileSync`: resident memory stays at one chunk, whatever the file.
|
|
192
|
+
for await (const chunk of createReadStream(path)) {
|
|
193
|
+
hash.update(chunk)
|
|
194
|
+
bytes += chunk.length
|
|
195
|
+
}
|
|
196
|
+
return { sha256: hash.digest('hex'), bytes }
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** `stat`, with "not there" told apart from "cannot be asked". */
|
|
200
|
+
async function statOrNull(path) {
|
|
201
|
+
try {
|
|
202
|
+
return await stat(path)
|
|
203
|
+
} catch (error) {
|
|
204
|
+
// ENOENT is an answer. EACCES is not: reporting a permission-denied file as "missing" is how a
|
|
205
|
+
// transfer silently retries forever against a path the process may never read.
|
|
206
|
+
if (error.code === 'ENOENT' || error.code === 'ENOTDIR') return null
|
|
207
|
+
throw error
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* What `dsh-remote/stat` reports about a path.
|
|
213
|
+
*
|
|
214
|
+
* @param {string} targetPath already resolved and absolute
|
|
215
|
+
* @param {{ hash?: boolean }} [opts] `hash: true` also computes the sha256 (expensive, opt-in)
|
|
216
|
+
* @returns {Promise<{ exists: boolean, isDir: boolean, size: number, mtime: string | null, sha256?: string }>}
|
|
217
|
+
*/
|
|
218
|
+
export async function statTarget(targetPath, opts = {}) {
|
|
219
|
+
const info = await statOrNull(targetPath)
|
|
220
|
+
// `mtime: null` rather than '' for a path that is not there: both are falsy, but `null` says "this
|
|
221
|
+
// field has no value" instead of "this file has an empty timestamp".
|
|
222
|
+
if (info === null) return { exists: false, isDir: false, size: 0, mtime: null }
|
|
223
|
+
const isDir = info.isDirectory()
|
|
224
|
+
const out = { exists: true, isDir, size: isDir ? 0 : info.size, mtime: info.mtime.toISOString() }
|
|
225
|
+
// Not computed unless asked for: hashing reads every byte of the file, and `stat` is the call a
|
|
226
|
+
// master makes before it decides whether to transfer anything at all.
|
|
227
|
+
if (opts.hash === true && !isDir) out.sha256 = (await sha256File(targetPath)).sha256
|
|
228
|
+
return out
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* List a directory.
|
|
233
|
+
*
|
|
234
|
+
* SORTED BY NAME, because the alternative is whatever order the filesystem hands back -- which
|
|
235
|
+
* differs between NTFS and ReFS and between two runs, so two `list` calls that returned the same
|
|
236
|
+
* entries would look like different answers. The master can re-sort; it cannot recover determinism.
|
|
237
|
+
*
|
|
238
|
+
* @param {string} targetPath
|
|
239
|
+
* @param {{ limit?: number }} [opts]
|
|
240
|
+
* @returns {Promise<{ ok: true, entries: object[], truncated: boolean } | { ok: false, status: number, error: string }>}
|
|
241
|
+
*/
|
|
242
|
+
export async function listDir(targetPath, opts = {}) {
|
|
243
|
+
const limit = Number.isFinite(opts.limit) && opts.limit >= 0 ? Math.floor(opts.limit) : DEFAULT_LIST_LIMIT
|
|
244
|
+
let dirents
|
|
245
|
+
try {
|
|
246
|
+
dirents = await readdir(targetPath, { withFileTypes: true })
|
|
247
|
+
} catch (error) {
|
|
248
|
+
if (error.code === 'ENOENT' || error.code === 'ENOTDIR') {
|
|
249
|
+
return { ok: false, status: 404, error: `${targetPath} is not a directory` }
|
|
250
|
+
}
|
|
251
|
+
throw error
|
|
252
|
+
}
|
|
253
|
+
dirents.sort((a, b) => a.name.localeCompare(b.name))
|
|
254
|
+
|
|
255
|
+
const entries = []
|
|
256
|
+
let truncated = false
|
|
257
|
+
for (const dirent of dirents) {
|
|
258
|
+
if (entries.length >= limit) { truncated = true; break }
|
|
259
|
+
try {
|
|
260
|
+
// `stat` rather than the dirent's own flags: a symlink to a directory is a directory for every
|
|
261
|
+
// purpose the master has, and the dirent would call it a symlink.
|
|
262
|
+
const info = await stat(join(targetPath, dirent.name))
|
|
263
|
+
const isDir = info.isDirectory()
|
|
264
|
+
entries.push({
|
|
265
|
+
name: dirent.name,
|
|
266
|
+
isDir,
|
|
267
|
+
size: isDir ? 0 : info.size,
|
|
268
|
+
mtime: info.mtime.toISOString(),
|
|
269
|
+
})
|
|
270
|
+
} catch { /* vanished between listing and stat: an entry that no longer exists is not an entry */ }
|
|
271
|
+
}
|
|
272
|
+
return { ok: true, entries, truncated }
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/** The `{ ok, status, json }` shape every rule below answers with, so the relay can just write it. */
|
|
276
|
+
function refusal(status, json) {
|
|
277
|
+
return { ok: false, status, json }
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Append one chunk of an upload to `<target>.part`.
|
|
282
|
+
*
|
|
283
|
+
* THE ORDER OF THE CHECKS IS THE POINT, and the two interesting ones are both refusals:
|
|
284
|
+
*
|
|
285
|
+
* * `413` BEFORE `409`: a chunk that would push the file over the ceiling is fatal to the whole
|
|
286
|
+
* transfer, while an offset mismatch is merely the master being out of step. Answering `409`
|
|
287
|
+
* first would make the master re-align, re-send, and only then be told the transfer cannot
|
|
288
|
+
* finish -- after moving gigabytes that were never going to be accepted.
|
|
289
|
+
* * `409` carries `expectedOffset`, which is the size of `.part` as it stands. That is the number
|
|
290
|
+
* the master needs to resume from, and it is the ONLY correct one: after an interrupted chunk the
|
|
291
|
+
* part file is neither the old size nor the requested offset.
|
|
292
|
+
*
|
|
293
|
+
* An interrupted body keeps `.part` on purpose. Those bytes are the resume point; deleting them would
|
|
294
|
+
* turn a dropped connection into a transfer that restarts from zero.
|
|
295
|
+
*
|
|
296
|
+
* @param {{ target: string, offset?: unknown, contentLength?: unknown, body: AsyncIterable<Buffer>,
|
|
297
|
+
* maxBytes?: number }} options
|
|
298
|
+
* @returns {Promise<{ ok: boolean, status: number, json: object, aborted?: boolean, received?: number }>}
|
|
299
|
+
*/
|
|
300
|
+
export async function putChunk(options) {
|
|
301
|
+
const { target, body } = options
|
|
302
|
+
const maxBytes = maxTransferBytesOf(options.maxBytes)
|
|
303
|
+
const part = partPathFor(target)
|
|
304
|
+
|
|
305
|
+
const start = offsetOf(options.offset ?? 0)
|
|
306
|
+
// Non-integer offsets are caller bugs, and the difference matters: 409 would tell a master to
|
|
307
|
+
// resume at a position it never asked for.
|
|
308
|
+
if (!start.ok) return refusal(400, { ok: false, error: start.error })
|
|
309
|
+
|
|
310
|
+
// 411 rather than 400: the protocol makes the differing header the identifying feature of "this
|
|
311
|
+
// request cannot be handled", and a chunked PUT has no length to check against the ceiling before
|
|
312
|
+
// the bytes arrive -- which is the same reason a proxy has to reject it. `null` and `''` are
|
|
313
|
+
// refused as well as `undefined`: `Number(null)` is 0, which would otherwise read as a legal
|
|
314
|
+
// zero-length chunk and let an unbounded body through.
|
|
315
|
+
const declared = options.contentLength
|
|
316
|
+
const length = declared === undefined || declared === null || declared === ''
|
|
317
|
+
? NaN
|
|
318
|
+
: Number(declared)
|
|
319
|
+
if (!Number.isFinite(length) || length < 0) {
|
|
320
|
+
return refusal(411, { ok: false, error: 'PUT /file requires a content-length; a chunked upload cannot be bounded before it is written' })
|
|
321
|
+
}
|
|
322
|
+
if (start.offset + length > maxBytes) {
|
|
323
|
+
return refusal(413, {
|
|
324
|
+
ok: false,
|
|
325
|
+
error: `this chunk would make the file ${start.offset + length} bytes, over the ${maxBytes}-byte ceiling (--max-transfer-bytes)`,
|
|
326
|
+
limit: maxBytes,
|
|
327
|
+
})
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
const partInfo = await statOrNull(part)
|
|
331
|
+
const partBytes = partInfo === null ? 0 : partInfo.size
|
|
332
|
+
if (partBytes !== start.offset) {
|
|
333
|
+
return refusal(409, {
|
|
334
|
+
ok: false,
|
|
335
|
+
error: `.part is ${partBytes} bytes but this chunk starts at ${start.offset}`,
|
|
336
|
+
expectedOffset: partBytes,
|
|
337
|
+
})
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
// Create-if-missing without truncating: `'a'` extends nothing and destroys nothing, and a second
|
|
341
|
+
// PUT to the same path must never silently discard the first chunk.
|
|
342
|
+
// A missing PARENT is the caller's mistake, not the server's: answering 500 tells the master the
|
|
343
|
+
// worker broke, and a 5xx invites a retry of a condition that will never change. 409 names the
|
|
344
|
+
// directory instead, which is the actionable half.
|
|
345
|
+
//
|
|
346
|
+
// Deliberately NOT creating it: the destination FILE is named by the caller, its directory is not,
|
|
347
|
+
// and silently materialising `D:\build` is not a side effect a file transfer should take on itself.
|
|
348
|
+
const parentDir = await statOrNull(dirname(part))
|
|
349
|
+
if (parentDir === null || !parentDir.isDirectory()) {
|
|
350
|
+
return refusal(409, {
|
|
351
|
+
ok: false,
|
|
352
|
+
error: `the directory ${dirname(part)} does not exist on the worker; create it there first, or send into a directory that already exists`,
|
|
353
|
+
})
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
const created = await open(part, 'a')
|
|
357
|
+
await created.close()
|
|
358
|
+
|
|
359
|
+
let received = 0
|
|
360
|
+
const counter = new Transform({
|
|
361
|
+
transform(chunk, _encoding, callback) {
|
|
362
|
+
received += chunk.length
|
|
363
|
+
callback(null, chunk)
|
|
364
|
+
},
|
|
365
|
+
})
|
|
366
|
+
try {
|
|
367
|
+
// `start: start.offset` means the write goes exactly where the master asked, not wherever the file
|
|
368
|
+
// happens to end -- the two are equal here because of the check above, and saying so explicitly
|
|
369
|
+
// is what keeps them equal if that check ever moves.
|
|
370
|
+
await pipeline(body, counter, createWriteStream(part, { flags: 'r+', start: start.offset }))
|
|
371
|
+
} catch (error) {
|
|
372
|
+
// The client is gone or the disk refused. Either way this is not a 200, but `.part` stays.
|
|
373
|
+
return {
|
|
374
|
+
ok: false,
|
|
375
|
+
status: 499, // nginx's "client closed request"; the client that would read it is gone
|
|
376
|
+
aborted: true,
|
|
377
|
+
received,
|
|
378
|
+
json: {
|
|
379
|
+
ok: false,
|
|
380
|
+
error: `upload interrupted after ${received} bytes: ${error.message}`,
|
|
381
|
+
part,
|
|
382
|
+
offset: start.offset,
|
|
383
|
+
partBytes: start.offset + received,
|
|
384
|
+
},
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
return {
|
|
389
|
+
ok: true,
|
|
390
|
+
status: 200,
|
|
391
|
+
received,
|
|
392
|
+
json: { ok: true, path: target, part, offset: start.offset, received, partBytes: start.offset + received },
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* Verify a completed `.part` and give it its real name.
|
|
398
|
+
*
|
|
399
|
+
* The hash is recomputed HERE, over the bytes that are actually on the disk, rather than trusted from
|
|
400
|
+
* the upload. That recomputation is the only evidence in the protocol that the file that arrived is
|
|
401
|
+
* the file that was sent -- a master's own hash proves what it meant to send, not what this disk
|
|
402
|
+
* received.
|
|
403
|
+
*
|
|
404
|
+
* A mismatch keeps `.part`. That is not tidiness: the same bytes are the resume point for a retry, and
|
|
405
|
+
* deleting them would make every failed hash a full re-upload.
|
|
406
|
+
*
|
|
407
|
+
* @param {{ target: string, sha256?: unknown, size?: unknown, overwrite?: boolean }} options
|
|
408
|
+
* @returns {Promise<{ ok: boolean, status: number, json: object }>}
|
|
409
|
+
*/
|
|
410
|
+
export async function commitPart(options) {
|
|
411
|
+
const { target } = options
|
|
412
|
+
const part = partPathFor(target)
|
|
413
|
+
|
|
414
|
+
const wanted = typeof options.sha256 === 'string' ? options.sha256.trim().toLowerCase() : ''
|
|
415
|
+
if (wanted === '') {
|
|
416
|
+
return refusal(400, { ok: false, error: 'sha256 is required: without it there is no evidence that the bytes on disk are the requested file' })
|
|
417
|
+
}
|
|
418
|
+
const wantedSize = Number.isFinite(options.size) ? Number(options.size) : undefined
|
|
419
|
+
|
|
420
|
+
const partInfo = await statOrNull(part)
|
|
421
|
+
if (partInfo === null || !partInfo.isFile()) {
|
|
422
|
+
// 404, not 400: the request is well-formed, the thing it refers to is not there. A commit is
|
|
423
|
+
// always the last step of an upload, so this is nearly always "the transfer never completed".
|
|
424
|
+
return refusal(404, { ok: false, error: `no ${PART_SUFFIX} file to commit at ${part}`, part })
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
const actual = await sha256File(part)
|
|
428
|
+
// The HASH is checked first and is what `expected`/`actual` carry, because that is the comparison
|
|
429
|
+
// the protocol calls "the only evidence the file is the one that was sent". The size, when the
|
|
430
|
+
// request gave one, rides along in the message: a truncated `.part` is the common cause of a
|
|
431
|
+
// mismatched digest, and "sha256 differs" alone sends the reader hunting for corruption that is
|
|
432
|
+
// not there.
|
|
433
|
+
if (actual.sha256 !== wanted) {
|
|
434
|
+
const sizeNote = wantedSize !== undefined && wantedSize !== actual.bytes
|
|
435
|
+
? ` (the request says ${wantedSize} bytes, ${part} holds ${actual.bytes})`
|
|
436
|
+
: ''
|
|
437
|
+
return refusal(400, {
|
|
438
|
+
ok: false,
|
|
439
|
+
error: `sha256 mismatch${sizeNote}; the partial file is unchanged and can be retried`,
|
|
440
|
+
expected: wanted,
|
|
441
|
+
actual: actual.sha256,
|
|
442
|
+
})
|
|
443
|
+
}
|
|
444
|
+
// Reached only when the digests agree, which two different lengths cannot do -- so this is a guard
|
|
445
|
+
// against that assumption rather than a live path. It costs one comparison that has already been
|
|
446
|
+
// paid for, and it keeps `size` meaning what a caller who sends it expects it to mean.
|
|
447
|
+
if (wantedSize !== undefined && actual.bytes !== wantedSize) {
|
|
448
|
+
return refusal(400, {
|
|
449
|
+
ok: false,
|
|
450
|
+
error: `size mismatch: the request says ${wantedSize} bytes, ${part} holds ${actual.bytes}`,
|
|
451
|
+
expected: wantedSize,
|
|
452
|
+
actual: actual.bytes,
|
|
453
|
+
})
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
const existing = await statOrNull(target)
|
|
457
|
+
if (existing !== null) {
|
|
458
|
+
// A directory can never be the target of a rename, whatever `overwrite` says.
|
|
459
|
+
if (existing.isDirectory()) {
|
|
460
|
+
return refusal(409, { ok: false, error: `${target} is a directory`, path: target })
|
|
461
|
+
}
|
|
462
|
+
if (options.overwrite !== true) {
|
|
463
|
+
return refusal(409, {
|
|
464
|
+
ok: false,
|
|
465
|
+
error: `${target} already exists; send overwrite: true to replace it`,
|
|
466
|
+
path: target,
|
|
467
|
+
})
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// fsync BEFORE the rename. Without it a crash between the rename and the writeback leaves the
|
|
472
|
+
// correct name on a file whose contents are still in the page cache -- the one state the `.part`
|
|
473
|
+
// scheme exists to make impossible.
|
|
474
|
+
const handle = await open(part, 'r+')
|
|
475
|
+
try {
|
|
476
|
+
await handle.sync()
|
|
477
|
+
} finally {
|
|
478
|
+
await handle.close()
|
|
479
|
+
}
|
|
480
|
+
// Atomic on both platforms (libuv asks Windows for MOVEFILE_REPLACE_EXISTING), and the check above
|
|
481
|
+
// is the last thing before it: this relay is the only writer of these paths by contract, and a
|
|
482
|
+
// rename is the only way to publish a complete file under its real name.
|
|
483
|
+
await rename(part, target)
|
|
484
|
+
|
|
485
|
+
const published = await stat(target)
|
|
486
|
+
return {
|
|
487
|
+
ok: true,
|
|
488
|
+
status: 200,
|
|
489
|
+
json: { ok: true, path: target, bytes: published.size, sha256: wanted, mtime: published.mtime.toISOString() },
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
/**
|
|
494
|
+
* Remove a transfer's `.part` and nothing else.
|
|
495
|
+
*
|
|
496
|
+
* THE TARGET IS NEVER TOUCHED. This is the master saying "I am abandoning this transfer", and a
|
|
497
|
+
* cleanup that could reach the finished file would turn an abandoned upload into deleted data -- the
|
|
498
|
+
* exact failure the `.part` scheme exists to prevent. `removed: false` is a success: the master wants
|
|
499
|
+
* the part gone, and it is gone.
|
|
500
|
+
*
|
|
501
|
+
* @param {string} target
|
|
502
|
+
* @returns {Promise<{ ok: true, status: 200, json: object }>}
|
|
503
|
+
*/
|
|
504
|
+
export async function deletePart(target) {
|
|
505
|
+
const part = partPathFor(target)
|
|
506
|
+
try {
|
|
507
|
+
await unlink(part)
|
|
508
|
+
return { ok: true, status: 200, json: { ok: true, removed: true, part } }
|
|
509
|
+
} catch (error) {
|
|
510
|
+
if (error.code === 'ENOENT') return { ok: true, status: 200, json: { ok: true, removed: false, part } }
|
|
511
|
+
throw error
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
/**
|
|
516
|
+
* Read and parse a small JSON request body, with a ceiling.
|
|
517
|
+
*
|
|
518
|
+
* Buffered on purpose, and the ceiling is why: the bodies this serves are a path and a hash, so
|
|
519
|
+
* keeping one in memory is free, while an unbounded one is a way for anything holding the token to
|
|
520
|
+
* make the relay allocate until it dies. The limit is checked as the body arrives, not afterwards --
|
|
521
|
+
* a `content-length` check alone would trust a header.
|
|
522
|
+
*
|
|
523
|
+
* @param {AsyncIterable<Buffer>} body
|
|
524
|
+
* @param {{ limitBytes?: number }} [opts]
|
|
525
|
+
* @returns {Promise<{ ok: true, value: object } | { ok: false, status: number, json: object }>}
|
|
526
|
+
*/
|
|
527
|
+
export async function readJsonBody(body, opts = {}) {
|
|
528
|
+
const limit = Number.isFinite(opts.limitBytes) && opts.limitBytes > 0 ? Math.floor(opts.limitBytes) : DEFAULT_JSON_BODY_BYTES
|
|
529
|
+
let text = ''
|
|
530
|
+
let bytes = 0
|
|
531
|
+
for await (const chunk of body) {
|
|
532
|
+
bytes += chunk.length
|
|
533
|
+
if (bytes > limit) {
|
|
534
|
+
return refusal(413, { ok: false, error: `request body is larger than ${limit} bytes`, limit })
|
|
535
|
+
}
|
|
536
|
+
text += chunk.toString('utf8')
|
|
537
|
+
}
|
|
538
|
+
if (text.trim() === '') return refusal(400, { ok: false, error: 'a JSON body is required' })
|
|
539
|
+
|
|
540
|
+
let value
|
|
541
|
+
try {
|
|
542
|
+
value = JSON.parse(text)
|
|
543
|
+
} catch (error) {
|
|
544
|
+
return refusal(400, { ok: false, error: `body is not valid JSON: ${error.message}` })
|
|
545
|
+
}
|
|
546
|
+
// An array or a bare scalar parses fine and then has no `.path`, which would surface as a confusing
|
|
547
|
+
// "path is required" instead of "that is not a request object".
|
|
548
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
|
549
|
+
return refusal(400, { ok: false, error: 'body must be a JSON object' })
|
|
550
|
+
}
|
|
551
|
+
return { ok: true, value }
|
|
552
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# integrity-probe.ps1 -- report the token integrity a process gets when launched from a workspace.
|
|
2
|
+
#
|
|
3
|
+
# WHY THIS EXISTS: DSH's workspace-write sandbox writes an inheritable Low mandatory label onto the
|
|
4
|
+
# workspace directory. Files copied there inherit it, and Windows launches a Low-labelled image at
|
|
5
|
+
# Low integrity even when the caller's token is High. A GUI application then cannot write
|
|
6
|
+
# %LOCALAPPDATA% (Medium) while the agent shell around it can -- a failure that looks like an ACL or
|
|
7
|
+
# application problem, not like a sandbox leftover (PITFALLS F41).
|
|
8
|
+
#
|
|
9
|
+
# The probe copies a signed Windows binary (whoami.exe) into the workspace and into %TEMP%, launches
|
|
10
|
+
# each copy, and reads the mandatory-label row out of `whoami /groups`. That measures the actual
|
|
11
|
+
# result for the same kind of image, without needing a GUI app or a real GUI session.
|
|
12
|
+
#
|
|
13
|
+
# Output is ONE line of compact JSON on stdout; diagnostics go to stderr. ASCII-only: Windows
|
|
14
|
+
# PowerShell 5.1 reads BOM-less files as ANSI.
|
|
15
|
+
param(
|
|
16
|
+
[string]$Workspace = ''
|
|
17
|
+
)
|
|
18
|
+
$ErrorActionPreference = 'Stop'
|
|
19
|
+
|
|
20
|
+
function Get-LevelNumber([string]$Sid) {
|
|
21
|
+
switch ($Sid) {
|
|
22
|
+
'S-1-16-0' { return 0 }
|
|
23
|
+
'S-1-16-4096' { return 4096 }
|
|
24
|
+
'S-1-16-8192' { return 8192 }
|
|
25
|
+
'S-1-16-12288' { return 12288 }
|
|
26
|
+
'S-1-16-16384' { return 16384 }
|
|
27
|
+
default { return -1 }
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function Get-LevelName([int]$Number) {
|
|
32
|
+
switch ($Number) {
|
|
33
|
+
0 { return 'Untrusted' }
|
|
34
|
+
4096 { return 'Low' }
|
|
35
|
+
8192 { return 'Medium' }
|
|
36
|
+
12288 { return 'High' }
|
|
37
|
+
16384 { return 'System' }
|
|
38
|
+
default { return 'unknown' }
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function Read-Integrity([string]$ExePath) {
|
|
43
|
+
if ([string]::IsNullOrEmpty($ExePath) -or -not (Test-Path -LiteralPath $ExePath)) { return $null }
|
|
44
|
+
$rows = @(& $ExePath /groups /fo csv /nh 2>$null)
|
|
45
|
+
if ($rows.Count -eq 0) { return $null }
|
|
46
|
+
$parsed = @($rows | ConvertFrom-Csv -Header Name, Kind, Sid, Attributes)
|
|
47
|
+
$label = $parsed | Where-Object { $_.Sid -match '^S-1-16-' } | Select-Object -First 1
|
|
48
|
+
if ($null -eq $label) { return $null }
|
|
49
|
+
$number = Get-LevelNumber $label.Sid
|
|
50
|
+
return [ordered]@{
|
|
51
|
+
sid = $label.Sid
|
|
52
|
+
name = $label.Name
|
|
53
|
+
level = Get-LevelName $number
|
|
54
|
+
levelNumber = $number
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
$result = [ordered]@{
|
|
59
|
+
probedAt = (Get-Date).ToString('o')
|
|
60
|
+
workspacePath = $Workspace
|
|
61
|
+
self = $null
|
|
62
|
+
workspace = $null
|
|
63
|
+
temp = $null
|
|
64
|
+
degraded = $false
|
|
65
|
+
error = ''
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
$whoamiExe = Join-Path $env:SystemRoot 'System32\whoami.exe'
|
|
69
|
+
try {
|
|
70
|
+
if (-not (Test-Path -LiteralPath $whoamiExe)) { throw "whoami.exe not found at $whoamiExe" }
|
|
71
|
+
# The current process's token, measured through a fresh child rather than assumed from $PID.
|
|
72
|
+
$result.self = Read-Integrity $whoamiExe
|
|
73
|
+
|
|
74
|
+
$tempCopy = Join-Path ([System.IO.Path]::GetTempPath()) ("dsh-il-probe-temp-" + $PID + "-" + [guid]::NewGuid().ToString('N') + ".exe")
|
|
75
|
+
try {
|
|
76
|
+
Copy-Item -LiteralPath $whoamiExe -Destination $tempCopy -Force
|
|
77
|
+
$result.temp = Read-Integrity $tempCopy
|
|
78
|
+
} finally {
|
|
79
|
+
Remove-Item -LiteralPath $tempCopy -Force -ErrorAction SilentlyContinue
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if (-not [string]::IsNullOrEmpty($Workspace) -and (Test-Path -LiteralPath $Workspace -PathType Container)) {
|
|
83
|
+
$workspaceCopy = Join-Path $Workspace ("dsh-il-probe-workspace-" + $PID + "-" + [guid]::NewGuid().ToString('N') + ".exe")
|
|
84
|
+
try {
|
|
85
|
+
Copy-Item -LiteralPath $whoamiExe -Destination $workspaceCopy -Force
|
|
86
|
+
$result.workspace = Read-Integrity $workspaceCopy
|
|
87
|
+
} finally {
|
|
88
|
+
Remove-Item -LiteralPath $workspaceCopy -Force -ErrorAction SilentlyContinue
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
if ($null -ne $result.workspace -and $null -ne $result.temp) {
|
|
93
|
+
$result.degraded = [bool]($result.workspace.levelNumber -lt $result.temp.levelNumber)
|
|
94
|
+
}
|
|
95
|
+
} catch {
|
|
96
|
+
$result.error = $_.Exception.Message
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
$result | ConvertTo-Json -Compress -Depth 6
|