@vzn/vx 0.0.16 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -89
- package/index.ts +10 -0
- package/launcher.mjs +10 -11
- package/package.json +19 -5
- package/plugins/local-cache/index.ts +2 -0
- package/plugins/local-executor/index.ts +2 -0
- package/plugins/schedule-history/index.ts +2 -0
- package/src/bin.ts +2 -2
- package/src/cache/archive.ts +557 -0
- package/src/cache/cache.ts +929 -250
- package/src/cache/cas-backend.ts +20 -2
- package/src/cache/chained-cache.ts +202 -0
- package/src/cache/digest.ts +0 -15
- package/src/cache/index.ts +13 -3
- package/src/cache/inputs.ts +445 -41
- package/src/cache/layered-cache.ts +88 -30
- package/src/cache/tar-stream.ts +358 -0
- package/src/cli/cache.ts +36 -10
- package/src/cli/help.ts +167 -128
- package/src/cli/index.ts +74 -43
- package/src/cli/info.ts +30 -0
- package/src/cli/last.ts +140 -0
- package/src/cli/migrate-emit.ts +21 -0
- package/src/cli/migrate-nx.ts +23 -0
- package/src/cli/migrate-scripts.ts +127 -0
- package/src/cli/migrate-turbo.ts +43 -6
- package/src/cli/migrate.ts +119 -33
- package/src/cli/plan-format.ts +58 -3
- package/src/cli/plugin-commands.ts +96 -0
- package/src/cli/prune.ts +291 -0
- package/src/cli/run.ts +272 -80
- package/src/cli/upgrade.ts +7 -6
- package/src/cli/watch.ts +259 -61
- package/src/cli/why.ts +193 -0
- package/src/config.ts +40 -23
- package/src/exec/env.ts +7 -1
- package/src/exec/executor.ts +239 -0
- package/src/exec/index.ts +19 -1
- package/src/exec/runner.ts +116 -29
- package/src/exec/sandbox-runtime.ts +194 -24
- package/src/graph/index.ts +3 -1
- package/src/graph/scheduler.ts +92 -18
- package/src/graph/task-graph.ts +220 -6
- package/src/index.ts +82 -84
- package/src/orchestrator/deferred-outputs.ts +144 -0
- package/src/orchestrator/download-policy.ts +171 -0
- package/src/orchestrator/events.ts +30 -0
- package/src/orchestrator/execute-task.ts +328 -94
- package/src/orchestrator/failure-mode.ts +61 -0
- package/src/orchestrator/framed-output.ts +87 -35
- package/src/orchestrator/history.ts +78 -31
- package/src/orchestrator/index.ts +43 -90
- package/src/orchestrator/local-shortcircuit.ts +22 -0
- package/src/orchestrator/logger.ts +196 -14
- package/src/orchestrator/metrics.ts +57 -1376
- package/src/orchestrator/missing-plugin.ts +9 -0
- package/src/orchestrator/options.ts +22 -4
- package/src/orchestrator/plan.ts +144 -1
- package/src/orchestrator/plugin-host.ts +199 -119
- package/src/orchestrator/plugin.ts +144 -57
- package/src/orchestrator/prepare.ts +220 -69
- package/src/orchestrator/remote-prefetch.ts +12 -8
- package/src/orchestrator/resources.ts +20 -34
- package/src/orchestrator/run-artifacts.ts +41 -16
- package/src/orchestrator/run-context.ts +167 -29
- package/src/orchestrator/run-report.ts +73 -46
- package/src/orchestrator/run.ts +590 -122
- package/src/orchestrator/stable-keys.ts +74 -33
- package/src/orchestrator/status-line.ts +47 -8
- package/src/orchestrator/summary.ts +22 -1
- package/src/orchestrator/tally.ts +78 -35
- package/src/orchestrator/task-hash.ts +105 -11
- package/src/orchestrator/task-log-buffer.ts +291 -0
- package/src/orchestrator/telemetry-host.ts +24 -6
- package/src/orchestrator/telemetry.ts +181 -21
- package/src/orchestrator/upstream.ts +72 -31
- package/src/orchestrator/verify.ts +12 -2
- package/src/plugins/local-cache/index.ts +11 -0
- package/src/plugins/local-executor/index.ts +45 -0
- package/src/plugins/schedule-history/index.ts +109 -0
- package/src/util/edit-distance.ts +21 -0
- package/src/util/errors.ts +15 -1
- package/src/util/index.ts +7 -2
- package/src/util/num.ts +37 -0
- package/src/util/paths.ts +43 -0
- package/src/util/settle.ts +48 -0
- package/src/util/size.ts +5 -1
- package/src/util/tail.ts +66 -0
- package/src/util/timing.ts +63 -0
- package/src/workspace/affected.ts +142 -15
- package/src/workspace/config-cache.ts +304 -0
- package/src/workspace/config-eval.ts +210 -0
- package/src/workspace/config-imports.ts +161 -0
- package/src/workspace/filter.ts +22 -4
- package/src/workspace/fingerprint.ts +16 -1
- package/src/workspace/index.ts +15 -3
- package/src/workspace/lockfile.ts +13 -2
- package/src/workspace/nested-dirs.ts +21 -7
- package/src/workspace/package-graph.ts +23 -20
- package/src/workspace/project-loader.ts +503 -59
- package/src/workspace/workspace.ts +213 -72
- package/src/cache/tar.ts +0 -343
- package/src/cli/backend.ts +0 -23
- package/src/cli/mcp-rpc.ts +0 -292
- package/src/cli/mcp.ts +0 -92
- package/src/orchestrator/devframe-surface.ts +0 -74
- package/src/orchestrator/predict.ts +0 -106
- package/src/orchestrator/protocol.ts +0 -156
- package/src/orchestrator/run-state.ts +0 -145
- package/src/orchestrator/wire-render.ts +0 -92
- package/src/orchestrator/wire.ts +0 -212
|
@@ -17,8 +17,11 @@
|
|
|
17
17
|
// monitor in real time; violations carry the offending command +
|
|
18
18
|
// syscall line, so we get structured detection.
|
|
19
19
|
// Linux — bwrap denies the read at the kernel boundary; the child
|
|
20
|
-
// sees ENOENT (or EPERM for some operations). SRT
|
|
21
|
-
// structured events
|
|
20
|
+
// sees ENOENT (or EPERM for some operations). SRT surfaces no
|
|
21
|
+
// structured events there, so we wrap the spawn in strace and parse
|
|
22
|
+
// the trace ourselves (`deniedCalls`) — detection is NOT
|
|
23
|
+
// enforcement-only on Linux, and the fail-on-violation branch in
|
|
24
|
+
// execute-task is live on both platforms.
|
|
22
25
|
|
|
23
26
|
import path from 'node:path'
|
|
24
27
|
import os from 'node:os'
|
|
@@ -27,10 +30,12 @@ import { unlink } from 'node:fs/promises'
|
|
|
27
30
|
import type { SandboxConfig, SandboxNetworkConfig } from '../config.js'
|
|
28
31
|
import {
|
|
29
32
|
armTimeout,
|
|
33
|
+
drainOrAbort,
|
|
30
34
|
shellQuote,
|
|
31
35
|
signalExitCode,
|
|
32
36
|
streamToString,
|
|
33
37
|
resourceUsageToCpuRss,
|
|
38
|
+
type CaptureConfig,
|
|
34
39
|
type RunResult,
|
|
35
40
|
} from './runner.js'
|
|
36
41
|
import { xxh3hex } from '../util/index.js'
|
|
@@ -155,6 +160,7 @@ export async function resetSandbox(): Promise<void> {
|
|
|
155
160
|
const { SandboxManager } = await loadSrt()
|
|
156
161
|
await SandboxManager.reset()
|
|
157
162
|
availabilityCache = undefined
|
|
163
|
+
straceAvailableCache = undefined
|
|
158
164
|
}
|
|
159
165
|
|
|
160
166
|
export interface SandboxedRunArgs {
|
|
@@ -168,6 +174,11 @@ export interface SandboxedRunArgs {
|
|
|
168
174
|
liveChildren?: Set<ReturnType<typeof Bun.spawn>>
|
|
169
175
|
/** See `RunOptions.timeoutMs` — SIGTERM the child after this many ms. */
|
|
170
176
|
timeoutMs?: number
|
|
177
|
+
/** See `CaptureConfig` — which streams are retained on the result. */
|
|
178
|
+
capture?: CaptureConfig
|
|
179
|
+
/** See `ExecuteSandbox.settleOnCleanExit` — pay the macOS settle window
|
|
180
|
+
* even on a 0-exit, because an empty store will be read as proof. */
|
|
181
|
+
settleOnCleanExit?: boolean
|
|
171
182
|
/**
|
|
172
183
|
* Baseline reads — paths the sandbox unconditionally allows. The
|
|
173
184
|
* caller builds this from resolved `cache.inputs.files`.
|
|
@@ -227,6 +238,32 @@ function toRealPath(p: string): string {
|
|
|
227
238
|
}
|
|
228
239
|
}
|
|
229
240
|
|
|
241
|
+
/**
|
|
242
|
+
* Canonical form of every path the sandbox policy is expressed in.
|
|
243
|
+
*
|
|
244
|
+
* `resolveSandboxConfig` already canonicalizes the USER's paths; the
|
|
245
|
+
* orchestrator-supplied baselines (resolved inputs, output prefixes, the
|
|
246
|
+
* workspace-root deny anchor) arrived raw, so a workspace reached through a
|
|
247
|
+
* symlink expressed HALF its policy in real paths and half in link paths.
|
|
248
|
+
* bwrap then died mounting the link path inside its new root
|
|
249
|
+
* (`Can't mount tmpfs on /newroot/<link>`) and EVERY sandboxed task failed —
|
|
250
|
+
* whatever the config, with an error naming an internal path the user has no
|
|
251
|
+
* way to act on. Canonicalizing here is what makes the two halves agree.
|
|
252
|
+
*/
|
|
253
|
+
function canonicalBaselines(args: SandboxedRunArgs): {
|
|
254
|
+
allowRead: string[]
|
|
255
|
+
allowWrite: string[]
|
|
256
|
+
denyRead: string[]
|
|
257
|
+
cwd: string
|
|
258
|
+
} {
|
|
259
|
+
return {
|
|
260
|
+
allowRead: args.baseAllowRead.map(toRealPath),
|
|
261
|
+
allowWrite: args.baseAllowWrite.map(toRealPath),
|
|
262
|
+
denyRead: args.baseDenyRead.map(toRealPath),
|
|
263
|
+
cwd: toRealPath(args.cwd),
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
230
267
|
/**
|
|
231
268
|
* Convert a user-facing `SandboxConfig` (paths may be relative / tilde)
|
|
232
269
|
* into a `ResolvedSandboxConfig` (all paths absolute + canonical) for a
|
|
@@ -291,7 +328,8 @@ export async function runSandboxed(args: SandboxedRunArgs): Promise<SandboxedRun
|
|
|
291
328
|
const tag = xxh3hex(`${args.cwd}|${userCommand}|${process.hrtime.bigint()}`).slice(0, 16)
|
|
292
329
|
const taggedCommand = `: 'vx-${tag}'; ${userCommand}`
|
|
293
330
|
|
|
294
|
-
const
|
|
331
|
+
const baselines = canonicalBaselines(args)
|
|
332
|
+
const customConfig = buildCustomConfig(args, baselines)
|
|
295
333
|
const wrapped = await SandboxManager.wrapWithSandbox(taggedCommand, undefined, customConfig)
|
|
296
334
|
|
|
297
335
|
// Linux: SRT's SandboxViolationStore is macOS-only, so structured
|
|
@@ -335,31 +373,92 @@ export async function runSandboxed(args: SandboxedRunArgs): Promise<SandboxedRun
|
|
|
335
373
|
const timeout = armTimeout(proc, args.timeoutMs)
|
|
336
374
|
const ac = new AbortController()
|
|
337
375
|
const streams = Promise.all([
|
|
338
|
-
streamToString(proc.stdout, args.onStdout, ac.signal),
|
|
339
|
-
streamToString(proc.stderr, args.onStderr, ac.signal),
|
|
376
|
+
streamToString(proc.stdout, args.onStdout, ac.signal, args.capture?.stdout ?? true),
|
|
377
|
+
streamToString(proc.stderr, args.onStderr, ac.signal, args.capture?.stderr ?? true),
|
|
340
378
|
])
|
|
341
|
-
// See runCommand: gate on child exit
|
|
342
|
-
// a
|
|
379
|
+
// See runCommand: gate on child exit; a lingering grandchild pipe (timeout
|
|
380
|
+
// OR a clean exit that backgrounds a process) can't hang the run — timeout
|
|
381
|
+
// aborts at once, otherwise drainOrAbort bounds the post-exit drain.
|
|
343
382
|
await proc.exited
|
|
344
383
|
timeout.clear()
|
|
345
384
|
if (timeout.timedOut()) ac.abort()
|
|
385
|
+
else await drainOrAbort(streams, ac)
|
|
346
386
|
const [stdout, stderr] = await streams
|
|
347
387
|
args.liveChildren?.delete(proc)
|
|
348
388
|
const exitCode = proc.exitCode ?? (proc.signalCode ? signalExitCode(proc.signalCode) : 1)
|
|
349
389
|
|
|
350
390
|
// macOS: read the violation store keyed by our tagged command.
|
|
391
|
+
//
|
|
392
|
+
// The store is fed ASYNCHRONOUSLY — SRT's monitor ingests macOS unified-log
|
|
393
|
+
// records, and log delivery lags under load — so reading it immediately
|
|
394
|
+
// after child exit races the pipeline: the denial happened, the child died
|
|
395
|
+
// on it, and the store is still empty. Observed locally as the 1-in-N
|
|
396
|
+
// "denied but zero violation lines" flake (decision log 2026-08-23/24).
|
|
397
|
+
// When the exit is a FAILURE and the store is empty, give the pipeline a
|
|
398
|
+
// bounded settle window; a clean exit skips the poll entirely, so the warm
|
|
399
|
+
// path pays nothing.
|
|
400
|
+
//
|
|
401
|
+
// MEASURED (2026-08-24, ~430 runs/arm under full-suite load): the poll
|
|
402
|
+
// HALVES the loss (5.0% → 2.2%) but cannot eliminate it — the residual
|
|
403
|
+
// failures survive the whole 1 s window, meaning those records were
|
|
404
|
+
// DROPPED by the unified log under pressure, not delayed. Reporting on
|
|
405
|
+
// macOS is therefore lossy-by-OS under load; ENFORCEMENT is unaffected
|
|
406
|
+
// (every observed loss still denied the read and failed the child).
|
|
351
407
|
const store = SandboxManager.getSandboxViolationStore()
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
408
|
+
// Ancestor-directory traversal is NOT an input (owner call, 2026-08-24,
|
|
409
|
+
// closing the recorded false-`undeclared-inputs` item): to reach its cwd
|
|
410
|
+
// at all, a process stats/opens every directory on the path — macOS
|
|
411
|
+
// reports those as `deny(1) file-read-data <ancestor-dir>` for dirs the
|
|
412
|
+
// baseline doesn't list, node survives them, and no file CONTENT flowed.
|
|
413
|
+
// Dropping them is scoped to exact ancestor-or-self DIRECTORY paths of
|
|
414
|
+
// the task's cwd — a denied read of any file INSIDE an ancestor still
|
|
415
|
+
// reports. (These records usually arrived too late to be seen at all;
|
|
416
|
+
// the settle window made the pre-existing class visible on every clean
|
|
417
|
+
// verify exit.)
|
|
418
|
+
const cwdReal = toRealPath(args.cwd)
|
|
419
|
+
const ancestorDirs = new Set<string>()
|
|
420
|
+
for (let d = cwdReal; ; d = path.dirname(d)) {
|
|
421
|
+
ancestorDirs.add(d)
|
|
422
|
+
if (d === path.dirname(d)) break
|
|
423
|
+
}
|
|
424
|
+
const isAncestorTraversal = (line: string): boolean => {
|
|
425
|
+
const m = /deny\(\d+\)\s+file-read-(?:data|metadata)\s+(\/\S+)\s*$/.exec(line)
|
|
426
|
+
if (m === null) return false
|
|
427
|
+
return ancestorDirs.has(toRealPath(m[1]!))
|
|
428
|
+
}
|
|
429
|
+
const readMacViolations = (): SandboxViolation[] =>
|
|
430
|
+
store
|
|
431
|
+
.getViolationsForCommand(taggedCommand)
|
|
432
|
+
.filter((v) => !isAncestorTraversal(v.line))
|
|
433
|
+
.map((v) => ({
|
|
434
|
+
line: v.line,
|
|
435
|
+
timestamp: v.timestamp,
|
|
436
|
+
}))
|
|
437
|
+
let macViolations = readMacViolations()
|
|
438
|
+
// The fail-exit gate keeps the warm path free — EXCEPT when the caller
|
|
439
|
+
// says a clean exit + empty store will be read as PROOF (verify=inputs):
|
|
440
|
+
// a leaky task that swallows its own read error exits 0, so without the
|
|
441
|
+
// settle window a late unified-log record becomes a FALSE PASS of the
|
|
442
|
+
// verify (measured locally 2026-08-24: 1/30 at idle, the same lossy
|
|
443
|
+
// channel as the fail-exit case).
|
|
444
|
+
const emptyStoreIsProof = args.settleOnCleanExit === true
|
|
445
|
+
if (
|
|
446
|
+
process.platform === 'darwin' &&
|
|
447
|
+
(exitCode !== 0 || emptyStoreIsProof) &&
|
|
448
|
+
macViolations.length === 0
|
|
449
|
+
) {
|
|
450
|
+
for (let i = 0; i < 10 && macViolations.length === 0; i++) {
|
|
451
|
+
await Bun.sleep(100)
|
|
452
|
+
macViolations = readMacViolations()
|
|
453
|
+
}
|
|
454
|
+
}
|
|
356
455
|
|
|
357
456
|
// Linux: parse the strace log and emit one violation per denied
|
|
358
457
|
// syscall on a path inside denyRead that wasn't unconditionally
|
|
359
458
|
// allowed. Best-effort — if parsing fails we surface no Linux
|
|
360
459
|
// violations rather than fail the whole task.
|
|
361
460
|
const linuxViolations: SandboxViolation[] = straceLog
|
|
362
|
-
? await parseStraceViolations(straceLog, args).catch(() => [])
|
|
461
|
+
? await parseStraceViolations(straceLog, args, baselines).catch(() => [])
|
|
363
462
|
: []
|
|
364
463
|
if (straceLog) await unlink(straceLog).catch(() => undefined)
|
|
365
464
|
|
|
@@ -419,15 +518,81 @@ async function wantsStraceDetection(): Promise<boolean> {
|
|
|
419
518
|
* <pid> access("<path>", <mode>) = -1 EACCES (...)
|
|
420
519
|
* <pid> statx(AT_FDCWD, "<path>", <flags>, <mask>, ...) = -1 ENOENT (...)
|
|
421
520
|
*
|
|
521
|
+
* …but ONLY when the syscall completes without another traced process
|
|
522
|
+
* interleaving. Under `-f` strace splits an interrupted call across two
|
|
523
|
+
* lines and the result never appears next to the path:
|
|
524
|
+
* <pid> openat(AT_FDCWD, "<path>", <flags> <unfinished ...>
|
|
525
|
+
* <pid> <... openat resumed>) = -1 ENOENT (...)
|
|
526
|
+
* A single-line regex silently drops every one of those, so a task that
|
|
527
|
+
* forks concurrent children reading undeclared files reported an
|
|
528
|
+
* INCOMPLETE violation list — which for `--verify=inputs` reads as
|
|
529
|
+
* `proven-complete`. We pair them by pid instead (a process has at most
|
|
530
|
+
* one syscall in flight, so the pid is a sufficient key).
|
|
531
|
+
*
|
|
422
532
|
* We capture the first quoted-string argument as the path. paths that
|
|
423
533
|
* are relative resolve against the task's cwd (set by Bun.spawn).
|
|
424
534
|
*/
|
|
425
|
-
const
|
|
426
|
-
|
|
535
|
+
const SYSCALLS = 'openat|access|statx|newfstatat'
|
|
536
|
+
const STRACE_DONE_RE = new RegExp(
|
|
537
|
+
`^(\\d+)\\s+(${SYSCALLS})\\([^"]*"([^"]+)"[^)]*\\)\\s*=\\s*-1\\s+(ENOENT|EACCES|EPERM)`,
|
|
538
|
+
)
|
|
539
|
+
const STRACE_UNFINISHED_RE = new RegExp(`^(\\d+)\\s+(${SYSCALLS})\\([^"]*"([^"]+)"[^)]*<unfinished`)
|
|
540
|
+
const STRACE_RESUMED_RE = new RegExp(
|
|
541
|
+
`^(\\d+)\\s+<\\.\\.\\. (${SYSCALLS}) resumed>.*?=\\s*-1\\s+(ENOENT|EACCES|EPERM)`,
|
|
542
|
+
)
|
|
543
|
+
/** A resumed call that SUCCEEDED — clears the pending entry, emits nothing. */
|
|
544
|
+
const STRACE_RESUMED_OK_RE = new RegExp(`^(\\d+)\\s+<\\.\\.\\. (${SYSCALLS}) resumed>`)
|
|
545
|
+
|
|
546
|
+
/** One denied syscall, however strace chose to lay it out. */
|
|
547
|
+
export interface DeniedCall {
|
|
548
|
+
syscall: string
|
|
549
|
+
rawPath: string
|
|
550
|
+
errno: string
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/**
|
|
554
|
+
* Walk the trace, pairing `<unfinished ...>` with its `<... resumed>` line.
|
|
555
|
+
*
|
|
556
|
+
* Exported for testing: this is the security-relevant half of the Linux
|
|
557
|
+
* detector, and a synthetic trace pins the split-line shapes deterministically
|
|
558
|
+
* where an end-to-end run only produces them when strace happens to interleave.
|
|
559
|
+
*/
|
|
560
|
+
export function deniedCalls(text: string): DeniedCall[] {
|
|
561
|
+
const pending = new Map<string, { syscall: string; rawPath: string }>()
|
|
562
|
+
const out: DeniedCall[] = []
|
|
563
|
+
for (const line of text.split('\n')) {
|
|
564
|
+
const done = STRACE_DONE_RE.exec(line)
|
|
565
|
+
if (done?.[2] !== undefined && done[3] !== undefined && done[4] !== undefined) {
|
|
566
|
+
out.push({ syscall: done[2], rawPath: done[3], errno: done[4] })
|
|
567
|
+
continue
|
|
568
|
+
}
|
|
569
|
+
const unfinished = STRACE_UNFINISHED_RE.exec(line)
|
|
570
|
+
if (
|
|
571
|
+
unfinished?.[1] !== undefined &&
|
|
572
|
+
unfinished[2] !== undefined &&
|
|
573
|
+
unfinished[3] !== undefined
|
|
574
|
+
) {
|
|
575
|
+
pending.set(unfinished[1], { syscall: unfinished[2], rawPath: unfinished[3] })
|
|
576
|
+
continue
|
|
577
|
+
}
|
|
578
|
+
const resumedOk = STRACE_RESUMED_OK_RE.exec(line)
|
|
579
|
+
if (resumedOk?.[1] === undefined) continue
|
|
580
|
+
const held = pending.get(resumedOk[1])
|
|
581
|
+
pending.delete(resumedOk[1])
|
|
582
|
+
const resumed = STRACE_RESUMED_RE.exec(line)
|
|
583
|
+
// Only a resume that carries a DENIAL is a violation; a successful
|
|
584
|
+
// resume just retires the pending entry.
|
|
585
|
+
if (held !== undefined && resumed?.[3] !== undefined) {
|
|
586
|
+
out.push({ syscall: held.syscall, rawPath: held.rawPath, errno: resumed[3] })
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
return out
|
|
590
|
+
}
|
|
427
591
|
|
|
428
592
|
async function parseStraceViolations(
|
|
429
593
|
logPath: string,
|
|
430
594
|
args: SandboxedRunArgs,
|
|
595
|
+
baselines: { allowRead: readonly string[]; denyRead: readonly string[]; cwd: string },
|
|
431
596
|
): Promise<SandboxViolation[]> {
|
|
432
597
|
const text = await Bun.file(logPath).text()
|
|
433
598
|
if (text.length === 0) return []
|
|
@@ -435,18 +600,18 @@ async function parseStraceViolations(
|
|
|
435
600
|
// Treat every baseAllow + sandbox.allowRead path as "this was
|
|
436
601
|
// explicitly permitted; any -ENOENT here is the user's own missing
|
|
437
602
|
// file, not a sandbox-induced denial". Same for absolute denyRead
|
|
438
|
-
// checks below.
|
|
603
|
+
// checks below. Canonical on BOTH sides — the policy is expressed in
|
|
604
|
+
// real paths (see `canonicalBaselines`), so comparing a link-path here
|
|
605
|
+
// would report an explicitly-allowed read as a violation.
|
|
439
606
|
const allowAbs = new Set<string>(
|
|
440
|
-
[...
|
|
607
|
+
[...baselines.allowRead, ...args.config.allowRead].map((p) => toRealPath(absolutize(p))),
|
|
441
608
|
)
|
|
442
|
-
const denyAnchors =
|
|
609
|
+
const denyAnchors = baselines.denyRead.map((p) => toRealPath(absolutize(p)))
|
|
443
610
|
|
|
444
611
|
const seen = new Set<string>()
|
|
445
612
|
const out: SandboxViolation[] = []
|
|
446
|
-
for (const
|
|
447
|
-
const
|
|
448
|
-
if (!rawPath) continue
|
|
449
|
-
const abs = absolutize(rawPath, args.cwd)
|
|
613
|
+
for (const { syscall, rawPath, errno } of deniedCalls(text)) {
|
|
614
|
+
const abs = toRealPath(absolutize(rawPath, baselines.cwd))
|
|
450
615
|
// Only report paths under the workspace-root deny anchor — system
|
|
451
616
|
// libs / /proc / /sys / etc. probes are not interesting violations.
|
|
452
617
|
if (!denyAnchors.some((root) => abs === root || abs.startsWith(root + path.sep))) continue
|
|
@@ -520,11 +685,16 @@ function filterIgnored(
|
|
|
520
685
|
*/
|
|
521
686
|
function buildCustomConfig(
|
|
522
687
|
args: SandboxedRunArgs,
|
|
688
|
+
baselines: {
|
|
689
|
+
allowRead: readonly string[]
|
|
690
|
+
allowWrite: readonly string[]
|
|
691
|
+
denyRead: readonly string[]
|
|
692
|
+
},
|
|
523
693
|
): Parameters<SrtModule['SandboxManager']['wrapWithSandbox']>[2] {
|
|
524
694
|
const c = args.config
|
|
525
|
-
const allowRead = unique([...
|
|
526
|
-
const denyRead = unique([...
|
|
527
|
-
const allowWrite = unique([...
|
|
695
|
+
const allowRead = unique([...baselines.allowRead, ...c.allowRead])
|
|
696
|
+
const denyRead = unique([...baselines.denyRead])
|
|
697
|
+
const allowWrite = unique([...baselines.allowWrite, ...c.allowWrite])
|
|
528
698
|
|
|
529
699
|
const custom: Parameters<SrtModule['SandboxManager']['wrapWithSandbox']>[2] = {
|
|
530
700
|
filesystem: {
|
package/src/graph/index.ts
CHANGED
|
@@ -16,12 +16,14 @@ export {
|
|
|
16
16
|
type TaskOutcome,
|
|
17
17
|
type TaskStatus,
|
|
18
18
|
type VerifyVerdict,
|
|
19
|
-
ZERO_COST,
|
|
20
19
|
} from './scheduler.js'
|
|
21
20
|
export {
|
|
22
21
|
buildTaskGraph,
|
|
22
|
+
detectCycle,
|
|
23
23
|
expandRequested,
|
|
24
24
|
isGroupTask,
|
|
25
25
|
markSurfacedDeps,
|
|
26
|
+
splitTaskId,
|
|
26
27
|
type TaskNode,
|
|
28
|
+
unresolvedRequests,
|
|
27
29
|
} from './task-graph.js'
|
package/src/graph/scheduler.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { isUserError } from '../util/index.js'
|
|
2
2
|
import type { TaskNode } from './task-graph.js'
|
|
3
3
|
|
|
4
4
|
export type TaskStatus =
|
|
@@ -8,8 +8,10 @@ export type TaskStatus =
|
|
|
8
8
|
| 'failed'
|
|
9
9
|
| 'skipped'
|
|
10
10
|
// A child killed by a shutdown signal (SIGINT/SIGTERM — e.g. Ctrl-C):
|
|
11
|
-
// the task never finished on its own terms, so it reverts to
|
|
12
|
-
//
|
|
11
|
+
// the task never finished on its own terms, so it reverts to aborted —
|
|
12
|
+
// excluded from the tally and the run history. It still PROPAGATES like
|
|
13
|
+
// a failure (see `willSkip`): the child died mid-write, so its outputs
|
|
14
|
+
// are partial.
|
|
13
15
|
| 'aborted'
|
|
14
16
|
|
|
15
17
|
/**
|
|
@@ -41,6 +43,10 @@ export type VerifyVerdict =
|
|
|
41
43
|
| { kind: 'no-outputs' }
|
|
42
44
|
/** Didn't execute (cache hit) / not cacheable / group / persistent. */
|
|
43
45
|
| { kind: 'not-verified' }
|
|
46
|
+
/** A `remote: 'only'` task under `--verify=inputs`: verify pins placement
|
|
47
|
+
* local, so the task no-ops — there is no execution to sandbox, locally
|
|
48
|
+
* or anywhere. Reported, not silent: the proof does not cover it. */
|
|
49
|
+
| { kind: 'unverifiable-remote-only' }
|
|
44
50
|
|
|
45
51
|
/**
|
|
46
52
|
* Content fingerprint of a task's output tree, computed under a `--verify*`
|
|
@@ -75,9 +81,35 @@ export interface TaskOutcome {
|
|
|
75
81
|
/** Cache key hash, if one was computed. Folded into dependents' keys
|
|
76
82
|
* (pure-input transitive — the upstream's input key, not its output). */
|
|
77
83
|
hash?: string
|
|
84
|
+
/**
|
|
85
|
+
* For cache-hit statuses: the exec time the ENTRY was stored with — i.e.
|
|
86
|
+
* the work this hit skipped. Distinct from `durationMs`, which is always
|
|
87
|
+
* what THIS run spent (a hit's restore cost). The two differ by whatever
|
|
88
|
+
* the exec:restore ratio happens to be, so a surface reporting "time
|
|
89
|
+
* saved" must read this one; a surface reporting "time spent" must not.
|
|
90
|
+
* Undefined on every non-hit outcome — nothing was skipped.
|
|
91
|
+
*/
|
|
92
|
+
storedDurationMs?: number
|
|
78
93
|
/** v11 analytics: CPU time + peak RSS for this task's child process. */
|
|
79
94
|
cpuMs?: number
|
|
80
95
|
peakRssBytes?: number
|
|
96
|
+
/**
|
|
97
|
+
* A GROUP task's own dependency outcomes — the tasks it stands for.
|
|
98
|
+
* A group has no `exec` and therefore no outputs and no cache entry: its
|
|
99
|
+
* hash is a synthetic roll-up (`computeGroupHash`), so asking the local
|
|
100
|
+
* index what it produced returns nothing. A dependent needs the real
|
|
101
|
+
* tasks beneath it to describe its own input set, and only the group
|
|
102
|
+
* itself is ever in a position to say which those are. Set on group
|
|
103
|
+
* outcomes only; never folded into any key.
|
|
104
|
+
*/
|
|
105
|
+
groupUpstream?: readonly TaskOutcome[]
|
|
106
|
+
/** Executor-reported placement label (`ExecuteResult.where`) — set only
|
|
107
|
+
* when the task ran somewhere other than this host. Telemetry-only. */
|
|
108
|
+
where?: string
|
|
109
|
+
/** `'deferred'` when the task's outputs were left in the remote store
|
|
110
|
+
* (`--download=none`) instead of landing on this machine. Absent for
|
|
111
|
+
* every ordinary outcome. Telemetry-only. */
|
|
112
|
+
outputs?: 'deferred'
|
|
81
113
|
/**
|
|
82
114
|
* v11 analytics: hrtime span relative to the parent run's t=0.
|
|
83
115
|
* Lets downstream analytics reconstruct the actual parallel timeline
|
|
@@ -141,15 +173,15 @@ export interface ResourceCost {
|
|
|
141
173
|
mem: number
|
|
142
174
|
}
|
|
143
175
|
|
|
144
|
-
|
|
176
|
+
const ZERO_COST: ResourceCost = { cpu: 0, mem: 0 }
|
|
145
177
|
|
|
146
178
|
export interface ScheduleOptions {
|
|
147
179
|
nodes: Map<string, TaskNode>
|
|
148
180
|
concurrency: number
|
|
149
181
|
/**
|
|
150
182
|
* Failure propagation (default 'deps-ok' — the historical behavior):
|
|
151
|
-
* - 'deps-ok': a failed/skipped upstream skips its
|
|
152
|
-
* independent siblings keep running.
|
|
183
|
+
* - 'deps-ok': a failed/skipped/aborted upstream skips its
|
|
184
|
+
* dependents; independent siblings keep running.
|
|
153
185
|
* - 'never': the first failure stops dispatch — in-flight tasks
|
|
154
186
|
* finish naturally, everything not yet started completes as
|
|
155
187
|
* skipped (restore-tier included: a fail-fast run stops
|
|
@@ -192,6 +224,14 @@ export interface ScheduleOptions {
|
|
|
192
224
|
* second cache.get. When undefined/empty, behavior is byte-identical.
|
|
193
225
|
*/
|
|
194
226
|
restoreTier?: ReadonlySet<string>
|
|
227
|
+
/**
|
|
228
|
+
* Pool assignment for tasks placed on an executor with its own capacity
|
|
229
|
+
* (a remote pool). Such a task occupies one of the pool's `capacity` slots
|
|
230
|
+
* instead of a local worker slot and reserves no local resources, so a
|
|
231
|
+
* remote pool is never throttled by the local CPU count. Undefined (or
|
|
232
|
+
* a `undefined` result) = the local pool.
|
|
233
|
+
*/
|
|
234
|
+
poolOf?: (id: string) => { name: string; capacity: number } | undefined
|
|
195
235
|
/**
|
|
196
236
|
* Resolved per-task resource reservations (`exec.resources`, resolved
|
|
197
237
|
* by the orchestrator against the run's budgets). An absent id means
|
|
@@ -458,8 +498,32 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
|
|
|
458
498
|
// is a cheap tar extract, not the task's real work — it reserves ZERO
|
|
459
499
|
// regardless of what the config declares, so it never holds budget
|
|
460
500
|
// against a real executor (and never parks).
|
|
501
|
+
const poolOf = options.poolOf
|
|
502
|
+
const poolActive = new Map<string, number>()
|
|
503
|
+
// A pooled task is admitted against ITS pool's capacity, a local one
|
|
504
|
+
// against `concurrency`; pooled tasks reserve no local resources.
|
|
505
|
+
const hasRoom = (id: string): boolean => {
|
|
506
|
+
const pool = poolOf?.(id)
|
|
507
|
+
if (pool === undefined) return active < concurrency
|
|
508
|
+
return (poolActive.get(pool.name) ?? 0) < pool.capacity
|
|
509
|
+
}
|
|
510
|
+
const admit = (id: string): (() => void) => {
|
|
511
|
+
const pool = poolOf?.(id)
|
|
512
|
+
if (pool === undefined) {
|
|
513
|
+
active++
|
|
514
|
+
return () => {
|
|
515
|
+
active--
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
poolActive.set(pool.name, (poolActive.get(pool.name) ?? 0) + 1)
|
|
519
|
+
return () => {
|
|
520
|
+
poolActive.set(pool.name, (poolActive.get(pool.name) ?? 1) - 1)
|
|
521
|
+
}
|
|
522
|
+
}
|
|
461
523
|
const costOf = (id: string): ResourceCost =>
|
|
462
|
-
options.restoreTier?.has(id)
|
|
524
|
+
options.restoreTier?.has(id) || poolOf?.(id) !== undefined
|
|
525
|
+
? ZERO_COST
|
|
526
|
+
: (costs?.get(id) ?? ZERO_COST)
|
|
463
527
|
|
|
464
528
|
// Zero never blocks; a within-budget cost needs headroom (with an
|
|
465
529
|
// exact-fill epsilon); an over-budget cost can never have headroom, so
|
|
@@ -502,7 +566,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
|
|
|
502
566
|
// Observer hook (the logger's taskComplete). Crash-isolated: a
|
|
503
567
|
// throwing observer must NOT break scheduling — it would otherwise
|
|
504
568
|
// skip the dependent-enqueue + tick below and hang the run. Same
|
|
505
|
-
// "observability never breaks a run" rule the telemetry
|
|
569
|
+
// "observability never breaks a run" rule the telemetry
|
|
506
570
|
// paths hold. Isolated here (not the completion arm) so a throw
|
|
507
571
|
// can't be mistaken for the task itself failing.
|
|
508
572
|
try {
|
|
@@ -523,7 +587,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
|
|
|
523
587
|
}
|
|
524
588
|
|
|
525
589
|
// True when the task would be finished as `skipped` without running
|
|
526
|
-
// (fail-fast tripped, or a failed/skipped upstream under
|
|
590
|
+
// (fail-fast tripped, or a failed/skipped/aborted upstream under
|
|
527
591
|
// continueMode !== 'always'). ONE predicate shared by the admission
|
|
528
592
|
// parker and the dispatch loop's skip branch — a would-skip task
|
|
529
593
|
// executes nothing, so it must never park on a resource fit (a
|
|
@@ -541,7 +605,13 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
|
|
|
541
605
|
const node = nodes.get(id) as TaskNode
|
|
542
606
|
return node.deps.some((d) => {
|
|
543
607
|
const u = outcomes.get(d)
|
|
544
|
-
|
|
608
|
+
// `aborted` propagates even though it is counted nowhere: the
|
|
609
|
+
// upstream was killed mid-write, so its declared outputs are
|
|
610
|
+
// partial. A dependent that ran anyway would CACHE what it built
|
|
611
|
+
// from them — under the key a healthy run derives, since the fold
|
|
612
|
+
// takes the upstream's INPUT key — so the next run replays those
|
|
613
|
+
// bytes as a green hit.
|
|
614
|
+
return u?.status === 'failed' || u?.status === 'skipped' || u?.status === 'aborted'
|
|
545
615
|
})
|
|
546
616
|
}
|
|
547
617
|
|
|
@@ -561,16 +631,20 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
|
|
|
561
631
|
// task returns without a fit check (finishing it is free); restore
|
|
562
632
|
// tasks cost zero by construction, so they never park.
|
|
563
633
|
const takeFitting = (): string | undefined => {
|
|
634
|
+
// No pools declared (the common case): a full local pool admits
|
|
635
|
+
// nothing, so do not scan — this is the legacy O(1) gate.
|
|
636
|
+
if (poolOf === undefined && active >= concurrency) return undefined
|
|
564
637
|
while (execReady.size > 0) {
|
|
565
638
|
const seq = execReady.peekSeq()
|
|
566
639
|
const id = execReady.pop() as string
|
|
567
|
-
if (!resourcesActive ||
|
|
640
|
+
if (willSkip(id) || ((!resourcesActive || fits(id)) && hasRoom(id))) return id
|
|
568
641
|
parked.push([id, seq])
|
|
569
642
|
}
|
|
570
|
-
|
|
643
|
+
// Restore-tier tasks are local work (a cache restore on this disk).
|
|
644
|
+
return active < concurrency ? restoreReady.pop() : undefined
|
|
571
645
|
}
|
|
572
646
|
|
|
573
|
-
|
|
647
|
+
for (;;) {
|
|
574
648
|
const id = takeFitting()
|
|
575
649
|
if (id === undefined) break // nothing ready is admissible right now
|
|
576
650
|
const node = nodes.get(id) as TaskNode
|
|
@@ -590,7 +664,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
|
|
|
590
664
|
continue
|
|
591
665
|
}
|
|
592
666
|
|
|
593
|
-
|
|
667
|
+
const leave = admit(id)
|
|
594
668
|
// Reserve on dispatch; capture the cost so the release in the
|
|
595
669
|
// completion callbacks is symmetric even if the maps change.
|
|
596
670
|
const cost = costOf(id)
|
|
@@ -612,7 +686,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
|
|
|
612
686
|
// wedging the solo-clamp gate.
|
|
613
687
|
execute(node, upstream).then(
|
|
614
688
|
(outcome) => {
|
|
615
|
-
|
|
689
|
+
leave()
|
|
616
690
|
release(cost)
|
|
617
691
|
finishOne(id, outcome)
|
|
618
692
|
tick()
|
|
@@ -625,7 +699,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
|
|
|
625
699
|
exitCode: 1,
|
|
626
700
|
durationMs: 0,
|
|
627
701
|
}
|
|
628
|
-
|
|
702
|
+
leave()
|
|
629
703
|
release(cost)
|
|
630
704
|
finishOne(id, outcome)
|
|
631
705
|
// Surface the error live; the outcome itself doesn't
|
|
@@ -633,7 +707,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
|
|
|
633
707
|
// UserError is a config/input failure (e.g. a failed
|
|
634
708
|
// `cache.inputs.runtime` command), not a vx bug — report it
|
|
635
709
|
// plainly, never as an "internal error".
|
|
636
|
-
if (err
|
|
710
|
+
if (isUserError(err)) {
|
|
637
711
|
process.stderr.write(`[vx] ${id}: ${message}\n`)
|
|
638
712
|
} else {
|
|
639
713
|
const named = err instanceof Error && err.name !== 'Error' ? `${err.name}: ` : ''
|
|
@@ -658,7 +732,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
|
|
|
658
732
|
})
|
|
659
733
|
}
|
|
660
734
|
|
|
661
|
-
function mergePriorities(
|
|
735
|
+
export function mergePriorities(
|
|
662
736
|
baseline: ReadonlyMap<string, number>,
|
|
663
737
|
overrides: ReadonlyMap<string, number>,
|
|
664
738
|
): ReadonlyMap<string, number> {
|