@vzn/vx 0.0.21 → 0.0.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -18
- package/package.json +6 -6
- package/src/bin.ts +52 -3
- package/src/cache/archive.ts +170 -21
- package/src/cache/cache.ts +667 -288
- package/src/cache/chained-cache.ts +14 -2
- package/src/cache/config-evals.ts +29 -6
- package/src/cache/file-hashes.ts +63 -35
- package/src/cache/git-inputs.ts +398 -69
- package/src/cache/index.ts +8 -1
- package/src/cache/inputs.ts +403 -102
- package/src/cache/key-fold.ts +232 -0
- package/src/cache/layer.ts +81 -11
- package/src/cache/layered-cache.ts +199 -32
- package/src/cache/output-index.ts +109 -10
- package/src/cache/run-history.ts +8 -3
- package/src/cache/zstd.ts +53 -30
- package/src/cli/cache.ts +51 -21
- package/src/cli/format.ts +4 -15
- package/src/cli/help.ts +13 -3
- package/src/cli/index.ts +35 -9
- package/src/cli/info.ts +49 -7
- package/src/cli/init.ts +17 -5
- package/src/cli/last.ts +96 -21
- package/src/cli/lock.ts +34 -17
- package/src/cli/plan-format.ts +4 -4
- package/src/cli/plugin-commands.ts +20 -5
- package/src/cli/run.ts +55 -12
- package/src/cli/select.ts +23 -13
- package/src/cli/show.ts +9 -4
- package/src/cli/upgrade.ts +134 -14
- package/src/cli/watch.ts +622 -102
- package/src/cli/why.ts +8 -16
- package/src/cli/workspace-config.ts +21 -9
- package/src/config.ts +46 -9
- package/src/exec/executor.ts +51 -5
- package/src/exec/index.ts +10 -0
- package/src/exec/kill-tree.ts +281 -0
- package/src/exec/local-executor.ts +1 -1
- package/src/exec/runner.ts +327 -82
- package/src/exec/sandbox-binds.ts +34 -5
- package/src/exec/sandbox-paths.ts +22 -2
- package/src/exec/sandbox-runtime.ts +627 -102
- package/src/exec/sandbox-violations.ts +36 -23
- package/src/graph/dependency-spec.ts +14 -14
- package/src/graph/index.ts +12 -2
- package/src/graph/priorities.ts +130 -0
- package/src/graph/scheduler.ts +182 -134
- package/src/graph/task-graph.ts +552 -188
- package/src/index.ts +16 -2
- package/src/orchestrator/admission.ts +41 -17
- package/src/orchestrator/colors.ts +8 -3
- package/src/orchestrator/deferred-outputs.ts +22 -7
- package/src/orchestrator/doctor.ts +122 -14
- package/src/orchestrator/events.ts +71 -2
- package/src/orchestrator/excluded-keys.ts +155 -0
- package/src/orchestrator/execute-task.ts +331 -37
- package/src/orchestrator/failure-recap.ts +121 -0
- package/src/orchestrator/fingerprint-watch.ts +89 -0
- package/src/orchestrator/framed-output.ts +81 -14
- package/src/orchestrator/history.ts +6 -7
- package/src/orchestrator/hit-restore.ts +82 -18
- package/src/orchestrator/index.ts +7 -1
- package/src/orchestrator/keyed-projects.ts +89 -0
- package/src/orchestrator/local-shortcircuit.ts +83 -23
- package/src/orchestrator/lockfile-claim.ts +152 -32
- package/src/orchestrator/logger.ts +116 -20
- package/src/orchestrator/metrics.ts +156 -60
- package/src/orchestrator/miss-save.ts +106 -43
- package/src/orchestrator/options.ts +30 -4
- package/src/orchestrator/persistent.ts +41 -21
- package/src/orchestrator/placement.ts +55 -25
- package/src/orchestrator/plan.ts +24 -11
- package/src/orchestrator/plugin-host.ts +17 -23
- package/src/orchestrator/plugin.ts +31 -8
- package/src/orchestrator/prepare.ts +99 -13
- package/src/orchestrator/projects.ts +19 -8
- package/src/orchestrator/remote-prefetch.ts +21 -1
- package/src/orchestrator/run-artifacts.ts +9 -0
- package/src/orchestrator/run-context.ts +19 -4
- package/src/orchestrator/run-lock.ts +349 -0
- package/src/orchestrator/run-records.ts +6 -23
- package/src/orchestrator/run-report.ts +4 -2
- package/src/orchestrator/run.ts +264 -57
- package/src/orchestrator/sandbox-request.ts +388 -62
- package/src/orchestrator/save-lane.ts +4 -7
- package/src/orchestrator/shell-verdict.ts +117 -0
- package/src/orchestrator/signals.ts +86 -30
- package/src/orchestrator/stable-keys.ts +202 -10
- package/src/orchestrator/status-line.ts +33 -1
- package/src/orchestrator/summary.ts +49 -2
- package/src/orchestrator/task-hash.ts +87 -7
- package/src/orchestrator/task-log-buffer.ts +7 -7
- package/src/orchestrator/telemetry-host.ts +16 -7
- package/src/orchestrator/telemetry.ts +69 -28
- package/src/orchestrator/upstream.ts +91 -45
- package/src/util/bun-version.ts +43 -0
- package/src/util/errors.ts +86 -0
- package/src/util/hash.ts +10 -1
- package/src/util/index.ts +32 -3
- package/src/util/paths.ts +147 -5
- package/src/util/procfs.ts +20 -0
- package/src/util/settle.ts +4 -2
- package/src/util/size.ts +33 -0
- package/src/util/task-id.ts +20 -0
- package/src/util/timing.ts +11 -1
- package/src/util/which.ts +48 -0
- package/src/workspace/affected.ts +272 -64
- package/src/workspace/config-cache.ts +262 -32
- package/src/workspace/config-eval.ts +49 -12
- package/src/workspace/config-imports.ts +126 -5
- package/src/workspace/config-schema.ts +237 -41
- package/src/workspace/filter.ts +33 -18
- package/src/workspace/fingerprint.ts +29 -11
- package/src/workspace/index.ts +7 -0
- package/src/workspace/json-data.ts +90 -0
- package/src/workspace/load-reads.ts +32 -0
- package/src/workspace/lockfile.ts +33 -11
- package/src/workspace/migrate-scripts.ts +60 -7
- package/src/workspace/migration.ts +40 -1
- package/src/workspace/package-graph.ts +154 -16
- package/src/workspace/project-loader.ts +227 -77
- package/src/workspace/workspace.ts +173 -23
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import path from 'node:path'
|
|
2
2
|
import type { ExecConfig, TaskConfig, CacheConfig } from '../config.js'
|
|
3
3
|
import {
|
|
4
|
+
ArtifactVanishedError,
|
|
4
5
|
type CacheEntry,
|
|
5
6
|
type CacheLayer,
|
|
6
7
|
type CachePolicy,
|
|
7
8
|
cleanOutputs,
|
|
8
9
|
cleanWorkspaceOutputs,
|
|
10
|
+
type OutputStamp,
|
|
11
|
+
ownOutputsSince,
|
|
12
|
+
stampOutputs,
|
|
9
13
|
FULL_CACHE_POLICY,
|
|
10
14
|
type GitFilesCache,
|
|
11
15
|
} from '../cache/index.js'
|
|
@@ -20,16 +24,31 @@ import {
|
|
|
20
24
|
signalExitCode,
|
|
21
25
|
type CaptureConfig,
|
|
22
26
|
type ExecuteRequest,
|
|
27
|
+
assertExecuteResult,
|
|
23
28
|
type ExecuteResult,
|
|
24
29
|
type SandboxViolation,
|
|
25
30
|
type TaskExecutor,
|
|
26
31
|
type TaskInputs,
|
|
32
|
+
PersistentReadyError,
|
|
27
33
|
} from '../exec/index.js'
|
|
28
|
-
import { isGroupTask, type TaskNode, type TaskOutcome } from '../graph/index.js'
|
|
29
|
-
import { span } from '../util/index.js'
|
|
30
|
-
import {
|
|
31
|
-
|
|
34
|
+
import { isGroupTask, RestoreDemoted, type TaskNode, type TaskOutcome } from '../graph/index.js'
|
|
35
|
+
import { relPosix, span } from '../util/index.js'
|
|
36
|
+
import {
|
|
37
|
+
mayWriteFingerprint,
|
|
38
|
+
type Placeholder,
|
|
39
|
+
placeholderSweeper,
|
|
40
|
+
reachedWithheld,
|
|
41
|
+
sandboxRequestFor,
|
|
42
|
+
sweepPlaceholders,
|
|
43
|
+
undeclaredWriteReach,
|
|
44
|
+
untouchedPlaceholderLine,
|
|
45
|
+
type WithheldLink,
|
|
46
|
+
withheldLinkLine,
|
|
47
|
+
} from './sandbox-request.js'
|
|
48
|
+
import { markUnsaved, saveMiss, type OutputDirSnapshot } from './miss-save.js'
|
|
49
|
+
import type { FingerprintWatch } from './fingerprint-watch.js'
|
|
32
50
|
import { restoreHit } from './hit-restore.js'
|
|
51
|
+
import { shellVerdict } from './shell-verdict.js'
|
|
33
52
|
// The hit path's entry stays importable from here (tests).
|
|
34
53
|
export { restoreHit, type RestoreHitArgs } from './hit-restore.js'
|
|
35
54
|
import type { DeferredOutputs } from './deferred-outputs.js'
|
|
@@ -39,8 +58,10 @@ import {
|
|
|
39
58
|
computeTaskHash,
|
|
40
59
|
describeTaskInputs,
|
|
41
60
|
type HashCache,
|
|
61
|
+
movedInput,
|
|
42
62
|
type TaskInputComponent,
|
|
43
63
|
} from './task-hash.js'
|
|
64
|
+
import { getContext } from './remote-prefetch.js'
|
|
44
65
|
|
|
45
66
|
export interface ExecuteArgs {
|
|
46
67
|
node: TaskNode
|
|
@@ -81,6 +102,13 @@ export interface ExecuteArgs {
|
|
|
81
102
|
nestedProjectDirs: string[]
|
|
82
103
|
/** Anchor for hrtime spans across all tasks in this run. */
|
|
83
104
|
runStartHrTimeNs: bigint
|
|
105
|
+
/**
|
|
106
|
+
* The run's stop (a process signal or the embedder's abort). An attempt
|
|
107
|
+
* that ends while it is aborted is `aborted`, whatever its exit: a task
|
|
108
|
+
* that traps the forwarded SIGINT and exits 0 did not finish on its own
|
|
109
|
+
* terms (item 962).
|
|
110
|
+
*/
|
|
111
|
+
stopSignal?: AbortSignal
|
|
84
112
|
/**
|
|
85
113
|
* Registry the orchestrator owns. For each persistent task we
|
|
86
114
|
* spawn, we stash the subprocess handle here so the orchestrator
|
|
@@ -129,6 +157,12 @@ export interface ExecuteArgs {
|
|
|
129
157
|
* sandbox. A cache hit never calls it: a hit needs no sandbox.
|
|
130
158
|
*/
|
|
131
159
|
armSandbox?: () => Promise<void>
|
|
160
|
+
/**
|
|
161
|
+
* The run's keyed-set lookup (keyed-projects.ts): the project directories
|
|
162
|
+
* a task's key answers for. A sandboxed task that declares `cache` is
|
|
163
|
+
* granted a linked workspace package only when it is in this set.
|
|
164
|
+
*/
|
|
165
|
+
keyedProjects: (node: TaskNode) => ReadonlySet<string>
|
|
132
166
|
/**
|
|
133
167
|
* Run-scoped list the miss path appends its output-directory snapshot
|
|
134
168
|
* request to, taken at run end (run.ts) instead of right after the save:
|
|
@@ -154,6 +188,18 @@ export interface ExecuteArgs {
|
|
|
154
188
|
* clean run's stale hit.
|
|
155
189
|
*/
|
|
156
190
|
taintedUpstream?: boolean
|
|
191
|
+
/**
|
|
192
|
+
* Whether a task has rewritten a file the workspace fingerprint folded
|
|
193
|
+
* since the run read it (fingerprint-watch.ts): past that, no key taken
|
|
194
|
+
* on the old digest is probed or saved.
|
|
195
|
+
*/
|
|
196
|
+
fingerprintWatch?: FingerprintWatch
|
|
197
|
+
/**
|
|
198
|
+
* No task in the run depends on this one, so none is ordered after it to
|
|
199
|
+
* read what it wrote: a miss that saves nothing marks nothing (the output
|
|
200
|
+
* walk cost 1,000 read-only misses 1.62 → 1.78 s, item 750).
|
|
201
|
+
*/
|
|
202
|
+
noDependants?: true
|
|
157
203
|
}
|
|
158
204
|
|
|
159
205
|
/**
|
|
@@ -231,16 +277,34 @@ async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
231
277
|
// (Until 2026-09-09 the block was accepted and silently ignored.)
|
|
232
278
|
let command = plainCommand
|
|
233
279
|
let bridgeTag: string | undefined
|
|
280
|
+
let signalChannel = false
|
|
281
|
+
// The empty files the request pre-created for a literal write grant; a
|
|
282
|
+
// server that never writes one gets it taken back when it exits, and a
|
|
283
|
+
// server that dies before readiness is told the directory spelling — the
|
|
284
|
+
// same trap and the same answer as a one-shot task's (sandbox-request.ts).
|
|
285
|
+
let placeholders: Placeholder[] = []
|
|
234
286
|
if (step.sandbox !== undefined) {
|
|
235
287
|
await args.armSandbox?.()
|
|
236
|
-
|
|
288
|
+
// A persistent task never declares `cache` (the loader refuses it), so
|
|
289
|
+
// it keeps the whole linked grant: no key of its can be stale.
|
|
290
|
+
const sb = await sandboxRequestFor(
|
|
291
|
+
node,
|
|
292
|
+
step.sandbox,
|
|
293
|
+
args.workspaceRoot,
|
|
294
|
+
undefined,
|
|
295
|
+
args.nestedProjectDirs,
|
|
296
|
+
)
|
|
297
|
+
placeholders = sb.placeholders
|
|
237
298
|
const wrapped = await wrapSandboxedCommand({
|
|
238
299
|
command: plainCommand,
|
|
239
300
|
cwd: node.projectDir,
|
|
240
|
-
|
|
301
|
+
env,
|
|
302
|
+
...sb.sandbox,
|
|
303
|
+
server: true,
|
|
241
304
|
})
|
|
242
305
|
command = wrapped.wrapped
|
|
243
306
|
bridgeTag = wrapped.tag
|
|
307
|
+
signalChannel = wrapped.forwardsSignals
|
|
244
308
|
}
|
|
245
309
|
const persistentOpts: Parameters<typeof runPersistent>[0] = {
|
|
246
310
|
command,
|
|
@@ -249,6 +313,7 @@ async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
249
313
|
onStdout: (chunk) => log.taskStdout(node, chunk),
|
|
250
314
|
onStderr: (chunk) => log.taskStderr(node, chunk),
|
|
251
315
|
...(args.liveChildren !== undefined ? { liveChildren: args.liveChildren } : {}),
|
|
316
|
+
...(signalChannel ? { signalChannel } : {}),
|
|
252
317
|
}
|
|
253
318
|
if (step.persistent.readyWhen !== undefined) {
|
|
254
319
|
persistentOpts.readyWhen = step.persistent.readyWhen
|
|
@@ -263,12 +328,28 @@ async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
263
328
|
const spawn = runPersistent(persistentOpts)
|
|
264
329
|
// The host side of a port bridge lives exactly as long as the server:
|
|
265
330
|
// released on the child's exit, whether the run tore it down or it died.
|
|
266
|
-
|
|
331
|
+
// A placeholder the server never wrote goes back the same way.
|
|
332
|
+
// ONE sweep, shared by the exit handler and the readiness failure below
|
|
333
|
+
// — whichever asks second must get the SAME list. A second sweep cannot
|
|
334
|
+
// produce it: the first one's `rm` already happened, so the file is gone
|
|
335
|
+
// and `sweepPlaceholders` skips it. Collecting both lists and unioning
|
|
336
|
+
// them does not close that, because the union only covers an exit
|
|
337
|
+
// handler that FINISHED; one still between its `rm` and its return has
|
|
338
|
+
// published nothing yet, and the readiness path then reports no
|
|
339
|
+
// placeholder at all. The failure then says only "File exists", which is
|
|
340
|
+
// the message the hint exists to explain (seen under the gate's parallel
|
|
341
|
+
// load, and reproduced deterministically by delaying each side, item 450).
|
|
342
|
+
const sweptUntouched = placeholderSweeper(placeholders)
|
|
343
|
+
if (bridgeTag !== undefined || placeholders.length > 0) {
|
|
267
344
|
const tag = bridgeTag
|
|
268
|
-
void
|
|
269
|
-
()
|
|
270
|
-
|
|
271
|
-
|
|
345
|
+
const onExit = async (): Promise<void> => {
|
|
346
|
+
if (tag !== undefined) releaseBridges(tag)
|
|
347
|
+
await sweptUntouched()
|
|
348
|
+
}
|
|
349
|
+
// A spawn that failed has no child to wait on: release now, or the
|
|
350
|
+
// tag would hold the sandbox's reset for the rest of the process.
|
|
351
|
+
if (spawn.child === undefined) void onExit()
|
|
352
|
+
else void spawn.child.exited.then(onExit, onExit)
|
|
272
353
|
}
|
|
273
354
|
try {
|
|
274
355
|
await spawn.ready
|
|
@@ -278,17 +359,28 @@ async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
278
359
|
// reader looks for why a task failed, and a run with a custom logger
|
|
279
360
|
// (an embedder, the MCP server) never saw a bare stderr write at all.
|
|
280
361
|
log.taskStderr(node, `\n[vx] ${node.id}: persistent task failed to become ready: ${message}\n`)
|
|
362
|
+
// The server is dead or being torn down, so the sweep on its exit may
|
|
363
|
+
// already be running: ask the shared one rather than starting a second.
|
|
364
|
+
for (const p of await sweptUntouched()) {
|
|
365
|
+
log.taskStderr(node, `${untouchedPlaceholderLine(node.projectDir, p)}\n`)
|
|
366
|
+
}
|
|
367
|
+
// The reason rides the outcome (every label reads it), and a child that
|
|
368
|
+
// exited before ready keeps its own exit code rather than a made-up 1.
|
|
369
|
+
const ready = err instanceof PersistentReadyError ? err : undefined
|
|
281
370
|
return {
|
|
282
371
|
node,
|
|
283
372
|
status: 'failed',
|
|
284
|
-
exitCode: 1,
|
|
373
|
+
exitCode: ready?.exitCode ?? 1,
|
|
285
374
|
durationMs: spawn.readyMs(),
|
|
375
|
+
...(ready !== undefined ? { notReady: ready.reason } : {}),
|
|
286
376
|
wallclockStartNs,
|
|
287
377
|
wallclockEndNs: process.hrtime.bigint() - args.runStartHrTimeNs,
|
|
288
378
|
}
|
|
289
379
|
}
|
|
290
380
|
|
|
291
381
|
args.persistentRegistry?.set(node.id, spawn.child)
|
|
382
|
+
forgetUndeclaredWrites(args, undeclaredWriteReach(node, args.workspaceRoot))
|
|
383
|
+
if (mayWriteFingerprint(node, args.workspaceRoot)) args.fingerprintWatch?.wrote()
|
|
292
384
|
return {
|
|
293
385
|
node,
|
|
294
386
|
status: 'success',
|
|
@@ -412,6 +504,14 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
412
504
|
nestedProjectDirs: args.nestedProjectDirs,
|
|
413
505
|
}
|
|
414
506
|
const wsCleanArgs = { workspaceRoot: args.workspaceRoot, outputs: wsOutputs }
|
|
507
|
+
// An ADDITIVE task (its outputs overlap an upstream's, with the edge that
|
|
508
|
+
// orders them; item 588) does not clean by glob — the glob selects the
|
|
509
|
+
// upstream's files it adds beside. Its outputs are stamped once before
|
|
510
|
+
// the first attempt, and after a 0 exit its own set is what the run
|
|
511
|
+
// added or changed against that stamp. Stale files of its own from an
|
|
512
|
+
// earlier run are its command's to clean, as they are under Turbo.
|
|
513
|
+
const additive = (node.addsToOutputsOf?.length ?? 0) > 0
|
|
514
|
+
let stampedBefore: ReadonlyMap<string, OutputStamp> | undefined
|
|
415
515
|
|
|
416
516
|
// Cache lookup. On hit, time the user-perceived restore op
|
|
417
517
|
// (clean+restore+log-replay) — that's what the framed-block footer
|
|
@@ -421,17 +521,37 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
421
521
|
// to the run path as a known stable miss. No second cache.get.
|
|
422
522
|
if (willRead) {
|
|
423
523
|
const cacheOpStart = performance.now()
|
|
524
|
+
// A hit whose artifact was removed after the probe (a prune, another
|
|
525
|
+
// workspace's retention on a shared cache directory) is a miss. The
|
|
526
|
+
// restore has wiped the declared outputs by then; the run path cleans
|
|
527
|
+
// them again and the task writes them.
|
|
528
|
+
const restoreOrMiss = async (hit: CacheEntry): Promise<TaskOutcome | null> => {
|
|
529
|
+
try {
|
|
530
|
+
return await restoreHit({ args, hash, hit, cacheOpStart, taskStartNs })
|
|
531
|
+
} catch (err) {
|
|
532
|
+
if (!(err instanceof ArtifactVanishedError)) throw err
|
|
533
|
+
log.status(`[vx] ${node.id}: ${err.message} — running it`)
|
|
534
|
+
return null
|
|
535
|
+
}
|
|
536
|
+
}
|
|
424
537
|
if (preProbed !== undefined) {
|
|
425
538
|
if (preProbed.hit !== null) {
|
|
426
|
-
|
|
539
|
+
const restored = await restoreOrMiss(preProbed.hit)
|
|
540
|
+
if (restored !== null) return restored
|
|
541
|
+
// The up-front probe's hit may be restoring AHEAD of this task's
|
|
542
|
+
// deps, and a command run now would build from outputs they have
|
|
543
|
+
// not written yet and save that under the good key. The scheduler
|
|
544
|
+
// runs it again once they are done (admission drops the probe).
|
|
545
|
+
throw new RestoreDemoted(node.id)
|
|
427
546
|
}
|
|
428
547
|
// Confirmed stable miss — skip the probe, fall through to run.
|
|
429
|
-
} else {
|
|
548
|
+
} else if (!fingerprintMoved()) {
|
|
430
549
|
const endProbe = span('cache.get')
|
|
431
|
-
const hit = await cache.get(hash,
|
|
550
|
+
const hit = await cache.get(hash, getContext(node, step.command))
|
|
432
551
|
endProbe()
|
|
433
552
|
if (hit) {
|
|
434
|
-
|
|
553
|
+
const restored = await restoreOrMiss(hit)
|
|
554
|
+
if (restored !== null) return restored
|
|
435
555
|
}
|
|
436
556
|
}
|
|
437
557
|
}
|
|
@@ -482,6 +602,12 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
482
602
|
// the single source of truth, and a violation fails the task (below).
|
|
483
603
|
const userSandbox = cfg.exec?.sandbox !== undefined
|
|
484
604
|
let violations: SandboxViolation[] = []
|
|
605
|
+
// The empty files the sandbox request created for a literal write grant;
|
|
606
|
+
// swept after the attempt (`sweepPlaceholders`).
|
|
607
|
+
let placeholders: Placeholder[] = []
|
|
608
|
+
// The linked workspace packages the request withheld: the key does not
|
|
609
|
+
// answer for them.
|
|
610
|
+
let withheld: WithheldLink[] = []
|
|
485
611
|
|
|
486
612
|
// Cache miss path (or caching disabled), up to `1 + retries` attempts.
|
|
487
613
|
// Explicit config wins over the run-level `--retry` default, including
|
|
@@ -506,7 +632,11 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
506
632
|
// tree would replace a stale build with nothing at all. The eligibility
|
|
507
633
|
// gate guarantees no key in this run can see what stays behind, and the
|
|
508
634
|
// summary names every deferred task so `dist/` is not silently stale.
|
|
509
|
-
if (willWrite && !deferralRequested && outputs.length > 0) {
|
|
635
|
+
if (additive && willWrite && !deferralRequested && outputs.length > 0) {
|
|
636
|
+
const endStamp = span('miss: stamp outputs')
|
|
637
|
+
stampedBefore ??= await stampOutputs(cleanArgs)
|
|
638
|
+
endStamp()
|
|
639
|
+
} else if (willWrite && !deferralRequested && outputs.length > 0) {
|
|
510
640
|
// Mark the wiped paths, exactly as the workspace twin below does. The
|
|
511
641
|
// git snapshot still lists them with their committed index OIDs, and
|
|
512
642
|
// resolveFiles SKIPS its existence probe for any path carrying a
|
|
@@ -536,13 +666,38 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
536
666
|
// no other trace. Rethrown unchanged — the scheduler still classifies it,
|
|
537
667
|
// and still prints it plainly for a UserError.
|
|
538
668
|
const endExec = span('miss: execute')
|
|
539
|
-
const res = await args.executor
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
669
|
+
const res = await args.executor
|
|
670
|
+
.execute(req)
|
|
671
|
+
.then((r: unknown) => {
|
|
672
|
+
assertExecuteResult(args.executor.name, node.id, r)
|
|
673
|
+
return r
|
|
674
|
+
})
|
|
675
|
+
.catch(async (err: unknown) => {
|
|
676
|
+
const message = err instanceof Error ? err.message : String(err)
|
|
677
|
+
log.taskStderr(node, `${message}\n`)
|
|
678
|
+
await sweepPlaceholders(placeholders)
|
|
679
|
+
throw err
|
|
680
|
+
})
|
|
544
681
|
endExec()
|
|
545
682
|
violations = [...res.violations]
|
|
683
|
+
// A denial under a dependency the key does not answer for says only
|
|
684
|
+
// ENOENT; the line beside it names the package and how to key it. Only
|
|
685
|
+
// with a denial there, so it never reddens a pass.
|
|
686
|
+
for (const w of reachedWithheld(withheld, violations)) {
|
|
687
|
+
violations.push({ timestamp: new Date(), line: withheldLinkLine(node.id, w) })
|
|
688
|
+
}
|
|
689
|
+
// A placeholder the task never wrote is not its output: take it back
|
|
690
|
+
// before the outputs are collected. On a failure it is also the one
|
|
691
|
+
// clue to a grant that meant a directory — say so beside the failure.
|
|
692
|
+
const untouched = await sweepPlaceholders(placeholders)
|
|
693
|
+
if (res.exitCode !== 0 && violations.length === 0) {
|
|
694
|
+
for (const p of untouched) {
|
|
695
|
+
violations.push({
|
|
696
|
+
timestamp: new Date(),
|
|
697
|
+
line: untouchedPlaceholderLine(node.projectDir, p),
|
|
698
|
+
})
|
|
699
|
+
}
|
|
700
|
+
}
|
|
546
701
|
// Fail-on-violation, on BOTH platforms: macOS reads SRT's structured
|
|
547
702
|
// violation store, Linux parses the strace log the sandboxed spawn
|
|
548
703
|
// writes. (This comment used to claim Linux violations are "always 0"
|
|
@@ -572,6 +727,20 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
572
727
|
// child already reports 143, so this only rewrites the trap-exit-0 case.
|
|
573
728
|
if (code === 0) code = signalExitCode('SIGTERM')
|
|
574
729
|
}
|
|
730
|
+
// The shell's 127 and 126 name the word and nothing about why — the
|
|
731
|
+
// PATH vx built, or a `#!` line the file itself carries (items 257,
|
|
732
|
+
// 258) — and an exit above 128 is a signal's number and nothing
|
|
733
|
+
// about what sent it (259). One frame line names the rule.
|
|
734
|
+
if (!res.timedOut) {
|
|
735
|
+
const verdict = shellVerdict({
|
|
736
|
+
code,
|
|
737
|
+
command: step.command,
|
|
738
|
+
cwd: node.projectDir,
|
|
739
|
+
bins: taskBinDirs(node, args.workspaceRoot),
|
|
740
|
+
signal: res.signal,
|
|
741
|
+
})
|
|
742
|
+
if (verdict !== undefined) log.taskStderr(node, `\n${verdict}\n`)
|
|
743
|
+
}
|
|
575
744
|
return { result: res, exitCode: code }
|
|
576
745
|
}
|
|
577
746
|
|
|
@@ -599,19 +768,33 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
599
768
|
}
|
|
600
769
|
}
|
|
601
770
|
|
|
771
|
+
// Pass or fail, the command may have written where its project's
|
|
772
|
+
// run-start facts describe; a remote executor wrote on its own disk.
|
|
773
|
+
const writeReach =
|
|
774
|
+
args.executor.remote === true ? 'none' : undeclaredWriteReach(node, args.workspaceRoot)
|
|
775
|
+
const writesFingerprint =
|
|
776
|
+
args.executor.remote !== true && mayWriteFingerprint(node, args.workspaceRoot)
|
|
602
777
|
for (;;) {
|
|
603
778
|
attempt++
|
|
604
779
|
const a = await runAttempt()
|
|
780
|
+
forgetUndeclaredWrites(args, writeReach)
|
|
781
|
+
if (writesFingerprint) args.fingerprintWatch?.wrote()
|
|
605
782
|
result = a.result
|
|
606
783
|
effectiveExitCode = a.exitCode
|
|
607
784
|
|
|
608
|
-
//
|
|
609
|
-
//
|
|
610
|
-
//
|
|
611
|
-
//
|
|
612
|
-
//
|
|
613
|
-
//
|
|
614
|
-
|
|
785
|
+
// An attempt that ended in a shutdown never finished on its own terms —
|
|
786
|
+
// it is aborted, so it is neither cached, counted, shown, nor RETRIED.
|
|
787
|
+
// Two ways to know: the child died of SIGINT / SIGTERM (Ctrl-C, or a
|
|
788
|
+
// `kill` vx never saw; a timeout also SIGTERMs, but `timedOut` marks it
|
|
789
|
+
// as our own deadline, a real and retryable failure), or the RUN is
|
|
790
|
+
// stopping. The child alone was not enough: one that traps the
|
|
791
|
+
// forwarded SIGINT and exits 0 was cached as a success with its partial
|
|
792
|
+
// outputs, one that exits 1 was retried after Ctrl-C, and one SIGKILLed
|
|
793
|
+
// at the end of the grace read as an OOM failure (item 962).
|
|
794
|
+
const shutdown =
|
|
795
|
+
args.stopSignal?.aborted === true ||
|
|
796
|
+
((result.signal === 'SIGINT' || result.signal === 'SIGTERM') && !result.timedOut)
|
|
797
|
+
if (shutdown) {
|
|
615
798
|
return {
|
|
616
799
|
node,
|
|
617
800
|
status: 'aborted',
|
|
@@ -626,7 +809,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
626
809
|
if (effectiveExitCode === 0 || attempt >= maxAttempts) break
|
|
627
810
|
log.taskStderr(
|
|
628
811
|
node,
|
|
629
|
-
`vx: retrying ${node.id} (attempt ${attempt + 1}/${maxAttempts}) after exit ${effectiveExitCode}\n`,
|
|
812
|
+
`vx: retrying ${node.id} (attempt ${attempt + 1}/${maxAttempts}) after ${result.timedOut === true ? 'a timeout' : `exit ${effectiveExitCode}`}\n`,
|
|
630
813
|
)
|
|
631
814
|
}
|
|
632
815
|
|
|
@@ -655,7 +838,16 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
655
838
|
}
|
|
656
839
|
if (!userSandbox) return base
|
|
657
840
|
await args.armSandbox?.()
|
|
658
|
-
|
|
841
|
+
const sb = await sandboxRequestFor(
|
|
842
|
+
node,
|
|
843
|
+
step.sandbox!,
|
|
844
|
+
args.workspaceRoot,
|
|
845
|
+
cfgCacheable ? args.keyedProjects(node) : undefined,
|
|
846
|
+
args.nestedProjectDirs,
|
|
847
|
+
)
|
|
848
|
+
placeholders = sb.placeholders
|
|
849
|
+
withheld = sb.withheld
|
|
850
|
+
return { ...base, sandbox: sb.sandbox }
|
|
659
851
|
}
|
|
660
852
|
|
|
661
853
|
const wallclockEndNs = process.hrtime.bigint() - args.runStartHrTimeNs
|
|
@@ -681,8 +873,13 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
681
873
|
},
|
|
682
874
|
})
|
|
683
875
|
}
|
|
684
|
-
} else if (effectiveExitCode === 0 && willSave) {
|
|
876
|
+
} else if (effectiveExitCode === 0 && willSave && (await keyStillTrue())) {
|
|
877
|
+
const ownOutputFiles =
|
|
878
|
+
additive && stampedBefore !== undefined
|
|
879
|
+
? await ownOutputsSince(cleanArgs, stampedBefore)
|
|
880
|
+
: undefined
|
|
685
881
|
const { landed } = await saveMiss({
|
|
882
|
+
...(ownOutputFiles !== undefined ? { ownOutputFiles } : {}),
|
|
686
883
|
node,
|
|
687
884
|
hash,
|
|
688
885
|
cache,
|
|
@@ -702,6 +899,76 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
702
899
|
deferSave: args.deferSave,
|
|
703
900
|
})
|
|
704
901
|
args.deferredSaves?.set(node.id, landed)
|
|
902
|
+
} else if (cfgCacheable && !remoteOnly && !deferralRequested && args.noDependants !== true) {
|
|
903
|
+
// Ran here and saves nothing — failed, a read-only policy, a tainted
|
|
904
|
+
// upstream, a key that no longer held — yet it wrote what a save would
|
|
905
|
+
// have marked, and may have rewritten an input the re-check before a
|
|
906
|
+
// save catches. A same-run reader keyed from the snapshot's OIDs for
|
|
907
|
+
// either replayed the bytes from before the command (item 750). A
|
|
908
|
+
// withheld save already dropped the project; this finds the same move
|
|
909
|
+
// again and drops it again.
|
|
910
|
+
const endCheck = span('miss: recheck inputs')
|
|
911
|
+
const moved = await movedSinceKey()
|
|
912
|
+
endCheck()
|
|
913
|
+
if (moved !== undefined) {
|
|
914
|
+
forgetUndeclaredWrites(args, wsOutputs.length > 0 ? 'workspace' : 'project')
|
|
915
|
+
} else {
|
|
916
|
+
await markUnsaved({
|
|
917
|
+
node,
|
|
918
|
+
workspaceRoot: args.workspaceRoot,
|
|
919
|
+
nestedProjectDirs: args.nestedProjectDirs,
|
|
920
|
+
gitFilesCache: args.gitFilesCache,
|
|
921
|
+
outputs,
|
|
922
|
+
wsOutputs,
|
|
923
|
+
})
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
/**
|
|
928
|
+
* The workspace fingerprint every key here folded has moved (a task
|
|
929
|
+
* rewrote the lockfile), said once per run.
|
|
930
|
+
*/
|
|
931
|
+
function fingerprintMoved(): boolean {
|
|
932
|
+
if (args.fingerprintWatch?.moved() === undefined) return false
|
|
933
|
+
args.fingerprintWatch.say(log)
|
|
934
|
+
return true
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
/**
|
|
938
|
+
* The input that no longer holds what the key folded: `null` when the
|
|
939
|
+
* key re-derived just before the command already differed (the file
|
|
940
|
+
* unnamed), undefined when none moved.
|
|
941
|
+
*/
|
|
942
|
+
async function movedSinceKey(): Promise<string | null | undefined> {
|
|
943
|
+
return described!.hash !== hash
|
|
944
|
+
? null
|
|
945
|
+
: await movedInput(described!.facts, cache, described!.describedAt)
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
/**
|
|
949
|
+
* Does the key still describe the inputs the command ran over? It was
|
|
950
|
+
* taken before the command — up front, or before `describeTaskInputs` —
|
|
951
|
+
* and a save files the outputs under it. A user's edit mid-run, or a
|
|
952
|
+
* task rewriting its own input (a formatter), saved bytes built from one
|
|
953
|
+
* state under the key of another: restore the old state and the next run
|
|
954
|
+
* replayed them as up-to-date (turborepo#10111, #1146, item 743). The
|
|
955
|
+
* result stands; only the entry is withheld, and the facts about the
|
|
956
|
+
* project go, since something wrote there. A lockfile a task rewrote
|
|
957
|
+
* withholds it too: the key folded the old one.
|
|
958
|
+
*/
|
|
959
|
+
async function keyStillTrue(): Promise<boolean> {
|
|
960
|
+
if (fingerprintMoved()) return false
|
|
961
|
+
const endCheck = span('miss: recheck inputs')
|
|
962
|
+
const moved = await movedSinceKey()
|
|
963
|
+
endCheck()
|
|
964
|
+
if (moved === undefined) return true
|
|
965
|
+
const what = moved === null ? 'its inputs' : `\`${relPosix(args.workspaceRoot, moved)}\``
|
|
966
|
+
log.status(
|
|
967
|
+
`[vx] ${node.id}: ${what} changed after its key was taken — the result stands, ` +
|
|
968
|
+
`but is not saved under a key that no longer describes it`,
|
|
969
|
+
)
|
|
970
|
+
forgetUndeclaredWrites(args, wsOutputs.length > 0 ? 'workspace' : 'project')
|
|
971
|
+
return false
|
|
705
972
|
}
|
|
706
973
|
|
|
707
974
|
const finalViolations = violations
|
|
@@ -713,6 +980,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
713
980
|
durationMs: result.durationMs,
|
|
714
981
|
hash,
|
|
715
982
|
...(attempt > 1 ? { attempts: attempt } : {}),
|
|
983
|
+
...(result.timedOut === true && effectiveExitCode !== 0 ? { timedOut: true as const } : {}),
|
|
716
984
|
...(result.cpuMs !== undefined ? { cpuMs: result.cpuMs } : {}),
|
|
717
985
|
...(result.peakRssBytes !== undefined ? { peakRssBytes: result.peakRssBytes } : {}),
|
|
718
986
|
...(result.where !== undefined ? { where: result.where } : {}),
|
|
@@ -730,6 +998,36 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
730
998
|
}
|
|
731
999
|
}
|
|
732
1000
|
|
|
1001
|
+
/**
|
|
1002
|
+
* Drop the run's facts about files a task may have written without
|
|
1003
|
+
* declaring them (`undeclaredWriteReach`): the project's git snapshot and
|
|
1004
|
+
* its index OIDs, and its `package.json` digest. The next reader
|
|
1005
|
+
* re-enumerates the project (one `git ls-files`) and hashes its files by
|
|
1006
|
+
* content; the per-declaration file-list memos are keyed on the snapshot
|
|
1007
|
+
* array, so they miss with it (item 743).
|
|
1008
|
+
*/
|
|
1009
|
+
function forgetUndeclaredWrites(args: ExecuteArgs, reach: 'none' | 'project' | 'workspace'): void {
|
|
1010
|
+
if (reach === 'none') return
|
|
1011
|
+
if (reach === 'workspace') {
|
|
1012
|
+
args.gitFilesCache?.clear()
|
|
1013
|
+
args.hashCache?.packageJson.clear()
|
|
1014
|
+
return
|
|
1015
|
+
}
|
|
1016
|
+
args.gitFilesCache?.delete(args.node.projectDir)
|
|
1017
|
+
args.gitFilesCache?.invalidateWorkspacePartition()
|
|
1018
|
+
args.hashCache?.packageJson.delete(args.node.projectDir)
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
/** The two bin directories a task's PATH starts with: its own, then the root's (once when they coincide). */
|
|
1022
|
+
function taskBinDirs(node: TaskNode, workspaceRoot: string): string[] {
|
|
1023
|
+
const bins = [path.join(node.projectDir, 'node_modules', '.bin')]
|
|
1024
|
+
const rootBin = path.join(workspaceRoot, 'node_modules', '.bin')
|
|
1025
|
+
// Identical when the root is itself a project — dedupe rather than list it
|
|
1026
|
+
// twice, so PATH reads the same either way.
|
|
1027
|
+
if (rootBin !== bins[0]) bins.push(rootBin)
|
|
1028
|
+
return bins
|
|
1029
|
+
}
|
|
1030
|
+
|
|
733
1031
|
/**
|
|
734
1032
|
* Build the child-process env for one task. Same arguments at every call site
|
|
735
1033
|
* (persistent + cached). Two `node_modules/.bin` directories are prepended to
|
|
@@ -747,11 +1045,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
747
1045
|
* it. npm/pnpm/yarn all put the ancestor chain on PATH for the same reason.
|
|
748
1046
|
*/
|
|
749
1047
|
function taskEnv(node: TaskNode, step: ExecConfig, workspaceRoot: string): NodeJS.ProcessEnv {
|
|
750
|
-
const bins =
|
|
751
|
-
const rootBin = path.join(workspaceRoot, 'node_modules', '.bin')
|
|
752
|
-
// Identical when the root is itself a project — dedupe rather than list it
|
|
753
|
-
// twice, so PATH reads the same either way.
|
|
754
|
-
if (rootBin !== bins[0]) bins.push(rootBin)
|
|
1048
|
+
const bins = taskBinDirs(node, workspaceRoot)
|
|
755
1049
|
const env = buildIsolatedEnv({
|
|
756
1050
|
passThrough: step.env?.passThrough ?? [],
|
|
757
1051
|
define: step.env?.define ?? {},
|