@vzn/vx 0.0.645 → 0.0.648
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -50
- package/launcher.cjs +9 -2
- package/launcher.sh +2 -2
- package/package.json +5 -5
- package/schemas/cache.json +5 -5
- package/schemas/error.json +1 -1
- package/schemas/info.json +16 -2
- package/schemas/last.json +127 -5
- package/schemas/plan.json +28 -2
- package/schemas/show.json +10 -1
- package/skills/vx/SKILL.md +19 -3
- package/src/bin.ts +54 -13
- package/src/cache/archive.ts +324 -23
- package/src/cache/cache.ts +330 -65
- package/src/cache/git-inputs.ts +274 -97
- package/src/cache/index.ts +7 -2
- package/src/cache/inputs.ts +515 -83
- package/src/cache/key-fold.ts +15 -6
- package/src/cache/layer.ts +74 -11
- package/src/cache/layered-cache.ts +154 -59
- package/src/cache/linked-inputs.ts +1 -1
- package/src/cache/output-index.ts +44 -14
- package/src/cache/plugin-cache.ts +49 -23
- package/src/cache/remote-hits.ts +72 -0
- package/src/cache/remote-lane.ts +100 -15
- package/src/cache/run-history.ts +77 -23
- package/src/cache/run-store.ts +11 -10
- package/src/cache/schema.ts +42 -6
- package/src/cache/tar-stream.ts +5 -0
- package/src/cache/umask.ts +1 -2
- package/src/cli/cache.ts +58 -27
- package/src/cli/completions.ts +12 -5
- package/src/cli/core-alias.ts +5 -2
- package/src/cli/docs.ts +42 -16
- package/src/cli/help.ts +62 -25
- package/src/cli/index.ts +79 -45
- package/src/cli/info.ts +12 -6
- package/src/cli/init.ts +222 -34
- package/src/cli/last.ts +443 -47
- package/src/cli/lock.ts +197 -52
- package/src/cli/plan-format.ts +91 -49
- package/src/cli/plugin-commands.ts +160 -11
- package/src/cli/plugin-templates.ts +13 -13
- package/src/cli/registry.ts +209 -0
- package/src/cli/run.ts +111 -41
- package/src/cli/select.ts +148 -49
- package/src/cli/show.ts +59 -16
- package/src/cli/task-verb.ts +16 -6
- package/src/cli/upgrade.ts +155 -54
- package/src/cli/watch-cycle.ts +10 -5
- package/src/cli/watch-filter.ts +71 -3
- package/src/cli/watch-fs.ts +22 -6
- package/src/cli/watch-judge.ts +48 -6
- package/src/cli/watch.ts +163 -42
- package/src/cli/why.ts +211 -73
- package/src/cli/workspace-config.ts +54 -15
- package/src/config.ts +13 -1
- package/src/exec/env.ts +7 -2
- package/src/exec/executor.ts +262 -15
- package/src/exec/index.ts +8 -2
- package/src/exec/kill-tree.ts +15 -0
- package/src/exec/local-executor.ts +1 -1
- package/src/exec/proc-sample.ts +70 -6
- package/src/exec/runner.ts +256 -53
- package/src/exec/sandbox-binds.ts +7 -3
- package/src/exec/sandbox-paths.ts +1 -1
- package/src/exec/sandbox-runtime.ts +222 -28
- package/src/exec/sandbox-violations.ts +52 -20
- package/src/graph/critical-path.ts +79 -0
- package/src/graph/default-build.ts +19 -0
- package/src/graph/definition.ts +38 -0
- package/src/graph/folds.ts +38 -5
- package/src/graph/index.ts +4 -1
- package/src/graph/priorities.ts +74 -8
- package/src/graph/scheduler.ts +326 -65
- package/src/graph/task-graph.ts +228 -35
- package/src/handoff.ts +97 -9
- package/src/history/failure-mode.ts +6 -3
- package/src/history/history.ts +268 -50
- package/src/history/index.ts +2 -0
- package/src/history/metrics.ts +142 -45
- package/src/index.ts +42 -10
- package/src/orchestrator/admission.ts +10 -6
- package/src/orchestrator/affected-tasks.ts +11 -6
- package/src/orchestrator/deferred-outputs.ts +2 -0
- package/src/orchestrator/doctor.ts +134 -42
- package/src/orchestrator/execute-args.ts +6 -0
- package/src/orchestrator/execute-task.ts +155 -42
- package/src/orchestrator/fingerprint-watch.ts +1 -1
- package/src/orchestrator/graph-memo.ts +140 -0
- package/src/orchestrator/graph-stage.ts +18 -1
- package/src/orchestrator/held-servers.ts +86 -0
- package/src/orchestrator/hit-restore.ts +227 -122
- package/src/orchestrator/index.ts +20 -3
- package/src/orchestrator/local-floor.ts +198 -0
- package/src/orchestrator/local-shortcircuit.ts +51 -8
- package/src/orchestrator/lockfile-claim.ts +96 -16
- package/src/orchestrator/miss-save.ts +52 -12
- package/src/orchestrator/options.ts +8 -6
- package/src/orchestrator/output-host.ts +43 -14
- package/src/orchestrator/output-log.ts +90 -10
- package/src/orchestrator/persistent-task.ts +97 -28
- package/src/orchestrator/persistent.ts +16 -4
- package/src/orchestrator/placement.ts +257 -14
- package/src/orchestrator/plan.ts +64 -7
- package/src/orchestrator/plugin-host.ts +230 -40
- package/src/orchestrator/plugin.ts +150 -37
- package/src/orchestrator/prepare.ts +178 -32
- package/src/orchestrator/projects.ts +65 -40
- package/src/orchestrator/remote-prefetch.ts +119 -25
- package/src/orchestrator/run-artifacts.ts +95 -32
- package/src/orchestrator/run-context.ts +39 -41
- package/src/orchestrator/run-failures.ts +65 -23
- package/src/orchestrator/run-id.ts +6 -3
- package/src/orchestrator/run-keys.ts +300 -0
- package/src/orchestrator/run-live.ts +294 -0
- package/src/orchestrator/run-lock.ts +13 -7
- package/src/orchestrator/run-records.ts +35 -23
- package/src/orchestrator/run-report.ts +3 -3
- package/src/orchestrator/run-steps.ts +58 -22
- package/src/orchestrator/run-stop.ts +22 -5
- package/src/orchestrator/run.ts +409 -79
- package/src/orchestrator/sandbox-fix.ts +7 -5
- package/src/orchestrator/sandbox-request.ts +78 -22
- package/src/orchestrator/shell-verdict.ts +5 -5
- package/src/orchestrator/signals.ts +26 -1
- package/src/orchestrator/stable-keys.ts +92 -8
- package/src/orchestrator/steps.ts +2 -2
- package/src/orchestrator/task-artifacts.ts +206 -0
- package/src/orchestrator/task-hash.ts +89 -14
- package/src/orchestrator/task-log-buffer.ts +72 -36
- package/src/orchestrator/task-process.ts +22 -0
- package/src/orchestrator/telemetry-host.ts +24 -3
- package/src/orchestrator/telemetry.ts +113 -38
- package/src/orchestrator/upstream.ts +15 -0
- package/src/report/events.ts +32 -13
- package/src/report/forecast.ts +36 -35
- package/src/report/framed-output.ts +173 -46
- package/src/report/index.ts +1 -0
- package/src/report/logger.ts +217 -186
- package/src/report/path-links.ts +12 -4
- package/src/report/plain-output.ts +13 -5
- package/src/report/status-line.ts +94 -46
- package/src/report/summary.ts +174 -38
- package/src/report/terminal-signals.ts +13 -2
- package/src/settings.ts +14 -6
- package/src/util/bun-version.ts +4 -1
- package/src/util/colors.ts +20 -3
- package/src/util/config-frame.ts +41 -4
- package/src/{graph → util}/dependency-spec.ts +22 -13
- package/src/util/edit-distance.ts +13 -1
- package/src/util/env-flag.ts +9 -0
- package/src/util/errors.ts +59 -3
- package/src/util/essential-env.ts +57 -0
- package/src/util/git-ignored.ts +1 -1
- package/src/util/hash.ts +35 -0
- package/src/util/hide-env.ts +45 -5
- package/src/util/index.ts +22 -5
- package/src/util/num.ts +8 -0
- package/src/util/paths.ts +121 -21
- package/src/util/report.ts +6 -1
- package/src/util/secret-mask.ts +62 -1
- package/src/util/secrets.ts +7 -2
- package/src/util/size.ts +11 -3
- package/src/util/tail.ts +6 -1
- package/src/util/timing.ts +13 -1
- package/src/util/ui.ts +65 -4
- package/src/util/which.ts +1 -1
- package/src/util/write-file.ts +67 -0
- package/src/workspace/affected.ts +710 -107
- package/src/workspace/bracket-literal.ts +84 -0
- package/src/workspace/canonical-dir.ts +37 -0
- package/src/workspace/config-bytecode.ts +282 -0
- package/src/workspace/config-eval.ts +29 -8
- package/src/workspace/config-imports.ts +199 -58
- package/src/workspace/config-schema.ts +242 -39
- package/src/workspace/filter.ts +37 -8
- package/src/workspace/index.ts +11 -2
- package/src/workspace/json-data.ts +3 -4
- package/src/workspace/lockfile.ts +57 -8
- package/src/workspace/migration.ts +72 -49
- package/src/workspace/package-graph.ts +5 -4
- package/src/workspace/project-loader.ts +314 -47
- package/src/workspace/repo-id.ts +1 -1
- package/src/workspace/workspace.ts +176 -33
- package/src/cli/foreign-flags.ts +0 -727
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
// what or how tasks run. Contrast `cache`/`executor`, which return objects
|
|
17
17
|
// core calls INTO; those are the behavior capabilities, kept separate.
|
|
18
18
|
|
|
19
|
-
import { sampleTrees } from '../exec/index.js'
|
|
19
|
+
import { sampleTrees, type MachineReport } from '../exec/index.js'
|
|
20
20
|
import {
|
|
21
21
|
type CacheSource,
|
|
22
22
|
deriveCacheSource,
|
|
@@ -27,7 +27,13 @@ import {
|
|
|
27
27
|
type TaskOutcome,
|
|
28
28
|
type TaskStatus,
|
|
29
29
|
} from '../graph/index.js'
|
|
30
|
-
import {
|
|
30
|
+
import {
|
|
31
|
+
maskedCommand,
|
|
32
|
+
settleWithin,
|
|
33
|
+
stageTimes,
|
|
34
|
+
teardownTimeoutMs,
|
|
35
|
+
thrownMessage,
|
|
36
|
+
} from '../util/index.js'
|
|
31
37
|
import type { RunEvent, RunEventSubscriber } from '../report/index.js'
|
|
32
38
|
|
|
33
39
|
/** Bumped when the record shape changes. Readers MUST check `v`. */
|
|
@@ -114,6 +120,10 @@ export interface TaskTelemetry {
|
|
|
114
120
|
/** Executor-reported placement (a worker id) — absent for local runs.
|
|
115
121
|
* Additive-optional, no schema bump (the `attempts` precedent). */
|
|
116
122
|
where?: string
|
|
123
|
+
/** A persistent task each dependant's executor started where that ran:
|
|
124
|
+
* part of those runs, never an execution of its own (2026-10-23:
|
|
125
|
+
* nothing runs on a distributed run's orchestrator). Additive-optional. */
|
|
126
|
+
carried?: true
|
|
117
127
|
/** `'deferred'` when the outputs stayed remote (`--cache-download=none`).
|
|
118
128
|
* Additive-optional, same no-bump precedent as `where`. */
|
|
119
129
|
outputs?: 'deferred'
|
|
@@ -127,6 +137,8 @@ export interface TaskTelemetry {
|
|
|
127
137
|
* fail-fast skip and on every other status. Additive (schema stays 2).
|
|
128
138
|
*/
|
|
129
139
|
blockedBy?: string
|
|
140
|
+
/** On a `skipped` task: its `exec.platforms` leave out the machine that ran the run. Additive. */
|
|
141
|
+
otherPlatform?: { readonly needs: readonly string[]; readonly host: string }
|
|
130
142
|
/** On a `failed` task: vx's own `timeout` killed it (exit 143 is the deadline, not a signal). Additive. */
|
|
131
143
|
timedOut?: true
|
|
132
144
|
/** On a sandboxed task: how many declared-boundary violations the sandbox recorded (a failure on its own). Additive. */
|
|
@@ -188,6 +200,10 @@ export type TelemetryRecord =
|
|
|
188
200
|
/** The run's canonical start (epoch ms) — equals the summary's
|
|
189
201
|
* `startedAt`; a sink derives per-task timing from it during the run. */
|
|
190
202
|
startedAt: number
|
|
203
|
+
/** Where the run's stage table begins (epoch ms), the first stage's
|
|
204
|
+
* start: at or before `startedAt`. A sink that draws the stages
|
|
205
|
+
* starts the run here. Additive. */
|
|
206
|
+
originAt?: number
|
|
191
207
|
}
|
|
192
208
|
| {
|
|
193
209
|
v: number
|
|
@@ -274,6 +290,17 @@ export interface RunSummaryRecord {
|
|
|
274
290
|
* Absent when nothing was uploaded or tried. Additive.
|
|
275
291
|
*/
|
|
276
292
|
uploads?: { count: number; bytes: number; ms: number; failed: number }
|
|
293
|
+
/**
|
|
294
|
+
* The machines an executor plugin ran tasks on, with what each did
|
|
295
|
+
* (`TaskExecutor.machines`); a task's `where` names one. Absent when no
|
|
296
|
+
* executor reports any. Additive.
|
|
297
|
+
*/
|
|
298
|
+
machines?: readonly MachineReport[]
|
|
299
|
+
/**
|
|
300
|
+
* The run's end had an executor placed its tasks perfectly
|
|
301
|
+
* (`TaskExecutor.idealMs`), beside `totalDurationMs`. Additive.
|
|
302
|
+
*/
|
|
303
|
+
idealMs?: number
|
|
277
304
|
}
|
|
278
305
|
|
|
279
306
|
/** One stage of a run, a wall window in epoch ms. */
|
|
@@ -305,6 +332,8 @@ export function assembleRunSummary(
|
|
|
305
332
|
abortedCount: number
|
|
306
333
|
stages?: readonly RunStage[]
|
|
307
334
|
uploads?: { count: number; bytes: number; ms: number; failed: number }
|
|
335
|
+
machines?: readonly MachineReport[]
|
|
336
|
+
idealMs?: number | undefined
|
|
308
337
|
},
|
|
309
338
|
): RunSummaryRecord {
|
|
310
339
|
let failedCount = 0
|
|
@@ -343,6 +372,10 @@ export function assembleRunSummary(
|
|
|
343
372
|
...(timing.uploads !== undefined && timing.uploads.count + timing.uploads.failed > 0
|
|
344
373
|
? { uploads: { ...timing.uploads } }
|
|
345
374
|
: {}),
|
|
375
|
+
...(timing.machines !== undefined && timing.machines.length > 0
|
|
376
|
+
? { machines: timing.machines }
|
|
377
|
+
: {}),
|
|
378
|
+
...(timing.idealMs !== undefined ? { idealMs: timing.idealMs } : {}),
|
|
346
379
|
}
|
|
347
380
|
}
|
|
348
381
|
|
|
@@ -400,12 +433,25 @@ export interface TelemetrySource {
|
|
|
400
433
|
* Sample this task's process tree every `SAMPLE_MS` until the returned
|
|
401
434
|
* function is called, the root exits, or the run ends. Present only when
|
|
402
435
|
* a sink wants `task.sample`: nobody asking costs no timer and no read.
|
|
436
|
+
* The returned stop answers what the samples saw, for the run history.
|
|
403
437
|
*/
|
|
404
|
-
readonly track?: (taskId: string, pid: number) => () =>
|
|
438
|
+
readonly track?: (taskId: string, pid: number) => () => TreeLoad | undefined
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* What a tracked task's samples saw: the most cores one interval between
|
|
443
|
+
* looks burned, and the mean resident memory over the looks. Undefined
|
|
444
|
+
* when no look landed (a task shorter than `SAMPLE_MS`).
|
|
445
|
+
*/
|
|
446
|
+
export interface TreeLoad {
|
|
447
|
+
peakCores: number
|
|
448
|
+
avgRssBytes: number
|
|
405
449
|
}
|
|
406
450
|
|
|
407
451
|
/** How often a running task's process tree is sampled. */
|
|
408
452
|
const SAMPLE_MS = 1000
|
|
453
|
+
/** How long an aborted flush may take to end its own requests. */
|
|
454
|
+
const ABORT_GRACE_MS = 250
|
|
409
455
|
|
|
410
456
|
/** Call `fn` every `ms` until the returned stop is called. */
|
|
411
457
|
type Every = (ms: number, fn: () => Promise<void>) => () => void
|
|
@@ -468,7 +514,7 @@ export function createTelemetrySource(args: {
|
|
|
468
514
|
if (disabled.has(sink)) return
|
|
469
515
|
disabled.add(sink)
|
|
470
516
|
warn?.(
|
|
471
|
-
`[vx] telemetry sink '${label(sink)}' threw in ${hook}; disabled for this run: ${
|
|
517
|
+
`[vx] telemetry sink '${label(sink)}' threw in ${hook}; disabled for this run: ${thrownMessage(err)}`,
|
|
472
518
|
)
|
|
473
519
|
}
|
|
474
520
|
// Precompute which sinks want each kind, so per-event fan-out is a plain
|
|
@@ -476,10 +522,18 @@ export function createTelemetrySource(args: {
|
|
|
476
522
|
const wantsLog = sinks.some((s) => (s.wants ?? DEFAULT_KINDS).includes('task.log'))
|
|
477
523
|
const wantsSample = sinks.some((s) => (s.wants ?? DEFAULT_KINDS).includes('task.sample'))
|
|
478
524
|
|
|
479
|
-
// taskId → the
|
|
480
|
-
// only while one runs; a tick still reading when the
|
|
481
|
-
// doubled.
|
|
482
|
-
|
|
525
|
+
// taskId → its running root and what the looks at it saw. One timer for
|
|
526
|
+
// every task, alive only while one runs; a tick still reading when the
|
|
527
|
+
// next is due is not doubled.
|
|
528
|
+
interface Tracked {
|
|
529
|
+
pid: number
|
|
530
|
+
lastTs: number
|
|
531
|
+
lastCpuMs: number
|
|
532
|
+
peakCores: number
|
|
533
|
+
rssSum: number
|
|
534
|
+
looks: number
|
|
535
|
+
}
|
|
536
|
+
const tracked = new Map<string, Tracked>()
|
|
483
537
|
const every = args.every ?? everyInterval
|
|
484
538
|
let stopEvery: (() => void) | undefined
|
|
485
539
|
let sampling = false
|
|
@@ -488,8 +542,8 @@ export function createTelemetrySource(args: {
|
|
|
488
542
|
stopEvery?.()
|
|
489
543
|
stopEvery = undefined
|
|
490
544
|
}
|
|
491
|
-
const untrack = (taskId: string,
|
|
492
|
-
if (tracked.get(taskId) ===
|
|
545
|
+
const untrack = (taskId: string, t: Tracked): void => {
|
|
546
|
+
if (tracked.get(taskId) === t) tracked.delete(taskId)
|
|
493
547
|
if (tracked.size === 0) stopTimer()
|
|
494
548
|
}
|
|
495
549
|
const tick = async (): Promise<void> => {
|
|
@@ -497,15 +551,23 @@ export function createTelemetrySource(args: {
|
|
|
497
551
|
sampling = true
|
|
498
552
|
try {
|
|
499
553
|
const entries = [...tracked]
|
|
500
|
-
const usage = await sampleTrees(entries.map(([,
|
|
554
|
+
const usage = await sampleTrees(entries.map(([, t]) => t.pid))
|
|
501
555
|
const ts = Date.now()
|
|
502
|
-
for (const [taskId,
|
|
503
|
-
if (ended || tracked.get(taskId) !==
|
|
504
|
-
const u = usage.get(pid)
|
|
556
|
+
for (const [taskId, t] of entries) {
|
|
557
|
+
if (ended || tracked.get(taskId) !== t) continue
|
|
558
|
+
const u = usage.get(t.pid)
|
|
505
559
|
if (u === undefined) {
|
|
506
|
-
untrack(taskId,
|
|
560
|
+
untrack(taskId, t)
|
|
507
561
|
continue
|
|
508
562
|
}
|
|
563
|
+
// A descendant that exited takes its CPU out of the live sum, so an
|
|
564
|
+
// interval can read negative: it says nothing of the peak.
|
|
565
|
+
const ms = ts - t.lastTs
|
|
566
|
+
if (ms > 0) t.peakCores = Math.max(t.peakCores, (u.cpuMs - t.lastCpuMs) / ms)
|
|
567
|
+
t.lastTs = ts
|
|
568
|
+
t.lastCpuMs = u.cpuMs
|
|
569
|
+
t.rssSum += u.rssBytes
|
|
570
|
+
t.looks++
|
|
509
571
|
deliver({
|
|
510
572
|
v: TELEMETRY_SCHEMA_VERSION,
|
|
511
573
|
kind: 'task.sample',
|
|
@@ -522,11 +584,17 @@ export function createTelemetrySource(args: {
|
|
|
522
584
|
sampling = false
|
|
523
585
|
}
|
|
524
586
|
}
|
|
525
|
-
const track = (taskId: string, pid: number): (() =>
|
|
526
|
-
if (ended) return () =>
|
|
527
|
-
|
|
587
|
+
const track = (taskId: string, pid: number): (() => TreeLoad | undefined) => {
|
|
588
|
+
if (ended) return () => undefined
|
|
589
|
+
const t: Tracked = { pid, lastTs: Date.now(), lastCpuMs: 0, peakCores: 0, rssSum: 0, looks: 0 }
|
|
590
|
+
tracked.set(taskId, t)
|
|
528
591
|
stopEvery ??= every(SAMPLE_MS, tick)
|
|
529
|
-
return () =>
|
|
592
|
+
return () => {
|
|
593
|
+
untrack(taskId, t)
|
|
594
|
+
return t.looks === 0
|
|
595
|
+
? undefined
|
|
596
|
+
: { peakCores: t.peakCores, avgRssBytes: Math.round(t.rssSum / t.looks) }
|
|
597
|
+
}
|
|
530
598
|
}
|
|
531
599
|
|
|
532
600
|
function deliver(record: TelemetryRecord): void {
|
|
@@ -556,6 +624,7 @@ export function createTelemetrySource(args: {
|
|
|
556
624
|
total: event.info.total,
|
|
557
625
|
ts,
|
|
558
626
|
startedAt: event.info.startedAtMs ?? ts,
|
|
627
|
+
...originOf(),
|
|
559
628
|
})
|
|
560
629
|
return
|
|
561
630
|
case 'task:start': {
|
|
@@ -639,28 +708,26 @@ export function createTelemetrySource(args: {
|
|
|
639
708
|
// accessed_at bumps and every later plugin's teardown lost with it.
|
|
640
709
|
// Sinks race concurrently, so each still gets the whole budget.
|
|
641
710
|
const deadline = new AbortController()
|
|
642
|
-
const
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
)
|
|
656
|
-
}
|
|
657
|
-
}),
|
|
658
|
-
),
|
|
659
|
-
ms,
|
|
711
|
+
const all = Promise.all(
|
|
712
|
+
sinks.map(async (sink) => {
|
|
713
|
+
if (disabled.has(sink) || sink.flush === undefined) return
|
|
714
|
+
try {
|
|
715
|
+
await step(`${args.owners?.get(sink) ?? label(sink)}:telemetry`, () =>
|
|
716
|
+
sink.flush!(deadline.signal),
|
|
717
|
+
)
|
|
718
|
+
} catch (err) {
|
|
719
|
+
// A flush failure can never break the run — but it still costs
|
|
720
|
+
// this sink its export, so say so rather than dropping it.
|
|
721
|
+
warn?.(`[vx] telemetry sink '${label(sink)}' failed to flush: ${thrownMessage(err)}`)
|
|
722
|
+
}
|
|
723
|
+
}),
|
|
660
724
|
)
|
|
661
|
-
if (!
|
|
725
|
+
if (!(await settleWithin(all, ms))) {
|
|
662
726
|
deadline.abort(new Error(`telemetry flush deadline (${ms}ms)`))
|
|
663
727
|
warn?.(`[vx] telemetry flush timed out after ${ms}ms; buffered records lost`)
|
|
728
|
+
// A sink told of the abort says what it lost as its request ends; the
|
|
729
|
+
// run ended first and that line was lost (vx-otel, on CI). Briefly.
|
|
730
|
+
await settleWithin(all, ABORT_GRACE_MS)
|
|
664
731
|
}
|
|
665
732
|
},
|
|
666
733
|
}
|
|
@@ -704,9 +771,11 @@ export function taskTelemetryOf(o: TaskOutcome): TaskTelemetry {
|
|
|
704
771
|
if (o.cpuMs !== undefined) t.cpuMs = o.cpuMs
|
|
705
772
|
if (o.peakRssBytes !== undefined) t.peakRssBytes = o.peakRssBytes
|
|
706
773
|
if (o.where !== undefined) t.where = o.where
|
|
774
|
+
if (o.carried === true) t.carried = true
|
|
707
775
|
if (o.outputs !== undefined) t.outputs = o.outputs
|
|
708
776
|
if (o.attempts !== undefined) t.attempts = o.attempts
|
|
709
777
|
if (o.blockedBy !== undefined) t.blockedBy = o.blockedBy
|
|
778
|
+
if (o.otherPlatform !== undefined) t.otherPlatform = o.otherPlatform
|
|
710
779
|
if (o.timedOut === true) t.timedOut = true
|
|
711
780
|
if (o.sandboxViolations !== undefined) t.sandboxViolations = o.sandboxViolations
|
|
712
781
|
if (o.notReady !== undefined) t.notReady = o.notReady
|
|
@@ -730,3 +799,9 @@ export function taskTelemetryOf(o: TaskOutcome): TaskTelemetry {
|
|
|
730
799
|
if (o.wallclockEndNs !== undefined) t.wallclockEndNs = o.wallclockEndNs.toString()
|
|
731
800
|
return t
|
|
732
801
|
}
|
|
802
|
+
|
|
803
|
+
/** The stage table's start, when a stage was marked. */
|
|
804
|
+
function originOf(): { originAt?: number } {
|
|
805
|
+
const first = stageTimes()[0]
|
|
806
|
+
return first === undefined ? {} : { originAt: first.startedAt }
|
|
807
|
+
}
|
|
@@ -24,6 +24,21 @@ export function keyUpstream(node: TaskNode, upstream: TaskOutcome[]): TaskOutcom
|
|
|
24
24
|
return node.excludedUpstream === undefined ? keyed : [...keyed, ...node.excludedUpstream]
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
+
/** The ids some task of the run depends on; the rest run with `noDependants`. */
|
|
28
|
+
export function dependedOn(nodes: Iterable<TaskNode>): Set<string> {
|
|
29
|
+
const ids = new Set<string>()
|
|
30
|
+
for (const n of nodes) for (const d of n.deps) ids.add(d)
|
|
31
|
+
return ids
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* A server's key is read only by a dependant (A-17), so with none in the
|
|
36
|
+
* run it derives none (`noDependants`), and the plan must not show one.
|
|
37
|
+
*/
|
|
38
|
+
export function keylessServer(node: TaskNode, depended: ReadonlySet<string>): boolean {
|
|
39
|
+
return node.config.exec?.persistent !== undefined && !depended.has(node.id)
|
|
40
|
+
}
|
|
41
|
+
|
|
27
42
|
/** A dependency as a key reads it: only `node` and `hash` matter. */
|
|
28
43
|
export function keyedOutcome(node: TaskNode, hash: string | undefined): TaskOutcome {
|
|
29
44
|
const o: TaskOutcome = { node, status: 'success', exitCode: 0, durationMs: 0 }
|
package/src/report/events.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
import { exitSignal } from '../exec/index.js'
|
|
10
10
|
import type { InputChanges, TaskNode, TaskOutcome } from '../graph/index.js'
|
|
11
11
|
import { isGroupTask } from '../graph/index.js'
|
|
12
|
-
import { maskedCommand, secretMask } from '../util/index.js'
|
|
12
|
+
import { heldSecrets, maskedCommand, type SecretMask, secretMask } from '../util/index.js'
|
|
13
13
|
import type { Logger } from './logger.js'
|
|
14
14
|
import type { RunContext } from './summary.js'
|
|
15
15
|
|
|
@@ -122,10 +122,19 @@ export function busLogger(bus: EventBus): Logger {
|
|
|
122
122
|
let done = false
|
|
123
123
|
// A status line carries plugin warnings, and a remote layer's warning
|
|
124
124
|
// quotes the server's reply, which may echo what it was sent (L-40).
|
|
125
|
-
|
|
125
|
+
// Built again once a plugin's keychain answered: its token is in no env.
|
|
126
|
+
let secrets: SecretMask | null = null
|
|
127
|
+
let built = -1
|
|
128
|
+
const masked = (line: string): string => {
|
|
129
|
+
if (built !== heldSecrets()) {
|
|
130
|
+
built = heldSecrets()
|
|
131
|
+
secrets = secretMask([process.env])
|
|
132
|
+
}
|
|
133
|
+
return secrets?.mask(line) ?? line
|
|
134
|
+
}
|
|
126
135
|
return {
|
|
127
|
-
status: (line) => bus.emit({ kind: 'run:status', line:
|
|
128
|
-
warn: (line) => bus.emit({ kind: 'run:status', line:
|
|
136
|
+
status: (line) => bus.emit({ kind: 'run:status', line: masked(line) }),
|
|
137
|
+
warn: (line) => bus.emit({ kind: 'run:status', line: masked(line), warn: true }),
|
|
129
138
|
taskStdout: (node, chunk) => bus.emit({ kind: 'task:stdout', node, chunk }),
|
|
130
139
|
taskStderr: (node, chunk) => bus.emit({ kind: 'task:stderr', node, chunk }),
|
|
131
140
|
taskComplete: (node, outcome) => bus.emit({ kind: 'task:complete', node, outcome }),
|
|
@@ -212,7 +221,7 @@ export function terminalSubscriber(sink: Logger): RunEventSubscriber {
|
|
|
212
221
|
export function wireForwarder(send: (event: WireEvent) => void): RunEventSubscriber {
|
|
213
222
|
// Which tasks the consumer has been told about. A skipped task never
|
|
214
223
|
// reaches the scheduler's onStart, so its completion arrives with no
|
|
215
|
-
// preceding task:start — and
|
|
224
|
+
// preceding task:start — and a consumer resolves a completion's
|
|
216
225
|
// node from the start it recorded, so it would drop the task entirely
|
|
217
226
|
// while the forwarded footer still counted it. Synthesize the start here,
|
|
218
227
|
// where the live TaskNode is in hand, so the projection stays full
|
|
@@ -292,6 +301,8 @@ export interface OutcomeView {
|
|
|
292
301
|
/** A skipped task: the failed (or aborted) task at the root of the block
|
|
293
302
|
* (see `TaskOutcome`); absent for fail-fast's skip, which nothing blocked. */
|
|
294
303
|
blockedBy?: string
|
|
304
|
+
/** A skipped task whose `exec.platforms` leave out this machine (see `TaskOutcome`). */
|
|
305
|
+
otherPlatform?: { readonly needs: readonly string[]; readonly host: string }
|
|
295
306
|
/** How long an `admit` policy held the task with a worker free (see `TaskOutcome`). */
|
|
296
307
|
admissionHeldMs?: number
|
|
297
308
|
/** How long the task waited ready for a worker (see `TaskOutcome`). */
|
|
@@ -319,7 +330,8 @@ export interface OutcomeView {
|
|
|
319
330
|
* header and footer, the focused one-liner, the hash-only audit line,
|
|
320
331
|
* the verbose summary table: `success` / `restored-local` /
|
|
321
332
|
* `restored-remote` / `up-to-date` / `failed` / `skipped`. A hit that
|
|
322
|
-
* materialized nothing is up-to-date
|
|
333
|
+
* materialized nothing is up-to-date; a remote hit always restored
|
|
334
|
+
* (`hitOutcome`), so it is never up to date.
|
|
323
335
|
*/
|
|
324
336
|
export function outcomeWord(o: Pick<OutcomeView, 'status' | 'restored'>): string {
|
|
325
337
|
switch (o.status) {
|
|
@@ -344,26 +356,32 @@ export function outcomeLabel(
|
|
|
344
356
|
| 'notReady'
|
|
345
357
|
| 'inputMoved'
|
|
346
358
|
| 'blockedBy'
|
|
359
|
+
| 'otherPlatform'
|
|
347
360
|
>,
|
|
348
361
|
): string {
|
|
349
362
|
if (o.status === 'failed')
|
|
350
363
|
return failedLabel(o.exitCode, o.timedOut, o.sandboxViolations, o.notReady, o.inputMoved)
|
|
351
|
-
if (o.status === 'skipped') return skippedLabel(o
|
|
364
|
+
if (o.status === 'skipped') return skippedLabel(o)
|
|
352
365
|
return outcomeWord(o)
|
|
353
366
|
}
|
|
354
367
|
|
|
355
368
|
/**
|
|
356
|
-
* Why a task was skipped, one copy for every surface: `
|
|
357
|
-
*
|
|
369
|
+
* Why a task was skipped, one copy for every surface: `needs darwin-arm64,
|
|
370
|
+
* this is linux-x64` (its `exec.platforms`), `blocked by lib#build` (the
|
|
371
|
+
* task at the root of the block), or '' for fail-fast's skip — the run stopped,
|
|
358
372
|
* nothing blocked it, and a label claiming an upstream failure would lie.
|
|
359
373
|
*/
|
|
360
|
-
export function skippedReason(
|
|
361
|
-
|
|
374
|
+
export function skippedReason(o: SkipView = {}): string {
|
|
375
|
+
if (o.otherPlatform !== undefined)
|
|
376
|
+
return `needs ${o.otherPlatform.needs.join(' or ')}, this is ${o.otherPlatform.host}`
|
|
377
|
+
return o.blockedBy === undefined ? '' : `blocked by ${o.blockedBy}`
|
|
362
378
|
}
|
|
363
379
|
|
|
380
|
+
type SkipView = Pick<OutcomeView, 'blockedBy' | 'otherPlatform'>
|
|
381
|
+
|
|
364
382
|
/** `skipped (blocked by lib#build)`, or a bare `skipped` for fail-fast's. */
|
|
365
|
-
function skippedLabel(
|
|
366
|
-
const why = skippedReason(
|
|
383
|
+
function skippedLabel(o: SkipView): string {
|
|
384
|
+
const why = skippedReason(o)
|
|
367
385
|
return why === '' ? 'skipped' : `skipped (${why})`
|
|
368
386
|
}
|
|
369
387
|
|
|
@@ -448,6 +466,7 @@ export function projectOutcome(outcome: TaskOutcome): OutcomeView {
|
|
|
448
466
|
if (outcome.notReady !== undefined) view.notReady = outcome.notReady
|
|
449
467
|
if (outcome.inputMoved !== undefined) view.inputMoved = outcome.inputMoved
|
|
450
468
|
if (outcome.blockedBy !== undefined) view.blockedBy = outcome.blockedBy
|
|
469
|
+
if (outcome.otherPlatform !== undefined) view.otherPlatform = outcome.otherPlatform
|
|
451
470
|
if (outcome.restored !== undefined) view.restored = outcome.restored
|
|
452
471
|
if (outcome.sandboxViolations !== undefined) view.sandboxViolations = outcome.sandboxViolations
|
|
453
472
|
if (outcome.sandboxViolationLines !== undefined)
|
package/src/report/forecast.ts
CHANGED
|
@@ -10,16 +10,23 @@ export interface LiveForecast {
|
|
|
10
10
|
etaMs?: number
|
|
11
11
|
/** The running task at the head of the longest unfinished chain. */
|
|
12
12
|
critical?: string
|
|
13
|
-
/**
|
|
14
|
-
|
|
13
|
+
/** Tasks not started because a dependency has not finished: the tasks line's `blocked`. */
|
|
14
|
+
waiting: number
|
|
15
15
|
/** Each running task's p50, where history knows it: the row's expected duration. */
|
|
16
16
|
expected?: ReadonlyMap<string, number>
|
|
17
17
|
/**
|
|
18
18
|
* Tasks not started because a dependency has not finished, in the order
|
|
19
19
|
* the scheduler would start them, at most one per worker: what an idle
|
|
20
|
-
* worker row holds (
|
|
20
|
+
* worker row holds (2026-10-15).
|
|
21
21
|
*/
|
|
22
22
|
blocked?: readonly BlockedTask[]
|
|
23
|
+
/**
|
|
24
|
+
* Each running task's count of the tasks not started that depend on it
|
|
25
|
+
* directly (through a group, which runs nothing): its row's `blocked`
|
|
26
|
+
* cell, why they are not running. A task waiting on two running tasks
|
|
27
|
+
* counts under both.
|
|
28
|
+
*/
|
|
29
|
+
blockedBy: ReadonlyMap<string, number>
|
|
23
30
|
}
|
|
24
31
|
|
|
25
32
|
/** A task an unfinished dependency holds back, and what holds it: the running task its longest chain waits on, else that dependency. */
|
|
@@ -34,16 +41,13 @@ export type Forecast = (
|
|
|
34
41
|
nowMs: number,
|
|
35
42
|
) => LiveForecast
|
|
36
43
|
|
|
37
|
-
/** How far past its p50 a running task may go before the forecast gives up. */
|
|
38
|
-
const OVERRUN = 1.25
|
|
39
|
-
|
|
40
44
|
/**
|
|
41
45
|
* The later of the longest unfinished chain and the unfinished work spread
|
|
42
46
|
* over the workers. A task still to run costs its p50 (a pending task may
|
|
43
47
|
* yet hit, so this leans long, and the hits finish first); a running one
|
|
44
48
|
* costs what its p50 has left, never below zero; a group or a task with no
|
|
45
|
-
* history costs nothing.
|
|
46
|
-
*
|
|
49
|
+
* history costs nothing. Zero while tasks still run means the run is past
|
|
50
|
+
* its estimate: the live region counts the time since (2026-10-27).
|
|
47
51
|
*/
|
|
48
52
|
export function createForecast(
|
|
49
53
|
nodes: ReadonlyMap<string, TaskNode>,
|
|
@@ -84,37 +88,31 @@ export function createForecast(
|
|
|
84
88
|
// graph order: once per run, as the scheduler computes it.
|
|
85
89
|
const rank = computeReverseDepCount(nodes)
|
|
86
90
|
const position = new Map(order.map((n, i) => [n.id, i]))
|
|
91
|
+
// The tasks each node waits on directly, a group's own looked through.
|
|
92
|
+
const direct = new Map<string, readonly string[]>()
|
|
93
|
+
const through = (id: string, into: Set<string>, seen: Set<string>): void => {
|
|
94
|
+
if (seen.has(id)) return
|
|
95
|
+
seen.add(id)
|
|
96
|
+
const d = nodes.get(id)
|
|
97
|
+
if (d === undefined || !isGroupTask(d)) into.add(id)
|
|
98
|
+
else for (const x of d.deps) through(x, into, seen)
|
|
99
|
+
}
|
|
100
|
+
for (const n of order) {
|
|
101
|
+
if (isGroupTask(n) || n.deps.length === 0) continue
|
|
102
|
+
const into = new Set<string>()
|
|
103
|
+
const seen = new Set<string>()
|
|
104
|
+
for (const d of n.deps) through(d, into, seen)
|
|
105
|
+
direct.set(n.id, [...into])
|
|
106
|
+
}
|
|
87
107
|
const before = (a: string, b: string): boolean => {
|
|
88
108
|
const d = (rank.get(a) ?? 0) - (rank.get(b) ?? 0)
|
|
89
109
|
return d !== 0 ? d > 0 : position.get(a)! < position.get(b)!
|
|
90
110
|
}
|
|
91
111
|
|
|
92
|
-
const dependents = new Map<string, string[]>()
|
|
93
|
-
for (const n of order)
|
|
94
|
-
for (const d of n.deps) {
|
|
95
|
-
const list = dependents.get(d)
|
|
96
|
-
if (list === undefined) dependents.set(d, [n.id])
|
|
97
|
-
else list.push(n.id)
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
const blocked = (id: string, finished: ReadonlySet<string>): number => {
|
|
101
|
-
const seen = new Set<string>()
|
|
102
|
-
const stack = [id]
|
|
103
|
-
while (stack.length > 0) {
|
|
104
|
-
for (const next of dependents.get(stack.pop()!) ?? []) {
|
|
105
|
-
if (seen.has(next) || finished.has(next)) continue
|
|
106
|
-
seen.add(next)
|
|
107
|
-
stack.push(next)
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
return seen.size
|
|
111
|
-
}
|
|
112
|
-
|
|
113
112
|
return (finished, running, nowMs) => {
|
|
114
|
-
const blocks = new Map<string, number>()
|
|
115
113
|
const expected = new Map<string, number>()
|
|
114
|
+
let waiting = 0
|
|
116
115
|
for (const id of running.keys()) {
|
|
117
|
-
blocks.set(id, blocked(id, finished))
|
|
118
116
|
const p = cost.get(id)
|
|
119
117
|
if (p !== undefined) expected.set(id, p)
|
|
120
118
|
}
|
|
@@ -125,9 +123,9 @@ export function createForecast(
|
|
|
125
123
|
let chainEnd: string | undefined
|
|
126
124
|
let work = 0
|
|
127
125
|
let known = false
|
|
128
|
-
let overrun = false
|
|
129
126
|
// The best `workers` blocked tasks, kept sorted as the pass meets them.
|
|
130
127
|
const held: BlockedTask[] = []
|
|
128
|
+
const blockedBy = new Map<string, number>()
|
|
131
129
|
for (const n of order) {
|
|
132
130
|
if (finished.has(n.id)) continue
|
|
133
131
|
let start = 0
|
|
@@ -145,6 +143,9 @@ export function createForecast(
|
|
|
145
143
|
const by =
|
|
146
144
|
since === undefined ? (head.get(n.id) ?? n.deps.find((d) => end.has(d))) : undefined
|
|
147
145
|
if (by !== undefined && !isGroupTask(n)) {
|
|
146
|
+
waiting++
|
|
147
|
+
for (const on of direct.get(n.id) ?? [])
|
|
148
|
+
if (running.has(on)) blockedBy.set(on, (blockedBy.get(on) ?? 0) + 1)
|
|
148
149
|
let at = held.length
|
|
149
150
|
while (at > 0 && before(n.id, held[at - 1]!.id)) at--
|
|
150
151
|
if (at < workers) {
|
|
@@ -156,7 +157,6 @@ export function createForecast(
|
|
|
156
157
|
let own = 0
|
|
157
158
|
if (p !== undefined) {
|
|
158
159
|
known = true
|
|
159
|
-
if (since !== undefined && nowMs - since > p * OVERRUN) overrun = true
|
|
160
160
|
own = since === undefined ? p : Math.max(0, p - (nowMs - since))
|
|
161
161
|
}
|
|
162
162
|
const e = start + own
|
|
@@ -167,14 +167,15 @@ export function createForecast(
|
|
|
167
167
|
}
|
|
168
168
|
work += own
|
|
169
169
|
}
|
|
170
|
-
if (!known
|
|
170
|
+
if (!known) return { waiting, expected, blocked: held, blockedBy }
|
|
171
171
|
const critical = chainEnd === undefined ? undefined : head.get(chainEnd)
|
|
172
172
|
return {
|
|
173
173
|
etaMs: Math.max(chain, work / workers),
|
|
174
174
|
...(critical !== undefined ? { critical } : {}),
|
|
175
|
-
|
|
175
|
+
waiting,
|
|
176
176
|
expected,
|
|
177
177
|
blocked: held,
|
|
178
|
+
blockedBy,
|
|
178
179
|
}
|
|
179
180
|
}
|
|
180
181
|
}
|