@vzn/vx 0.0.637 → 0.0.639
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/caching.md +84 -28
- package/docs/cli.md +273 -408
- package/docs/execution.md +19 -44
- package/docs/features.md +13 -15
- package/docs/patterns.md +5 -7
- package/docs/schema.md +68 -61
- package/package.json +5 -5
- package/schemas/last.json +7 -4
- package/schemas/plan.json +2 -2
- package/skills/vx/SKILL.md +19 -6
- package/src/bin.ts +13 -7
- package/src/cache/cache.ts +60 -11
- package/src/cache/git-inputs.ts +6 -0
- package/src/cache/index.ts +10 -2
- package/src/cache/inputs.ts +0 -24
- package/src/cache/layer.ts +32 -12
- package/src/cache/layered-cache.ts +112 -315
- package/src/cache/plugin-cache.ts +240 -0
- package/src/cache/policy.ts +9 -0
- package/src/cache/remote-lane.ts +383 -0
- package/src/cache/run-history.ts +92 -8
- package/src/cache/run-store.ts +77 -0
- package/src/cache/schema.ts +44 -13
- package/src/cache/scoped-cache.ts +22 -2
- package/src/cli/cache.ts +33 -16
- package/src/cli/completions.ts +11 -7
- package/src/cli/foreign-flags.ts +10 -22
- package/src/cli/help.ts +22 -38
- package/src/cli/index.ts +25 -11
- package/src/cli/info.ts +8 -6
- package/src/cli/init.ts +14 -15
- package/src/cli/last.ts +20 -10
- package/src/cli/lock.ts +39 -27
- package/src/cli/plan-format.ts +10 -61
- package/src/cli/plugin-commands.ts +77 -6
- package/src/cli/plugin-templates.ts +1 -1
- package/src/cli/run.ts +31 -191
- package/src/cli/select.ts +18 -9
- package/src/cli/show.ts +72 -40
- package/src/cli/task-verb.ts +4 -4
- package/src/cli/upgrade.ts +132 -39
- package/src/cli/watch-cycle.ts +2 -1
- package/src/cli/watch-filter.ts +7 -2
- package/src/cli/watch-judge.ts +6 -1
- package/src/cli/watch.ts +73 -34
- package/src/cli/why.ts +14 -5
- package/src/cli/workspace-config.ts +2 -2
- package/src/config.ts +0 -6
- package/src/exec/executor.ts +1 -1
- package/src/exec/sandbox-runtime.ts +1 -1
- package/src/exec/sandbox-violations.ts +3 -1
- package/src/graph/index.ts +9 -1
- package/src/graph/priorities.ts +1 -1
- package/src/graph/scheduler.ts +12 -16
- package/src/graph/task-graph.ts +119 -21
- package/src/graph/task-status.ts +95 -0
- package/src/{orchestrator → history}/failure-mode.ts +20 -11
- package/src/{orchestrator → history}/history.ts +61 -5
- package/src/history/index.ts +48 -0
- package/src/{orchestrator → history}/metrics.ts +14 -23
- package/src/{orchestrator → history}/miss-reason.ts +1 -1
- package/src/index.ts +21 -29
- package/src/orchestrator/affected-tasks.ts +8 -3
- package/src/orchestrator/doctor.ts +2 -2
- package/src/orchestrator/execute-args.ts +80 -0
- package/src/orchestrator/execute-task.ts +47 -429
- package/src/orchestrator/fingerprint-watch.ts +2 -2
- package/src/orchestrator/index.ts +3 -74
- package/src/orchestrator/miss-save.ts +33 -3
- package/src/orchestrator/options.ts +1 -36
- package/src/orchestrator/output-host.ts +4 -3
- package/src/orchestrator/persistent-task.ts +321 -0
- package/src/orchestrator/persistent.ts +130 -3
- package/src/orchestrator/placement.ts +3 -3
- package/src/orchestrator/plan.ts +18 -5
- package/src/orchestrator/plugin-host.ts +77 -47
- package/src/orchestrator/plugin.ts +86 -14
- package/src/orchestrator/prepare.ts +74 -31
- package/src/orchestrator/projects.ts +3 -0
- package/src/orchestrator/remote-prefetch.ts +23 -6
- package/src/orchestrator/run-artifacts.ts +5 -177
- package/src/orchestrator/run-context.ts +55 -0
- package/src/orchestrator/run-failures.ts +15 -3
- package/src/orchestrator/run-records.ts +36 -4
- package/src/orchestrator/run-report.ts +81 -179
- package/src/orchestrator/run-settings.ts +1 -16
- package/src/orchestrator/run-stop.ts +153 -0
- package/src/orchestrator/run.ts +313 -555
- package/src/orchestrator/sandbox-fix.ts +0 -16
- package/src/orchestrator/signals.ts +14 -6
- package/src/orchestrator/stable-keys.ts +107 -3
- package/src/orchestrator/steps.ts +50 -0
- package/src/orchestrator/task-hash.ts +3 -16
- package/src/orchestrator/task-log-buffer.ts +1 -1
- package/src/orchestrator/task-process.ts +94 -0
- package/src/orchestrator/telemetry-host.ts +1 -1
- package/src/orchestrator/telemetry.ts +6 -83
- package/src/{orchestrator → report}/events.ts +41 -8
- package/src/{orchestrator → report}/forecast.ts +35 -2
- package/src/{orchestrator → report}/framed-output.ts +39 -34
- package/src/report/index.ts +36 -0
- package/src/{orchestrator → report}/logger.ts +166 -21
- package/src/{orchestrator → report}/status-line.ts +84 -7
- package/src/{orchestrator → report}/summary.ts +179 -63
- package/src/report/tally.ts +83 -0
- package/src/settings.ts +13 -23
- package/src/{orchestrator → util}/colors.ts +49 -4
- package/src/util/config-frame.ts +31 -2
- package/src/util/errors.ts +32 -0
- package/src/util/git-ignored.ts +143 -0
- package/src/util/index.ts +25 -3
- package/src/util/refusal-notice.ts +111 -0
- package/src/util/report.ts +39 -0
- package/src/util/settle.ts +14 -39
- package/src/util/tail.ts +1 -1
- package/src/util/ui.ts +197 -0
- package/src/workspace/affected.ts +20 -23
- package/src/workspace/dependency-rules.ts +12 -4
- package/src/workspace/migration.ts +87 -91
- package/src/workspace/workspace.ts +5 -4
- package/schemas/summary.json +0 -133
- package/src/cache/chained-cache.ts +0 -276
- package/src/orchestrator/tally.ts +0 -112
- /package/src/{orchestrator → report}/path-links.ts +0 -0
- /package/src/{orchestrator → report}/plain-output.ts +0 -0
- /package/src/{orchestrator → report}/terminal-signals.ts +0 -0
|
@@ -19,6 +19,12 @@ import {
|
|
|
19
19
|
parseCachePolicy,
|
|
20
20
|
GitFilesCache,
|
|
21
21
|
LayeredCache,
|
|
22
|
+
absentPluginCache,
|
|
23
|
+
pluginCache,
|
|
24
|
+
putRunEntry,
|
|
25
|
+
writesRunStore,
|
|
26
|
+
type PluginCache,
|
|
27
|
+
type ResolvedRemote,
|
|
22
28
|
ScopedCache,
|
|
23
29
|
applyGitEnumeration,
|
|
24
30
|
gitPathspecs,
|
|
@@ -72,9 +78,10 @@ import { applyGraphStage } from './graph-stage.js'
|
|
|
72
78
|
import { keyExcludedDependencies } from './excluded-keys.js'
|
|
73
79
|
import { FingerprintWatch } from './fingerprint-watch.js'
|
|
74
80
|
import type { VxPlugin } from './plugin.js'
|
|
81
|
+
import { askIgnoredOutputs, gatedOutputs } from './stable-keys.js'
|
|
75
82
|
import { currentBranch } from './run-context.js'
|
|
76
83
|
import { createHashCache, type HashCache } from './task-hash.js'
|
|
77
|
-
import type
|
|
84
|
+
import { type Logger, warnOf } from '../report/index.js'
|
|
78
85
|
import type { RunOptions } from './options.js'
|
|
79
86
|
import { refuseFiltersThroughGroups } from './upstream.js'
|
|
80
87
|
import { runRules } from '../settings.js'
|
|
@@ -116,6 +123,13 @@ export interface PreparedRun {
|
|
|
116
123
|
* Learned on first ask, once per run.
|
|
117
124
|
*/
|
|
118
125
|
branch: () => string | null
|
|
126
|
+
/** Each plugin's `ctx.cache`: the run's local store and remotes, under its policy and scope. */
|
|
127
|
+
cacheFor: (plugin: string) => PluginCache
|
|
128
|
+
/**
|
|
129
|
+
* Keep `bytes` of this run under `name` (run-store.ts); undefined when no
|
|
130
|
+
* remote takes the run's writes, and so core keeps nothing.
|
|
131
|
+
*/
|
|
132
|
+
keepRunEntry: ((name: string, bytes: Uint8Array) => Promise<boolean>) | undefined
|
|
119
133
|
nodes: Map<string, TaskNode>
|
|
120
134
|
/**
|
|
121
135
|
* The tasks `--exclude-dependencies` took out of the schedule: keyed as a
|
|
@@ -228,9 +242,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
228
242
|
(reused !== undefined ? gitOfDiscovery(reused.projects) : undefined) ??
|
|
229
243
|
lazyGitEnumeration(workspaceRoot)
|
|
230
244
|
if (wholeTree) void git.start()
|
|
231
|
-
const { workspaceConfig, plugins } = await loadWorkspacePlugins(workspaceRoot, (
|
|
232
|
-
log.status(m),
|
|
233
|
-
)
|
|
245
|
+
const { workspaceConfig, plugins } = await loadWorkspacePlugins(workspaceRoot, warnOf(log))
|
|
234
246
|
mark('workspace config')
|
|
235
247
|
let learnedBranch: string | null | undefined
|
|
236
248
|
const branch = (): string | null =>
|
|
@@ -250,14 +262,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
250
262
|
const projectMetas =
|
|
251
263
|
reused !== undefined
|
|
252
264
|
? reused.projects
|
|
253
|
-
: await discoverProjects(
|
|
254
|
-
workspaceRoot,
|
|
255
|
-
readLock,
|
|
256
|
-
plugins,
|
|
257
|
-
cacheDir,
|
|
258
|
-
(m) => log.status(m),
|
|
259
|
-
git,
|
|
260
|
-
)
|
|
265
|
+
: await discoverProjects(workspaceRoot, readLock, plugins, cacheDir, warnOf(log), git)
|
|
261
266
|
mark('discover projects')
|
|
262
267
|
|
|
263
268
|
// SCOPED config loading: configs are programs, and evaluating 1090
|
|
@@ -355,7 +360,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
355
360
|
projectMetas,
|
|
356
361
|
packageGraph,
|
|
357
362
|
configs: readLock,
|
|
358
|
-
warn: (
|
|
363
|
+
warn: warnOf(log),
|
|
359
364
|
}
|
|
360
365
|
let loaded: LoadedProjects
|
|
361
366
|
try {
|
|
@@ -451,26 +456,33 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
451
456
|
// winning prevents double-wrapping when the workspace also declares a
|
|
452
457
|
// cache plugin.
|
|
453
458
|
let cache: CacheLayer
|
|
459
|
+
const remotes: ResolvedRemote[] = []
|
|
460
|
+
if (options.remoteCache) {
|
|
461
|
+
remotes.push({
|
|
462
|
+
remote: options.remoteCache,
|
|
463
|
+
read: policy.remoteRead,
|
|
464
|
+
write: policy.remoteWrite,
|
|
465
|
+
onError: (err) => warnOf(log)(`[vx] remote cache: ${err.message}`),
|
|
466
|
+
})
|
|
467
|
+
}
|
|
454
468
|
try {
|
|
455
469
|
cache = options.remoteCache
|
|
456
|
-
? new LayeredCache(localCache,
|
|
457
|
-
policy,
|
|
458
|
-
onRemoteError: (err) => log.status(`[vx] remote cache: ${err.message}`),
|
|
459
|
-
})
|
|
470
|
+
? new LayeredCache(localCache, remotes)
|
|
460
471
|
: await resolveCache(plugins, {
|
|
461
472
|
workspaceRoot,
|
|
462
473
|
cacheDir,
|
|
463
|
-
warn: (
|
|
474
|
+
warn: warnOf(log),
|
|
464
475
|
scope: cacheScope,
|
|
465
476
|
localCache,
|
|
466
477
|
policy,
|
|
478
|
+
remotes,
|
|
467
479
|
})
|
|
468
480
|
} catch (err) {
|
|
469
481
|
// A cache plugin that throws or returns something off-contract is
|
|
470
482
|
// refused by name; the local handle opened above must not leak with it,
|
|
471
483
|
// nor what another plugin's factory opened: the factories have run
|
|
472
484
|
// (C-4, item 1029's contract).
|
|
473
|
-
await teardownPlugins(plugins, (
|
|
485
|
+
await teardownPlugins(plugins, warnOf(log))
|
|
474
486
|
localCache.close()
|
|
475
487
|
throw err
|
|
476
488
|
}
|
|
@@ -509,8 +521,10 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
509
521
|
localCache,
|
|
510
522
|
hasRemoteLayer,
|
|
511
523
|
cachePolicy: policy,
|
|
524
|
+
keepRunEntry: undefined,
|
|
512
525
|
priorities: () => Promise.resolve(new Map()),
|
|
513
526
|
branch,
|
|
527
|
+
cacheFor: () => absentPluginCache,
|
|
514
528
|
nodes: new Map(),
|
|
515
529
|
keyOnly: new Map(),
|
|
516
530
|
unresolvedTasks,
|
|
@@ -560,7 +574,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
560
574
|
workspaceRoot,
|
|
561
575
|
cacheDir,
|
|
562
576
|
projectMetas,
|
|
563
|
-
warn: (
|
|
577
|
+
warn: warnOf(log),
|
|
564
578
|
rules: graphRules,
|
|
565
579
|
})
|
|
566
580
|
// `--affected` selects over the FINAL graph: an edge or an input a
|
|
@@ -657,11 +671,16 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
657
671
|
if ((n.config.cache?.inputs.workspaceFiles?.length ?? 0) > 0) usesWorkspaceInputs = true
|
|
658
672
|
}
|
|
659
673
|
const projectDirs = [...graphDirs]
|
|
660
|
-
const
|
|
674
|
+
const enumerating =
|
|
675
|
+
git.started ??
|
|
661
676
|
startGitEnumeration(
|
|
662
677
|
workspaceRoot,
|
|
663
678
|
gitPathspecs(workspaceRoot, projectDirs, usesWorkspaceInputs),
|
|
664
|
-
)
|
|
679
|
+
)
|
|
680
|
+
// While git walks: the gate's pass over every task cost 2-8 ms at
|
|
681
|
+
// 1,000 projects when it ran before the spawn.
|
|
682
|
+
const settleIgnored = askIgnoredOutputs(gatedOutputs(nodes, workspaceRoot), workspaceRoot)
|
|
683
|
+
const enumeration = await enumerating
|
|
665
684
|
await applyGitEnumeration(
|
|
666
685
|
enumeration,
|
|
667
686
|
workspaceRoot,
|
|
@@ -670,6 +689,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
670
689
|
usesWorkspaceInputs,
|
|
671
690
|
localCache,
|
|
672
691
|
)
|
|
692
|
+
await settleIgnored(gitFilesCache)
|
|
673
693
|
mark('git enumeration')
|
|
674
694
|
if (hasGraphHook && !hooksFirst) await graphStage()
|
|
675
695
|
// Args after `--` go to requested commands only; with none among them
|
|
@@ -685,7 +705,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
685
705
|
}
|
|
686
706
|
}
|
|
687
707
|
if (hasHook(plugins, 'key')) {
|
|
688
|
-
await applyKeyHooks(plugins, nodes, { workspaceRoot, cacheDir, warn: (
|
|
708
|
+
await applyKeyHooks(plugins, nodes, { workspaceRoot, cacheDir, warn: warnOf(log) })
|
|
689
709
|
}
|
|
690
710
|
refuseFiltersThroughGroups(nodes)
|
|
691
711
|
const exclude = options.excludeDependencies
|
|
@@ -708,16 +728,35 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
708
728
|
})
|
|
709
729
|
}
|
|
710
730
|
}
|
|
731
|
+
const pluginCaches = new Map<string, PluginCache>()
|
|
732
|
+
const cacheFor = (plugin: string): PluginCache => {
|
|
733
|
+
let own = pluginCaches.get(plugin)
|
|
734
|
+
if (own === undefined) {
|
|
735
|
+
own = pluginCache(plugin, {
|
|
736
|
+
db: localCache.dbHandle(),
|
|
737
|
+
policy,
|
|
738
|
+
remotes,
|
|
739
|
+
scope: cacheScope,
|
|
740
|
+
})
|
|
741
|
+
pluginCaches.set(plugin, own)
|
|
742
|
+
}
|
|
743
|
+
return own
|
|
744
|
+
}
|
|
711
745
|
let scheduled: Promise<ReadonlyMap<string, number>> | undefined
|
|
712
746
|
const priorities = (): Promise<ReadonlyMap<string, number>> =>
|
|
713
747
|
(scheduled ??= hasHook(plugins, 'schedule')
|
|
714
|
-
? applyScheduleHooks(
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
748
|
+
? applyScheduleHooks(
|
|
749
|
+
plugins,
|
|
750
|
+
nodes,
|
|
751
|
+
{
|
|
752
|
+
workspaceRoot,
|
|
753
|
+
cacheDir,
|
|
754
|
+
warn: warnOf(log),
|
|
755
|
+
localCache,
|
|
756
|
+
branch: branch(),
|
|
757
|
+
},
|
|
758
|
+
cacheFor,
|
|
759
|
+
)
|
|
721
760
|
: Promise.resolve(new Map()))
|
|
722
761
|
|
|
723
762
|
return {
|
|
@@ -731,6 +770,10 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
731
770
|
cachePolicy: policy,
|
|
732
771
|
priorities,
|
|
733
772
|
branch,
|
|
773
|
+
cacheFor,
|
|
774
|
+
keepRunEntry: writesRunStore(remotes)
|
|
775
|
+
? (name, bytes) => putRunEntry(remotes, cacheScope, name, bytes)
|
|
776
|
+
: undefined,
|
|
734
777
|
nodes,
|
|
735
778
|
keyOnly,
|
|
736
779
|
unresolvedTasks,
|
|
@@ -746,7 +789,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
746
789
|
empty: nodes.size === 0 ? 'empty-graph' : null,
|
|
747
790
|
}
|
|
748
791
|
} catch (err) {
|
|
749
|
-
await teardownPlugins(plugins, (
|
|
792
|
+
await teardownPlugins(plugins, warnOf(log))
|
|
750
793
|
cache.close()
|
|
751
794
|
throw err
|
|
752
795
|
}
|
|
@@ -163,6 +163,8 @@ interface LoadProjectsBase {
|
|
|
163
163
|
*/
|
|
164
164
|
configs: () => Promise<Lockfile>
|
|
165
165
|
warn: (message: string) => void
|
|
166
|
+
/** `vx lock`'s audit: the `project` stage's `ctx.lock`. */
|
|
167
|
+
lock?: boolean
|
|
166
168
|
/**
|
|
167
169
|
* Entries a load in this same process already produced with the same
|
|
168
170
|
* cache dir and lock (the CLI's selection pass, which stages every
|
|
@@ -323,6 +325,7 @@ export async function loadProjects(args: LoadProjectsArgs): Promise<LoadedProjec
|
|
|
323
325
|
name: meta.name,
|
|
324
326
|
dir: meta.dir,
|
|
325
327
|
projects: args.projectMetas,
|
|
328
|
+
lock: args.lock === true,
|
|
326
329
|
},
|
|
327
330
|
(plugin) =>
|
|
328
331
|
validateProjectConfig(
|
|
@@ -12,8 +12,9 @@
|
|
|
12
12
|
// GET per key.
|
|
13
13
|
//
|
|
14
14
|
// HARD SCOPE: this runs ONLY when a remote layer is configured. It
|
|
15
|
-
//
|
|
16
|
-
//
|
|
15
|
+
// asks the local index only which keys it holds (no local get / no
|
|
16
|
+
// isOutputsCurrent), and probes the remote for the rest: a run the local
|
|
17
|
+
// cache answers makes no remote call (M, 2026-10-15). Local-only
|
|
17
18
|
// runs are byte-identical to before; the caller gates on the layer's
|
|
18
19
|
// own `hasRemote`, so a third-party remote layer gets the overlap too.
|
|
19
20
|
// The two batch-probe hooks are OPTIONAL on the contract: a remote layer
|
|
@@ -40,6 +41,11 @@ export interface PrefetchArgs {
|
|
|
40
41
|
concurrency: number
|
|
41
42
|
/** When false (remote reads disabled), no prefetch fires. */
|
|
42
43
|
remoteRead: boolean
|
|
44
|
+
/**
|
|
45
|
+
* With reads off and writes on (`--force`), the keys go to
|
|
46
|
+
* `probeBeforeUpload` instead, so a key a remote holds is not re-sent.
|
|
47
|
+
*/
|
|
48
|
+
remoteWrite: boolean
|
|
43
49
|
}
|
|
44
50
|
|
|
45
51
|
/**
|
|
@@ -54,7 +60,15 @@ export interface PrefetchArgs {
|
|
|
54
60
|
* is always correct.
|
|
55
61
|
*/
|
|
56
62
|
export function startRemotePrefetch(args: PrefetchArgs): Promise<void> {
|
|
57
|
-
if (!args.remoteRead)
|
|
63
|
+
if (!args.remoteRead) {
|
|
64
|
+
if (!args.remoteWrite || args.cache.probeBeforeUpload === undefined) return Promise.resolve()
|
|
65
|
+
const hashes = deriveStableKeys(args).then((keys) => [...new Set(keys.map((k) => k.hash))])
|
|
66
|
+
args.cache.probeBeforeUpload(hashes)
|
|
67
|
+
return hashes.then(
|
|
68
|
+
() => {},
|
|
69
|
+
() => {},
|
|
70
|
+
)
|
|
71
|
+
}
|
|
58
72
|
// Detached from EXECUTION: the caller does NOT await this before
|
|
59
73
|
// scheduling, so prefetch network latency overlaps the run. But the
|
|
60
74
|
// caller DOES await the returned handle before closing the cache, so
|
|
@@ -75,8 +89,11 @@ async function runPrefetch(args: PrefetchArgs): Promise<void> {
|
|
|
75
89
|
// would 404. When the remote can't batch (`null` — an older remote, reads
|
|
76
90
|
// disabled, or a layer that doesn't implement the hook at all), fall back to
|
|
77
91
|
// prefetching every stable key, exactly as before.
|
|
78
|
-
|
|
79
|
-
|
|
92
|
+
const held =
|
|
93
|
+
(await args.cache.localHasMany?.([...new Set(stableKeys.map((k) => k.hash))])) ?? null
|
|
94
|
+
let toPrefetch = held === null ? stableKeys : stableKeys.filter((k) => !held.has(k.hash))
|
|
95
|
+
if (toPrefetch.length === 0) return
|
|
96
|
+
const uniqueHashes = [...new Set(toPrefetch.map((k) => k.hash))]
|
|
80
97
|
const present = (await args.cache.remoteHasMany?.(uniqueHashes)) ?? null
|
|
81
98
|
if (present !== null) {
|
|
82
99
|
args.cache.markRemoteAbsent?.(uniqueHashes.filter((h) => !present.has(h)))
|
|
@@ -85,7 +102,7 @@ async function runPrefetch(args: PrefetchArgs): Promise<void> {
|
|
|
85
102
|
// economy: deleting it alone GETs nothing extra, deleting both GETs
|
|
86
103
|
// every absent key. What it decides is which hashes the pump asks
|
|
87
104
|
// the layer for, and that is what the row pins (item 643, 645).
|
|
88
|
-
toPrefetch =
|
|
105
|
+
toPrefetch = toPrefetch.filter((k) => present.has(k.hash))
|
|
89
106
|
if (toPrefetch.length === 0) return
|
|
90
107
|
}
|
|
91
108
|
|
|
@@ -1,182 +1,10 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// cpu_ms, peak_rss_bytes, hrtime spans).
|
|
6
|
-
// Default path: <cacheDir>/runs/<run_id>.json.
|
|
7
|
-
//
|
|
8
|
-
// --profile[=<path>] → Chrome-trace JSON of the run. Each task is
|
|
9
|
-
// a single complete event (`ph: 'X'`) with
|
|
10
|
-
// ts / dur in microseconds derived from the
|
|
11
|
-
// hrtime spans the runner already captures.
|
|
12
|
-
// Default path: profile.json (cwd-relative).
|
|
13
|
-
//
|
|
14
|
-
// Both are no-ops if the corresponding RunOptions field is undefined.
|
|
1
|
+
// `--profile[=<path>]`: Chrome-trace JSON of the run. Each task is a single
|
|
2
|
+
// complete event (`ph: 'X'`) with ts / dur in microseconds derived from the
|
|
3
|
+
// hrtime spans the runner already captures. Default path: profile.json
|
|
4
|
+
// (cwd-relative).
|
|
15
5
|
|
|
16
6
|
import path from 'node:path'
|
|
17
|
-
import {
|
|
18
|
-
import { tallyOutcomes, type Tally } from './tally.js'
|
|
19
|
-
import type { FlakyFinding } from './failure-mode.js'
|
|
20
|
-
import { isCacheHit } from './telemetry.js'
|
|
21
|
-
import { ranNoCache } from './events.js'
|
|
22
|
-
|
|
23
|
-
export interface SummarizeArgs {
|
|
24
|
-
/** Empty string → default path; otherwise the explicit file path. */
|
|
25
|
-
target: string
|
|
26
|
-
cacheDir: string
|
|
27
|
-
cwd: string
|
|
28
|
-
runId: string
|
|
29
|
-
startedAtMs: number
|
|
30
|
-
endedAtMs: number
|
|
31
|
-
totalMs: number
|
|
32
|
-
/** The run's verdict — the same value the CLI turns into the exit code. */
|
|
33
|
-
ok: boolean
|
|
34
|
-
/** The process's exit code where it is not `ok ? 0 : 1`: a stopping signal's. */
|
|
35
|
-
exitCode?: number
|
|
36
|
-
outcomes: readonly TaskOutcome[]
|
|
37
|
-
/** What `detectFlaky` found this run; each row's flaky note, typed. */
|
|
38
|
-
flaky?: readonly FlakyFinding[]
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
/** One task of `--summarize`: the wire, stated by `schemas/summary.json`. */
|
|
42
|
-
export interface SummaryTaskJson {
|
|
43
|
-
id: string
|
|
44
|
-
project: string
|
|
45
|
-
task: string
|
|
46
|
-
status: TaskOutcome['status']
|
|
47
|
-
exitCode: number
|
|
48
|
-
durationMs: number
|
|
49
|
-
hash: string | null
|
|
50
|
-
restored?: boolean
|
|
51
|
-
noCache?: true
|
|
52
|
-
flaky?: { passes: number; failures: number; attempts: number }
|
|
53
|
-
cpuMs?: number
|
|
54
|
-
peakRssBytes?: number
|
|
55
|
-
storedDurationMs?: number
|
|
56
|
-
storedCpuMs?: number
|
|
57
|
-
storedPeakRssBytes?: number
|
|
58
|
-
admissionHeldMs?: number
|
|
59
|
-
blockedBy?: string
|
|
60
|
-
timedOut?: true
|
|
61
|
-
sandboxViolations?: number
|
|
62
|
-
notReady?: 'timeout' | 'exited' | 'spawn'
|
|
63
|
-
wallclockStartNs?: string
|
|
64
|
-
wallclockEndNs?: string
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
/** The `--summarize` document. */
|
|
68
|
-
export interface RunSummaryJson {
|
|
69
|
-
runId: string
|
|
70
|
-
ok: boolean
|
|
71
|
-
exitCode: number
|
|
72
|
-
startedAt: string
|
|
73
|
-
endedAt: string
|
|
74
|
-
totalMs: number
|
|
75
|
-
/** What the hits skipped: the sum of their `storedDurationMs`. */
|
|
76
|
-
savedMs: number
|
|
77
|
-
tasks: SummaryTaskJson[]
|
|
78
|
-
aborted: SummaryTaskJson[]
|
|
79
|
-
summary: Tally
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
/** One task's entry. Shared by `tasks` and `aborted` so they read alike. */
|
|
83
|
-
function taskEntry(o: TaskOutcome, flaky?: FlakyFinding): SummaryTaskJson {
|
|
84
|
-
return {
|
|
85
|
-
id: o.node.id,
|
|
86
|
-
project: o.node.projectName,
|
|
87
|
-
task: o.node.taskName,
|
|
88
|
-
status: o.status,
|
|
89
|
-
exitCode: o.exitCode,
|
|
90
|
-
durationMs: o.durationMs,
|
|
91
|
-
hash: o.hash ?? null,
|
|
92
|
-
// On a hit: whether it restored outputs or found them up to date, as
|
|
93
|
-
// `summary`'s restoredLocal / restoredRemote / upToDate count it.
|
|
94
|
-
...(isCacheHit(o.status) ? { restored: o.restored === true } : {}),
|
|
95
|
-
// A task with no `cache` block executes every run by design; without
|
|
96
|
-
// the flag a consumer computing a hit rate cannot tell a miss from a
|
|
97
|
-
// task that could never hit (its `hash` is still set — dependents fold
|
|
98
|
-
// it). Present only when true, so every other row is byte-identical.
|
|
99
|
-
...(ranNoCache(o) ? { noCache: true } : {}),
|
|
100
|
-
// Present only when the run proved the task flaky: this exact key
|
|
101
|
-
// failed after it had passed (this run counted), or the task
|
|
102
|
-
// needed a retry. A consumer gating on `failed` can tell a break from a
|
|
103
|
-
// flake without the history; every other row is byte-identical.
|
|
104
|
-
...(flaky !== undefined
|
|
105
|
-
? { flaky: { passes: flaky.passes, failures: flaky.failures, attempts: flaky.attempts } }
|
|
106
|
-
: {}),
|
|
107
|
-
...(o.cpuMs !== undefined ? { cpuMs: o.cpuMs } : {}),
|
|
108
|
-
...(o.peakRssBytes !== undefined ? { peakRssBytes: o.peakRssBytes } : {}),
|
|
109
|
-
// A hit's row names what the PRODUCING execution used under its own
|
|
110
|
-
// keys, so a consumer never reads a remote worker's peak as this run's.
|
|
111
|
-
...(o.storedDurationMs !== undefined ? { storedDurationMs: o.storedDurationMs } : {}),
|
|
112
|
-
...(o.storedCpuMs !== undefined ? { storedCpuMs: o.storedCpuMs } : {}),
|
|
113
|
-
...(o.storedPeakRssBytes !== undefined ? { storedPeakRssBytes: o.storedPeakRssBytes } : {}),
|
|
114
|
-
// Present only when an `admit` policy held the task with a worker free.
|
|
115
|
-
...(o.admissionHeldMs !== undefined ? { admissionHeldMs: o.admissionHeldMs } : {}),
|
|
116
|
-
// A skipped row names the failed (or aborted) task at the root of what
|
|
117
|
-
// blocked it; a fail-fast skip has none.
|
|
118
|
-
...(o.blockedBy !== undefined ? { blockedBy: o.blockedBy } : {}),
|
|
119
|
-
// A failure vx's own `timeout` killed; absent on every other row.
|
|
120
|
-
...(o.timedOut === true ? { timedOut: true } : {}),
|
|
121
|
-
// The sandbox's violation count; present only on a task it recorded any for.
|
|
122
|
-
...(o.sandboxViolations !== undefined ? { sandboxViolations: o.sandboxViolations } : {}),
|
|
123
|
-
// A persistent task that never became ready, and why.
|
|
124
|
-
...(o.notReady !== undefined ? { notReady: o.notReady } : {}),
|
|
125
|
-
// hrtime spans are bigints → emit as strings so JSON.parse on
|
|
126
|
-
// the consumer side doesn't truncate the ns precision.
|
|
127
|
-
...(o.wallclockStartNs !== undefined ? { wallclockStartNs: String(o.wallclockStartNs) } : {}),
|
|
128
|
-
...(o.wallclockEndNs !== undefined ? { wallclockEndNs: String(o.wallclockEndNs) } : {}),
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
export async function writeRunSummary(args: SummarizeArgs): Promise<string> {
|
|
133
|
-
const outPath =
|
|
134
|
-
args.target === ''
|
|
135
|
-
? path.join(args.cacheDir, 'runs', `${args.runId}.json`)
|
|
136
|
-
: path.resolve(args.cwd, args.target)
|
|
137
|
-
await Bun.write(outPath, JSON.stringify(runSummaryJson(args), null, 2))
|
|
138
|
-
return outPath
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
/** The `--summarize` document, which `vx run --format json` prints. */
|
|
142
|
-
export function runSummaryJson(
|
|
143
|
-
args: Omit<SummarizeArgs, 'target' | 'cacheDir' | 'cwd'>,
|
|
144
|
-
): RunSummaryJson {
|
|
145
|
-
// `tasks` and `summary` must describe the same population, or one artifact
|
|
146
|
-
// contradicts itself (a group task listed in `tasks` but absent from
|
|
147
|
-
// `summary.total`). `tallyOutcomes` owns the rule; mirror its filter here.
|
|
148
|
-
const counted = args.outcomes.filter((o) => !isGroupTask(o.node) && o.status !== 'aborted')
|
|
149
|
-
// A task killed by a shutdown signal is in no bucket and no total, yet it
|
|
150
|
-
// makes the run red. Listing it separately keeps `tasks.length ===
|
|
151
|
-
// summary.total` while leaving the artifact able to explain a non-zero
|
|
152
|
-
// exit — the terminal has said so in its Aborted section all along, and a
|
|
153
|
-
// parser must not be told less than a human is.
|
|
154
|
-
const aborted = args.outcomes.filter((o) => !isGroupTask(o.node) && o.status === 'aborted')
|
|
155
|
-
const flakyById = new Map((args.flaky ?? []).map((f) => [f.taskId, f]))
|
|
156
|
-
const exitCode = args.exitCode ?? (args.ok ? 0 : 1)
|
|
157
|
-
const payload: RunSummaryJson = {
|
|
158
|
-
runId: args.runId,
|
|
159
|
-
// The run-level verdict, first: a consumer gating on this artifact must
|
|
160
|
-
// not have to re-derive it by re-implementing the bucket rules.
|
|
161
|
-
ok: exitCode === 0,
|
|
162
|
-
exitCode,
|
|
163
|
-
startedAt: new Date(args.startedAtMs).toISOString(),
|
|
164
|
-
endedAt: new Date(args.endedAtMs).toISOString(),
|
|
165
|
-
totalMs: args.totalMs,
|
|
166
|
-
// The summary row's `saved`, by the same rule: hits only, groups aside.
|
|
167
|
-
savedMs: counted.reduce(
|
|
168
|
-
(ms, o) => (isCacheHit(o.status) ? ms + (o.storedDurationMs ?? 0) : ms),
|
|
169
|
-
0,
|
|
170
|
-
),
|
|
171
|
-
tasks: counted.map((o) => taskEntry(o, flakyById.get(o.node.id))),
|
|
172
|
-
aborted: aborted.map((o) => taskEntry(o)),
|
|
173
|
-
// The FULL list: `tallyOutcomes` applies the same group/aborted
|
|
174
|
-
// exclusions internally, so every counted bucket is unchanged by passing
|
|
175
|
-
// it — but `summary.aborted` is only non-zero if it sees them.
|
|
176
|
-
summary: tallyOutcomes(args.outcomes),
|
|
177
|
-
}
|
|
178
|
-
return payload
|
|
179
|
-
}
|
|
7
|
+
import type { TaskOutcome } from '../graph/index.js'
|
|
180
8
|
|
|
181
9
|
export interface ProfileArgs {
|
|
182
10
|
target: string
|
|
@@ -15,6 +15,8 @@ import os from 'node:os'
|
|
|
15
15
|
import path from 'node:path'
|
|
16
16
|
import { executablePath, gitEnv, relPosix, ulid, xxh3hex } from '../util/index.js'
|
|
17
17
|
import { resolveSetting } from '../settings.js'
|
|
18
|
+
import { VERSION } from '../version.js'
|
|
19
|
+
import type { RunContextRecord } from './telemetry.js'
|
|
18
20
|
|
|
19
21
|
export interface GitContext {
|
|
20
22
|
/** `git rev-parse HEAD`, or null outside a repo / on failure. */
|
|
@@ -522,3 +524,56 @@ export function currentBranch(
|
|
|
522
524
|
const set = resolveSetting('run.branch', workspace, flag, env).value as string | undefined
|
|
523
525
|
return set ?? captureGitContext(workspaceRoot, null, env).branch
|
|
524
526
|
}
|
|
527
|
+
|
|
528
|
+
export interface TelemetryRunContextArgs {
|
|
529
|
+
runId: string
|
|
530
|
+
/** The invocation, already masked. */
|
|
531
|
+
command: string
|
|
532
|
+
requestedTasks: readonly string[]
|
|
533
|
+
cachePolicy: string
|
|
534
|
+
concurrency: number
|
|
535
|
+
flow: RunContextRecord['flow']
|
|
536
|
+
git: GitContext
|
|
537
|
+
ci: CiContext
|
|
538
|
+
host: HostContext
|
|
539
|
+
workspaceRoot: string
|
|
540
|
+
/** `remote.origin.url` as the run's enumeration read it; see `captureWorkspaceIdentity`. */
|
|
541
|
+
originUrl: string | null | undefined
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/**
|
|
545
|
+
* The run-context record telemetry sinks receive: the invocation header's
|
|
546
|
+
* git/CI/host data shaped as the export contract. Paid only when a sink can
|
|
547
|
+
* exist: the workspace identity may write `.vx/cache/workspace-id`.
|
|
548
|
+
*/
|
|
549
|
+
export function telemetryRunContext(a: TelemetryRunContextArgs): RunContextRecord {
|
|
550
|
+
const ws = captureWorkspaceIdentity(a.workspaceRoot, a.originUrl)
|
|
551
|
+
const { ci } = a
|
|
552
|
+
return {
|
|
553
|
+
runId: a.runId,
|
|
554
|
+
vxVersion: VERSION,
|
|
555
|
+
command: a.command,
|
|
556
|
+
requestedTasks: [...a.requestedTasks],
|
|
557
|
+
cachePolicy: a.cachePolicy,
|
|
558
|
+
concurrency: a.concurrency,
|
|
559
|
+
flow: a.flow,
|
|
560
|
+
commitSha: a.git.commitSha,
|
|
561
|
+
branch: a.git.branch,
|
|
562
|
+
defaultBranch: captureDefaultBranch(process.env, a.workspaceRoot),
|
|
563
|
+
dirty: a.git.dirty,
|
|
564
|
+
ci: ci.ci,
|
|
565
|
+
ciProvider: ci.provider,
|
|
566
|
+
...(ci.runUrl !== undefined ? { ciRunUrl: ci.runUrl } : {}),
|
|
567
|
+
...(ci.change !== undefined ? { ciChange: ci.change } : {}),
|
|
568
|
+
...(ci.pipeline !== undefined ? { ciPipeline: ci.pipeline } : {}),
|
|
569
|
+
...(ci.job !== undefined ? { ciJob: ci.job } : {}),
|
|
570
|
+
...(ci.attempt !== undefined ? { ciAttempt: ci.attempt } : {}),
|
|
571
|
+
host: a.host.host,
|
|
572
|
+
os: a.host.os,
|
|
573
|
+
arch: a.host.arch,
|
|
574
|
+
workspaceId: ws.id,
|
|
575
|
+
workspaceName: ws.name,
|
|
576
|
+
...(ws.repository !== undefined ? { repository: ws.repository } : {}),
|
|
577
|
+
...(ws.path !== undefined ? { workspacePath: ws.path } : {}),
|
|
578
|
+
}
|
|
579
|
+
}
|
|
@@ -8,9 +8,9 @@ import { mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync
|
|
|
8
8
|
import path from 'node:path'
|
|
9
9
|
import { isGroupTask, type TaskOutcome } from '../graph/index.js'
|
|
10
10
|
import { decodeOutputLog } from './output-log.js'
|
|
11
|
-
import { PASS_STATUSES } from '
|
|
12
|
-
import { splitTaskId } from '../util/index.js'
|
|
13
|
-
import { fileLocations, fileMemo, type OutputLocation } from '
|
|
11
|
+
import { PASS_STATUSES } from '../history/index.js'
|
|
12
|
+
import { isFsRefusal, splitTaskId } from '../util/index.js'
|
|
13
|
+
import { fileLocations, fileMemo, type OutputLocation } from '../report/index.js'
|
|
14
14
|
|
|
15
15
|
/** Failed runs whose output is kept; run ids are UUIDv7, so names sort by age. */
|
|
16
16
|
const KEPT = 50
|
|
@@ -78,14 +78,26 @@ function failuresOf(outcomes: readonly TaskOutcome[]): StoredFailure[] {
|
|
|
78
78
|
/**
|
|
79
79
|
* Write the run's failures, when any outcome carries output, and drop all
|
|
80
80
|
* but the newest `KEPT` files. Atomic: a reader sees the whole file or none.
|
|
81
|
+
* A refused write (a full disk, a read-only cache dir) costs only this
|
|
82
|
+
* output: `warn` says so and the run's verdict stands.
|
|
81
83
|
*/
|
|
82
84
|
export function writeRunFailures(
|
|
83
85
|
cacheDir: string,
|
|
84
86
|
runId: string,
|
|
85
87
|
outcomes: readonly TaskOutcome[],
|
|
88
|
+
warn: (message: string) => void,
|
|
86
89
|
): void {
|
|
87
90
|
const tasks = failuresOf(outcomes)
|
|
88
91
|
if (tasks.length === 0) return
|
|
92
|
+
try {
|
|
93
|
+
keep(cacheDir, runId, tasks)
|
|
94
|
+
} catch (err) {
|
|
95
|
+
if (!isFsRefusal(err)) throw err
|
|
96
|
+
warn(`[vx] failed tasks' output not kept: ${err.message} — the verdict above stands`)
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function keep(cacheDir: string, runId: string, tasks: TaskFailure[]): void {
|
|
89
101
|
const dir = dirOf(cacheDir)
|
|
90
102
|
mkdirSync(dir, { recursive: true })
|
|
91
103
|
const file = path.join(dir, `${runId}.json`)
|