@vzn/vx 0.0.642 → 0.0.644
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/schemas/info.json +0 -11
- package/skills/vx/SKILL.md +1 -1
- package/src/cache/archive.ts +153 -2
- package/src/cache/cache.ts +417 -183
- package/src/cache/file-hashes.ts +2 -14
- package/src/cache/fold-memo.ts +7 -3
- package/src/cache/index.ts +10 -2
- package/src/cache/key-fold.ts +4 -1
- package/src/cache/layer.ts +11 -4
- package/src/cache/layered-cache.ts +82 -8
- package/src/cache/plugin-cache.ts +51 -43
- package/src/cache/remote-lane.ts +18 -13
- package/src/cache/run-store.ts +4 -1
- package/src/cache/scoped-cache.ts +74 -239
- package/src/cache/tar-stream.ts +12 -7
- package/src/cli/format.ts +1 -2
- package/src/cli/help.ts +8 -4
- package/src/cli/info.ts +2 -19
- package/src/cli/plugin-commands.ts +11 -6
- package/src/cli/run.ts +7 -12
- package/src/cli/select.ts +7 -9
- package/src/cli/upgrade.ts +38 -8
- package/src/cli/workspace-config.ts +8 -2
- package/src/config.ts +22 -26
- package/src/exec/sandbox-paths.ts +2 -2
- package/src/graph/scheduler.ts +3 -1
- package/src/graph/task-graph.ts +15 -11
- package/src/index.ts +5 -1
- package/src/orchestrator/affected-tasks.ts +23 -2
- package/src/orchestrator/doctor.ts +0 -17
- package/src/orchestrator/execute-task.ts +1 -1
- package/src/orchestrator/index.ts +4 -0
- package/src/orchestrator/options.ts +15 -1
- package/src/orchestrator/plugin-host.ts +11 -4
- package/src/orchestrator/plugin.ts +23 -2
- package/src/orchestrator/prepare.ts +81 -18
- package/src/orchestrator/projects.ts +26 -5
- package/src/orchestrator/remote-prefetch.ts +11 -4
- package/src/orchestrator/run-context.ts +85 -0
- package/src/orchestrator/run-settings.ts +3 -0
- package/src/orchestrator/run.ts +9 -26
- package/src/orchestrator/task-hash.ts +6 -0
- package/src/report/logger.ts +0 -4
- package/src/report/summary.ts +3 -11
- package/src/settings.ts +54 -25
- package/src/util/index.ts +1 -0
- package/src/util/paths.ts +42 -33
- package/src/util/size.ts +4 -5
- package/src/workspace/affected.ts +120 -63
- package/src/workspace/config-schema.ts +44 -52
- package/src/workspace/index.ts +3 -0
|
@@ -26,20 +26,23 @@ import {
|
|
|
26
26
|
writesRunStore,
|
|
27
27
|
type PluginCache,
|
|
28
28
|
type ResolvedRemote,
|
|
29
|
-
|
|
29
|
+
BranchScope,
|
|
30
30
|
applyGitEnumeration,
|
|
31
31
|
gitPathspecs,
|
|
32
32
|
MAX_SCOPED_PATHSPECS,
|
|
33
33
|
startGitEnumeration,
|
|
34
34
|
lazyGitEnumeration,
|
|
35
35
|
measureFsClockLag,
|
|
36
|
+
DEFAULT_STORE,
|
|
37
|
+
type StoreOptions,
|
|
36
38
|
} from '../cache/index.js'
|
|
37
39
|
import {
|
|
38
40
|
buildPackageGraph,
|
|
39
41
|
computeNestedProjectDirs,
|
|
40
42
|
findWorkspaceRoot,
|
|
41
|
-
|
|
42
|
-
|
|
43
|
+
type AffectedChanges,
|
|
44
|
+
branchChanges,
|
|
45
|
+
configuredBaseBranch,
|
|
43
46
|
type LoadReads,
|
|
44
47
|
PROJECT_CONFIG_FILENAMES,
|
|
45
48
|
readClaimedFiles,
|
|
@@ -78,17 +81,17 @@ import {
|
|
|
78
81
|
lockReader,
|
|
79
82
|
type LoadedProjects,
|
|
80
83
|
} from './projects.js'
|
|
81
|
-
import { keptByAffected } from './affected-tasks.js'
|
|
84
|
+
import { keptByAffected, keyedByChange } from './affected-tasks.js'
|
|
82
85
|
import { applyGraphStage } from './graph-stage.js'
|
|
83
86
|
import { keyExcludedDependencies } from './excluded-keys.js'
|
|
84
87
|
import { FingerprintWatch } from './fingerprint-watch.js'
|
|
85
88
|
import type { VxPlugin } from './plugin.js'
|
|
86
89
|
import { askIgnoredOutputs, gatedOutputs } from './stable-keys.js'
|
|
87
|
-
import { currentBranch } from './run-context.js'
|
|
90
|
+
import { currentBranch, resolveCacheScope } from './run-context.js'
|
|
88
91
|
import { createHashCache, type HashCache } from './task-hash.js'
|
|
89
92
|
import { type Logger, warnOf } from '../report/index.js'
|
|
90
93
|
import type { RunOptions } from './options.js'
|
|
91
|
-
import { runRules } from '../settings.js'
|
|
94
|
+
import { resolveSetting, runRules } from '../settings.js'
|
|
92
95
|
|
|
93
96
|
export interface PreparedRun {
|
|
94
97
|
workspaceRoot: string
|
|
@@ -113,6 +116,8 @@ export interface PreparedRun {
|
|
|
113
116
|
hasRemoteLayer: boolean
|
|
114
117
|
/** `--cache-policy` with the workspace's `cacheScope` applied: what the layers were built under. */
|
|
115
118
|
cachePolicy: CachePolicy
|
|
119
|
+
/** The scope this run's cache reads and writes, when one is set; named on the summary's `info` row. */
|
|
120
|
+
cacheScope: string | undefined
|
|
116
121
|
/** The run's remotes in declared order, as the policy gates each; `openRemotes` turns one off. */
|
|
117
122
|
remotes: ResolvedRemote[]
|
|
118
123
|
/**
|
|
@@ -249,8 +254,21 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
249
254
|
(reused !== undefined ? gitOfDiscovery(reused.projects) : undefined) ??
|
|
250
255
|
lazyGitEnumeration(workspaceRoot)
|
|
251
256
|
if (wholeTree) void git.start()
|
|
252
|
-
const {
|
|
257
|
+
const {
|
|
258
|
+
workspaceConfig,
|
|
259
|
+
plugins,
|
|
260
|
+
branch: branchFacts,
|
|
261
|
+
} = await loadWorkspacePlugins(workspaceRoot, warnOf(log))
|
|
253
262
|
mark('workspace config')
|
|
263
|
+
const baseBranch = configuredBaseBranch(workspaceConfig)
|
|
264
|
+
const cacheScope =
|
|
265
|
+
options.cacheScope ??
|
|
266
|
+
resolveCacheScope(
|
|
267
|
+
workspaceRoot,
|
|
268
|
+
branchFacts.trusted === true ? null : branchFacts.scope,
|
|
269
|
+
plugins.some((p) => p.cache !== undefined),
|
|
270
|
+
baseBranch,
|
|
271
|
+
)
|
|
254
272
|
let learnedBranch: string | null | undefined
|
|
255
273
|
const branch = (): string | null =>
|
|
256
274
|
learnedBranch === undefined
|
|
@@ -269,10 +287,38 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
269
287
|
// Asked now, read when the cache opens: a repository with no remote is
|
|
270
288
|
// named by its root commit, a git spawn the discovery below overlaps.
|
|
271
289
|
const storeRoot = options.cacheDir ? null : resolveStoreRoot(workspaceRoot, workspaceConfig)
|
|
272
|
-
const
|
|
290
|
+
const discovering =
|
|
273
291
|
reused !== undefined
|
|
274
|
-
? reused.projects
|
|
275
|
-
:
|
|
292
|
+
? Promise.resolve(reused.projects)
|
|
293
|
+
: discoverProjects(workspaceRoot, readLock, plugins, cacheDir, warnOf(log), git)
|
|
294
|
+
// What the branch changed decides which tasks read the scope (owner,
|
|
295
|
+
// 2026-10-17). Only a scoped run's first remote call asks git: a run the
|
|
296
|
+
// local cache answers makes no remote call, and diffing beside discovery
|
|
297
|
+
// cost it 20-60 ms at 1k-5k projects.
|
|
298
|
+
const scopeChanges: (() => Promise<AffectedChanges>) | undefined =
|
|
299
|
+
cacheScope === undefined
|
|
300
|
+
? undefined
|
|
301
|
+
: options.scopeChanges !== undefined
|
|
302
|
+
? async () => options.scopeChanges!
|
|
303
|
+
: () =>
|
|
304
|
+
branchChanges(
|
|
305
|
+
workspaceRoot,
|
|
306
|
+
options.scopeBase ?? branchFacts.base,
|
|
307
|
+
baseBranch,
|
|
308
|
+
discovering,
|
|
309
|
+
{
|
|
310
|
+
untracked: async () => (await git.start()).untracked,
|
|
311
|
+
// A scoped enumeration lists only its projects: null asks git.
|
|
312
|
+
links: async () => {
|
|
313
|
+
const g = await git.start()
|
|
314
|
+
if (!g.scopes.includes('.')) return null
|
|
315
|
+
const out: string[] = []
|
|
316
|
+
for (const [rel, oid] of g.trusted) if (oid.startsWith('120000:')) out.push(rel)
|
|
317
|
+
return out
|
|
318
|
+
},
|
|
319
|
+
},
|
|
320
|
+
)
|
|
321
|
+
const projectMetas = await discovering
|
|
276
322
|
mark('discover projects')
|
|
277
323
|
|
|
278
324
|
// SCOPED config loading: configs are programs, and evaluating 1090
|
|
@@ -314,12 +360,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
314
360
|
// so a reader profiling that stage was sent to the wrong code.
|
|
315
361
|
mark('package graph')
|
|
316
362
|
|
|
317
|
-
// `--cache-scope`, then `VX_CACHE_SCOPE`, win over the workspace (and `github()`'s guess):
|
|
318
|
-
// a CI job that knows its trust level says so without editing config.
|
|
319
|
-
const envScope = process.env['VX_CACHE_SCOPE'] || undefined
|
|
320
|
-
if (envScope !== undefined && !isCacheScope(envScope)) throw cacheScopeEnvError()
|
|
321
363
|
const workspacePolicy = workspaceCachePolicy(workspaceConfig)
|
|
322
|
-
const cacheScope = options.cacheScope ?? envScope ?? workspaceConfig?.cache?.scope
|
|
323
364
|
const asked =
|
|
324
365
|
options.cache ?? applyCacheFlags(workspacePolicy ?? FULL_CACHE_POLICY, options.cacheFlags)
|
|
325
366
|
// A laptop reads the shared cache and does not write it unless a policy
|
|
@@ -337,6 +378,14 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
337
378
|
options.artifactCeiling,
|
|
338
379
|
'open',
|
|
339
380
|
await storeRoot,
|
|
381
|
+
{
|
|
382
|
+
embeddings: (resolveSetting('cache.store.embeddings', workspaceConfig).value ??
|
|
383
|
+
DEFAULT_STORE.embeddings) as number,
|
|
384
|
+
decompress: (resolveSetting('cache.store.decompress', workspaceConfig).value ??
|
|
385
|
+
DEFAULT_STORE.decompress) as boolean,
|
|
386
|
+
verify: (resolveSetting('cache.store.verify', workspaceConfig).value ??
|
|
387
|
+
DEFAULT_STORE.verify) as StoreOptions['verify'],
|
|
388
|
+
},
|
|
340
389
|
)
|
|
341
390
|
const unwritable = localCache.readOnlyNotice()
|
|
342
391
|
if (unwritable !== null) warnOf(log)(`[vx] ${unwritable}`)
|
|
@@ -470,6 +519,9 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
470
519
|
// cache plugin.
|
|
471
520
|
let cache: CacheLayer
|
|
472
521
|
const remotes: ResolvedRemote[] = []
|
|
522
|
+
// A scope only picks remote keys (owner, 2026-10-17): the local store
|
|
523
|
+
// keeps every branch's entries under the task keys.
|
|
524
|
+
const branchScope = cacheScope === undefined ? undefined : new BranchScope(cacheScope)
|
|
473
525
|
if (options.remoteCache) {
|
|
474
526
|
remotes.push({
|
|
475
527
|
remote: options.remoteCache,
|
|
@@ -480,7 +532,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
480
532
|
}
|
|
481
533
|
try {
|
|
482
534
|
cache = options.remoteCache
|
|
483
|
-
? new LayeredCache(localCache, remotes)
|
|
535
|
+
? new LayeredCache(localCache, remotes, branchScope, options.uploadConcurrency)
|
|
484
536
|
: await resolveCache(plugins, {
|
|
485
537
|
workspaceRoot,
|
|
486
538
|
cacheDir,
|
|
@@ -489,6 +541,10 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
489
541
|
localCache,
|
|
490
542
|
policy,
|
|
491
543
|
remotes,
|
|
544
|
+
...(branchScope !== undefined ? { branch: branchScope } : {}),
|
|
545
|
+
...(options.uploadConcurrency !== undefined
|
|
546
|
+
? { uploadConcurrency: options.uploadConcurrency }
|
|
547
|
+
: {}),
|
|
492
548
|
})
|
|
493
549
|
} catch (err) {
|
|
494
550
|
// A cache plugin that throws or returns something off-contract is
|
|
@@ -499,8 +555,6 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
499
555
|
localCache.close()
|
|
500
556
|
throw err
|
|
501
557
|
}
|
|
502
|
-
// One scoping rule above every layer, local and remote alike.
|
|
503
|
-
if (cacheScope !== undefined) cache = new ScopedCache(cache, cacheScope)
|
|
504
558
|
// Ask the LAYER, don't infer. Identity against `localCache` answers a
|
|
505
559
|
// DIFFERENT question — "did the plugin hand back something other than the
|
|
506
560
|
// handle I passed in?" — which an ordinary pass-through decorator (a
|
|
@@ -534,6 +588,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
534
588
|
localCache,
|
|
535
589
|
hasRemoteLayer,
|
|
536
590
|
cachePolicy: policy,
|
|
591
|
+
cacheScope,
|
|
537
592
|
remotes,
|
|
538
593
|
keepRunEntry: undefined,
|
|
539
594
|
priorities: () => Promise.resolve(new Map()),
|
|
@@ -721,6 +776,13 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
721
776
|
if (hasHook(plugins, 'key')) {
|
|
722
777
|
await applyKeyHooks(plugins, nodes, { workspaceRoot, cacheDir, warn: warnOf(log) })
|
|
723
778
|
}
|
|
779
|
+
// The final inputs decide which tasks the branch changed: they read
|
|
780
|
+
// the scope, every other task the default key.
|
|
781
|
+
if (branchScope !== undefined && scopeChanges !== undefined) {
|
|
782
|
+
branchScope.routeTasks(async () =>
|
|
783
|
+
keyedByChange(nodes, await scopeChanges(), projects, packageGraph),
|
|
784
|
+
)
|
|
785
|
+
}
|
|
724
786
|
refuseFiltersThroughGroups(nodes)
|
|
725
787
|
const exclude = options.excludeDependencies
|
|
726
788
|
let keyOnly: ReadonlyMap<string, TaskNode> = new Map()
|
|
@@ -783,6 +845,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
783
845
|
localCache,
|
|
784
846
|
hasRemoteLayer,
|
|
785
847
|
cachePolicy: policy,
|
|
848
|
+
cacheScope,
|
|
786
849
|
remotes,
|
|
787
850
|
priorities,
|
|
788
851
|
branch,
|
|
@@ -25,6 +25,8 @@ import type { WorkspaceConfig } from '../config.js'
|
|
|
25
25
|
import { lazyGitEnumeration, type LazyGitEnumeration } from '../cache/index.js'
|
|
26
26
|
import { relPosix, UserError } from '../util/index.js'
|
|
27
27
|
import {
|
|
28
|
+
cacheScopeError,
|
|
29
|
+
isCacheScope,
|
|
28
30
|
loadWorkspaceConfig,
|
|
29
31
|
discoveredProject,
|
|
30
32
|
linkDiscoveredEdges,
|
|
@@ -33,7 +35,7 @@ import {
|
|
|
33
35
|
} from '../workspace/index.js'
|
|
34
36
|
import { parseDependencySpec } from '../graph/index.js'
|
|
35
37
|
import { applyConfigHooks, applyDiscoverHooks, applyProjectHooks, hasHook } from './plugin-host.js'
|
|
36
|
-
import type { VxPlugin } from './plugin.js'
|
|
38
|
+
import type { BranchFacts, VxPlugin } from './plugin.js'
|
|
37
39
|
|
|
38
40
|
/**
|
|
39
41
|
* The workspace config with the plugin `config` stage applied — the first
|
|
@@ -43,10 +45,16 @@ import type { VxPlugin } from './plugin.js'
|
|
|
43
45
|
export async function loadWorkspacePlugins(
|
|
44
46
|
workspaceRoot: string,
|
|
45
47
|
warn: (message: string) => void,
|
|
46
|
-
): Promise<{
|
|
48
|
+
): Promise<{
|
|
49
|
+
workspaceConfig: WorkspaceConfig | null
|
|
50
|
+
plugins: readonly VxPlugin[]
|
|
51
|
+
branch: BranchFacts
|
|
52
|
+
}> {
|
|
47
53
|
const loaded = await loadWorkspaceConfig(workspaceRoot)
|
|
48
54
|
const plugins = (loaded?.plugins ?? []) as readonly VxPlugin[]
|
|
49
|
-
|
|
55
|
+
const branch: BranchFacts = {}
|
|
56
|
+
if (loaded === null || !hasHook(plugins, 'config'))
|
|
57
|
+
return { workspaceConfig: loaded, plugins, branch }
|
|
50
58
|
// Bun keeps one module per specifier, so the file's export is one object
|
|
51
59
|
// per process: the CLI's selection pass, the run and each `vx watch`
|
|
52
60
|
// cycle handed the hooks what the last load's hooks had edited
|
|
@@ -60,10 +68,23 @@ export async function loadWorkspacePlugins(
|
|
|
60
68
|
// Re-validated after each plugin, as the `project` stage is: an edit
|
|
61
69
|
// the loader would refuse from a user (`concurrency: -3`) hung the run,
|
|
62
70
|
// and `cacheDir: 42` reached `path.resolve` as a TypeError.
|
|
63
|
-
|
|
71
|
+
const setBranch = (facts: BranchFacts): void => {
|
|
72
|
+
if (facts.scope !== undefined && !isCacheScope(facts.scope))
|
|
73
|
+
throw new UserError(`setBranch: ${cacheScopeError('scope')}`)
|
|
74
|
+
if (facts.trusted === true) {
|
|
75
|
+
delete branch.scope
|
|
76
|
+
branch.trusted = true
|
|
77
|
+
}
|
|
78
|
+
if (facts.scope !== undefined) {
|
|
79
|
+
branch.scope = facts.scope
|
|
80
|
+
delete branch.trusted
|
|
81
|
+
}
|
|
82
|
+
if (facts.base !== undefined) branch.base = facts.base
|
|
83
|
+
}
|
|
84
|
+
await applyConfigHooks(plugins, workspaceConfig, { workspaceRoot, warn, setBranch }, (plugin) =>
|
|
64
85
|
validateWorkspace(workspaceConfig, `.vx/workspace (after plugin '${plugin.name}')`),
|
|
65
86
|
)
|
|
66
|
-
return { workspaceConfig, plugins }
|
|
87
|
+
return { workspaceConfig, plugins, branch }
|
|
67
88
|
}
|
|
68
89
|
|
|
69
90
|
/**
|
|
@@ -37,7 +37,7 @@ export interface PrefetchArgs {
|
|
|
37
37
|
nestedDirsByProject: Map<string, string[]>
|
|
38
38
|
gitFilesCache: GitFilesCache
|
|
39
39
|
hashCache: HashCache
|
|
40
|
-
/** Cap on concurrent in-flight prefetches
|
|
40
|
+
/** Cap on concurrent in-flight prefetches (`cache.concurrency.download`). */
|
|
41
41
|
concurrency: number
|
|
42
42
|
/** When false (remote reads disabled), no prefetch fires. */
|
|
43
43
|
remoteRead: boolean
|
|
@@ -75,10 +75,14 @@ export function startRemotePrefetch(args: PrefetchArgs): Promise<void> {
|
|
|
75
75
|
// a still-in-flight prefetch can never ingest into a closed DB. Errors
|
|
76
76
|
// degrade inside LayeredCache.prefetch; the catch keeps the handle
|
|
77
77
|
// from ever rejecting.
|
|
78
|
-
|
|
78
|
+
const started = Promise.withResolvers<void>()
|
|
79
|
+
args.cache.probeFirst?.(started.promise)
|
|
80
|
+
return runPrefetch(args, started.resolve)
|
|
81
|
+
.catch(() => {})
|
|
82
|
+
.finally(started.resolve)
|
|
79
83
|
}
|
|
80
84
|
|
|
81
|
-
async function runPrefetch(args: PrefetchArgs): Promise<void> {
|
|
85
|
+
async function runPrefetch(args: PrefetchArgs, probing: () => void): Promise<void> {
|
|
82
86
|
const stableKeys = await deriveStableKeys(args)
|
|
83
87
|
if (stableKeys.length === 0) return
|
|
84
88
|
|
|
@@ -94,7 +98,10 @@ async function runPrefetch(args: PrefetchArgs): Promise<void> {
|
|
|
94
98
|
let toPrefetch = held === null ? stableKeys : stableKeys.filter((k) => !held.has(k.hash))
|
|
95
99
|
if (toPrefetch.length === 0) return
|
|
96
100
|
const uniqueHashes = [...new Set(toPrefetch.map((k) => k.hash))]
|
|
97
|
-
const
|
|
101
|
+
const asked = args.cache.remoteHasMany?.(uniqueHashes)
|
|
102
|
+
// Every lookup that waited on the probe now finds its hash in it.
|
|
103
|
+
probing()
|
|
104
|
+
const present = (await asked) ?? null
|
|
98
105
|
if (present !== null) {
|
|
99
106
|
args.cache.markRemoteAbsent?.(uniqueHashes.filter((h) => !present.has(h)))
|
|
100
107
|
// The pre-mark above already answers a pull for an absent hash from
|
|
@@ -15,6 +15,7 @@ import os from 'node:os'
|
|
|
15
15
|
import path from 'node:path'
|
|
16
16
|
import { executablePath, gitEnv, relPosix, ulid, xxh3hex } from '../util/index.js'
|
|
17
17
|
import { resolveSetting } from '../settings.js'
|
|
18
|
+
import { branchCacheScope, cacheScopeEnvError, isCacheScope } from '../workspace/index.js'
|
|
18
19
|
import { VERSION } from '../version.js'
|
|
19
20
|
import type { RunContextRecord } from './telemetry.js'
|
|
20
21
|
|
|
@@ -525,6 +526,90 @@ export function currentBranch(
|
|
|
525
526
|
return set ?? captureGitContext(workspaceRoot, null, env).branch
|
|
526
527
|
}
|
|
527
528
|
|
|
529
|
+
/**
|
|
530
|
+
* The cache scope a checkout on a branch reads, from the `.git` files
|
|
531
|
+
* alone (owner, 2026-10-17): `<owner>/<repo>:<branch>`, the branch it
|
|
532
|
+
* tracks (`branch.<name>.merge` on `branch.<name>.remote`), else its own
|
|
533
|
+
* name on `origin`, the repository named by that remote's URL. None on
|
|
534
|
+
* the default branch, on a detached HEAD, or with no remote URL. CI writes
|
|
535
|
+
* that scope for the same branch (`github()`), so a laptop reads what its
|
|
536
|
+
* branch's CI saved.
|
|
537
|
+
*/
|
|
538
|
+
export function localBranchScope(
|
|
539
|
+
workspaceRoot: string,
|
|
540
|
+
baseBranch: string | undefined,
|
|
541
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
542
|
+
): string | undefined {
|
|
543
|
+
const head = readHeadDirect(workspaceRoot)
|
|
544
|
+
if (head === null || head.branch === null) return undefined
|
|
545
|
+
const config = gitConfigSections(workspaceRoot)
|
|
546
|
+
const own = config.get(`branch "${head.branch}"`)
|
|
547
|
+
const name = own?.get('merge')?.replace(/^refs\/heads\//, '') ?? head.branch
|
|
548
|
+
const url = config.get(`remote "${own?.get('remote') ?? 'origin'}"`)?.get('url')
|
|
549
|
+
const repo = url === undefined ? undefined : repoOfUrl(url)
|
|
550
|
+
if (repo === undefined) return undefined
|
|
551
|
+
const trunk = baseBranch ?? captureDefaultBranch(env, workspaceRoot)
|
|
552
|
+
if (trunk === null ? name === 'main' || name === 'master' : name === trunk) return undefined
|
|
553
|
+
return branchCacheScope(repo, name)
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
/**
|
|
557
|
+
* The repository's path on its host (`owner/repo`; a GitLab subgroup's
|
|
558
|
+
* `group/sub/repo` whole, as `gitlab()` names it); undefined for a local path.
|
|
559
|
+
*/
|
|
560
|
+
function repoOfUrl(url: string): string | undefined {
|
|
561
|
+
const parts = normalizeRemoteUrl(url).split('/')
|
|
562
|
+
return parts.length < 3 ? undefined : parts.slice(1).join('/')
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/** `.git/config`'s sections, each a key → value map (last wins); empty when it cannot be read. */
|
|
566
|
+
function gitConfigSections(workspaceRoot: string): Map<string, Map<string, string>> {
|
|
567
|
+
const sections = new Map<string, Map<string, string>>()
|
|
568
|
+
try {
|
|
569
|
+
const dirs = refDirsOf(workspaceRoot)
|
|
570
|
+
if (dirs === null) return sections
|
|
571
|
+
let at: Map<string, string> | undefined
|
|
572
|
+
for (const raw of fs.readFileSync(path.join(dirs.commonDir, 'config'), 'utf8').split('\n')) {
|
|
573
|
+
const line = raw.trim()
|
|
574
|
+
const section = /^\[\s*(\w+)(?:\s+"(.*)")?\s*\]$/.exec(line)
|
|
575
|
+
if (section !== null) {
|
|
576
|
+
const key = section[2] === undefined ? section[1]! : `${section[1]} "${section[2]}"`
|
|
577
|
+
at = sections.get(key) ?? new Map()
|
|
578
|
+
sections.set(key, at)
|
|
579
|
+
continue
|
|
580
|
+
}
|
|
581
|
+
const kv = /^(\w+)\s*=\s*(.*)$/.exec(line)
|
|
582
|
+
if (kv !== null) at?.set(kv[1]!.toLowerCase(), kv[2]!.trim())
|
|
583
|
+
}
|
|
584
|
+
} catch {
|
|
585
|
+
// no config to read: no remote, so no scope.
|
|
586
|
+
}
|
|
587
|
+
return sections
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* The run's cache scope (owner, 2026-10-17): `VX_CACHE_SCOPE` (empty: none),
|
|
592
|
+
* else a plugin's (`setBranch`, `github()` on Actions; null: the plugin
|
|
593
|
+
* named the run trusted), else, with a cache plugin to share it, the
|
|
594
|
+
* branch's from git. Undefined: the default keys.
|
|
595
|
+
*/
|
|
596
|
+
export function resolveCacheScope(
|
|
597
|
+
workspaceRoot: string,
|
|
598
|
+
pluginScope: string | null | undefined,
|
|
599
|
+
hasRemote: boolean,
|
|
600
|
+
baseBranch: string | undefined,
|
|
601
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
602
|
+
): string | undefined {
|
|
603
|
+
const named = env['VX_CACHE_SCOPE']
|
|
604
|
+
if (named !== undefined) {
|
|
605
|
+
if (named === '') return undefined
|
|
606
|
+
if (!isCacheScope(named)) throw cacheScopeEnvError()
|
|
607
|
+
return named
|
|
608
|
+
}
|
|
609
|
+
if (pluginScope === null) return undefined
|
|
610
|
+
return pluginScope ?? (hasRemote ? localBranchScope(workspaceRoot, baseBranch, env) : undefined)
|
|
611
|
+
}
|
|
612
|
+
|
|
528
613
|
export interface TelemetryRunContextArgs {
|
|
529
614
|
runId: string
|
|
530
615
|
/** The invocation, already masked. */
|
|
@@ -14,6 +14,9 @@ const FROM_OPTIONS = {
|
|
|
14
14
|
retries: 'run.retries',
|
|
15
15
|
continueMode: 'run.continue',
|
|
16
16
|
download: 'cache.download',
|
|
17
|
+
downloadConcurrency: 'cache.concurrency.download',
|
|
18
|
+
uploadConcurrency: 'cache.concurrency.upload',
|
|
19
|
+
restoreConcurrency: 'cache.concurrency.restore',
|
|
17
20
|
} as const satisfies Partial<Record<keyof RunOptions, string>>
|
|
18
21
|
|
|
19
22
|
/**
|
package/src/orchestrator/run.ts
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
type CachePolicy,
|
|
12
12
|
cachesNothing,
|
|
13
13
|
formatCachePolicy,
|
|
14
|
+
networkConcurrency,
|
|
14
15
|
stopRuntimeProbes,
|
|
15
16
|
sweepStaging,
|
|
16
17
|
} from '../cache/index.js'
|
|
@@ -259,6 +260,10 @@ function refuseRunOptions(options: RunOptions): void {
|
|
|
259
260
|
const { concurrency, retries, timeout } = options
|
|
260
261
|
if (concurrency !== undefined && !(Number.isInteger(concurrency) && concurrency > 0))
|
|
261
262
|
refuse('concurrency', concurrency, 'a positive integer')
|
|
263
|
+
for (const field of ['downloadConcurrency', 'uploadConcurrency', 'restoreConcurrency'] as const) {
|
|
264
|
+
const v = options[field]
|
|
265
|
+
if (v !== undefined && !(Number.isInteger(v) && v > 0)) refuse(field, v, 'a positive integer')
|
|
266
|
+
}
|
|
262
267
|
if (retries !== undefined && !(Number.isInteger(retries) && retries >= 0))
|
|
263
268
|
refuse('retries', retries, 'a non-negative integer')
|
|
264
269
|
if (
|
|
@@ -618,7 +623,7 @@ async function runOnBus(
|
|
|
618
623
|
// `vx cache clean` can evict entries the run just hit. Once-only
|
|
619
624
|
// because the normal path closes before the persistent-task wait —
|
|
620
625
|
// holding the handle open for a dev server's whole lifetime would be
|
|
621
|
-
// worse
|
|
626
|
+
// worse.
|
|
622
627
|
let cacheClosed = false
|
|
623
628
|
const closeCache = async (): Promise<void> => {
|
|
624
629
|
if (cacheClosed) return
|
|
@@ -799,6 +804,7 @@ async function runOnBus(
|
|
|
799
804
|
version: VERSION,
|
|
800
805
|
packageCount: packagesInScope.size,
|
|
801
806
|
remoteCacheEnabled,
|
|
807
|
+
cacheScope: prepared.cacheScope,
|
|
802
808
|
cache: {
|
|
803
809
|
local: policy.localRead || policy.localWrite,
|
|
804
810
|
localWrite: policy.localWrite,
|
|
@@ -870,7 +876,7 @@ async function runOnBus(
|
|
|
870
876
|
nestedDirsByProject,
|
|
871
877
|
gitFilesCache,
|
|
872
878
|
hashCache,
|
|
873
|
-
concurrency,
|
|
879
|
+
concurrency: options.downloadConcurrency ?? networkConcurrency(),
|
|
874
880
|
remoteRead: policy.remoteRead,
|
|
875
881
|
remoteWrite: policy.remoteWrite,
|
|
876
882
|
}),
|
|
@@ -1087,6 +1093,7 @@ async function runOnBus(
|
|
|
1087
1093
|
const outcomes = await runGraph({
|
|
1088
1094
|
nodes,
|
|
1089
1095
|
concurrency,
|
|
1096
|
+
restoreConcurrency: options.restoreConcurrency ?? 2 * concurrency,
|
|
1090
1097
|
settledOf: (o) => deferredSaves.get(o.node.id),
|
|
1091
1098
|
...(hasPooledExecutor(executors) ? { poolOf: poolOfPlacement(placements) } : {}),
|
|
1092
1099
|
...(admit !== undefined ? { admit } : {}),
|
|
@@ -1348,9 +1355,6 @@ async function runOnBus(
|
|
|
1348
1355
|
// The miss path's output-directory snapshots first (miss-save.ts).
|
|
1349
1356
|
await step('vx:snapshots', () => recordOutputDirSnapshots(cache, outputDirSnapshots))
|
|
1350
1357
|
mark('output dir snapshots')
|
|
1351
|
-
// Not on a stopped run: one stopped while it waited on another run's
|
|
1352
|
-
// lock never held it, and its prune evicted under that run (item 858).
|
|
1353
|
-
if (!stop.signal.aborted) await step('vx:retention', () => applyCacheRetention(prepared))
|
|
1354
1358
|
// A plugin hears the run until its teardown and nothing after: released
|
|
1355
1359
|
// only in the finally, its handlers heard a kept server through the
|
|
1356
1360
|
// whole keep-alive wait below (C-66). Idempotent; the finally's stay.
|
|
@@ -1487,27 +1491,6 @@ async function runOnBus(
|
|
|
1487
1491
|
}
|
|
1488
1492
|
}
|
|
1489
1493
|
|
|
1490
|
-
/**
|
|
1491
|
-
* `cache.retention`, after every save and upload of this run
|
|
1492
|
-
* has landed (so nothing this run wrote is mid-flight) and before the cache
|
|
1493
|
-
* closes. Housekeeping, not the run's work: silent, and a failure is never
|
|
1494
|
-
* a failed run. Declared nowhere → one property read.
|
|
1495
|
-
*/
|
|
1496
|
-
async function applyCacheRetention(prepared: PreparedRun): Promise<void> {
|
|
1497
|
-
const ws = prepared.workspaceConfig
|
|
1498
|
-
const maxAgeMs = resolveSetting('cache.retention.olderThan', ws).value as number | undefined
|
|
1499
|
-
const maxBytes = resolveSetting('cache.retention.maxSize', ws).value as number | undefined
|
|
1500
|
-
if (maxAgeMs === undefined && maxBytes === undefined) return
|
|
1501
|
-
try {
|
|
1502
|
-
await prepared.localCache.evictIfDue({
|
|
1503
|
-
...(maxAgeMs !== undefined ? { maxAgeMs } : {}),
|
|
1504
|
-
...(maxBytes !== undefined ? { maxBytes } : {}),
|
|
1505
|
-
})
|
|
1506
|
-
} catch {
|
|
1507
|
-
// Housekeeping says nothing (owner): the next due run tries again.
|
|
1508
|
-
}
|
|
1509
|
-
}
|
|
1510
|
-
|
|
1511
1494
|
/** Why an `--affected` run has nothing to run, said alike by the run and its `--dry` plan. */
|
|
1512
1495
|
function noneAffected(prepared: PreparedRun, options: RunOptions): string {
|
|
1513
1496
|
// A changed project declares the task, so `--affected`'s task walk is
|
|
@@ -97,6 +97,12 @@ export interface ComputeHashArgs {
|
|
|
97
97
|
*/
|
|
98
98
|
export async function computeTaskHash(args: ComputeHashArgs): Promise<string> {
|
|
99
99
|
if (isKeyless(args.node.config)) return keylessHash(args.node.id)
|
|
100
|
+
const key = await taskHash(args)
|
|
101
|
+
args.cache.noteKey?.(args.node.id, key)
|
|
102
|
+
return key
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async function taskHash(args: ComputeHashArgs): Promise<string> {
|
|
100
106
|
const end = span('task hash')
|
|
101
107
|
try {
|
|
102
108
|
const folds = args.captureInto === undefined ? args.hashCache?.folds : undefined
|
package/src/report/logger.ts
CHANGED
|
@@ -382,7 +382,6 @@ export function defaultLogger(
|
|
|
382
382
|
let saved = 0
|
|
383
383
|
let notSaved = 0
|
|
384
384
|
let skippedCount = 0
|
|
385
|
-
let savedMs = 0
|
|
386
385
|
// Each finished task's cache bucket by its key: its remote traffic moves it.
|
|
387
386
|
const buckets = new Map<string, CacheBucket>()
|
|
388
387
|
// The graph is done and vx's own closing steps run: the region stays,
|
|
@@ -530,7 +529,6 @@ export function defaultLogger(
|
|
|
530
529
|
notSaved,
|
|
531
530
|
miss: succeeded + failed - noCache,
|
|
532
531
|
noCache,
|
|
533
|
-
savedMs,
|
|
534
532
|
left: total - done - stopped - notRun,
|
|
535
533
|
...(etaMs !== undefined && !tail && !stopping ? { etaMs } : {}),
|
|
536
534
|
buckets,
|
|
@@ -1028,12 +1026,10 @@ export function defaultLogger(
|
|
|
1028
1026
|
case 'cache-hit':
|
|
1029
1027
|
if (outcome.restored === false) upToDate++
|
|
1030
1028
|
else restoredLocal++
|
|
1031
|
-
savedMs += outcome.storedDurationMs ?? 0
|
|
1032
1029
|
break
|
|
1033
1030
|
case 'cache-hit-remote':
|
|
1034
1031
|
if (outcome.restored === false) upToDate++
|
|
1035
1032
|
else restoredRemote++
|
|
1036
|
-
savedMs += outcome.storedDurationMs ?? 0
|
|
1037
1033
|
break
|
|
1038
1034
|
case 'skipped':
|
|
1039
1035
|
skippedCount++
|
package/src/report/summary.ts
CHANGED
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
|
|
6
6
|
import type { TaskOutcome } from '../graph/index.js'
|
|
7
7
|
import { tallyOutcomes } from './tally.js'
|
|
8
|
-
import { isCacheHit } from '../graph/index.js'
|
|
9
8
|
import { outcomeLabel, projectOutcome, ranNoCache } from './events.js'
|
|
10
9
|
import { isGroupTask } from '../graph/index.js'
|
|
11
10
|
import {
|
|
@@ -106,8 +105,6 @@ export interface SummaryStats {
|
|
|
106
105
|
held?: { count: number; sumMs: number }
|
|
107
106
|
/** Live section only: the forecast of what is left, from history. */
|
|
108
107
|
etaMs?: number
|
|
109
|
-
/** The hits' stored exec times summed: the work the cache skipped. */
|
|
110
|
-
savedMs?: number
|
|
111
108
|
/** A run of groups only: their names, so `0 tasks` says why. */
|
|
112
109
|
emptyGroups?: readonly string[]
|
|
113
110
|
/** Each finished task's cache bucket by its key and its id (its run bytes), so its remote traffic can move it. */
|
|
@@ -188,6 +185,8 @@ export interface RunContext {
|
|
|
188
185
|
/** Worker-pool size for this run; shown on the `info` row. */
|
|
189
186
|
concurrency?: number
|
|
190
187
|
remoteCacheEnabled: boolean
|
|
188
|
+
/** The run's cache scope, named after the cache mode: `scope vznjs/vx:feat-x`. */
|
|
189
|
+
cacheScope?: string | undefined
|
|
191
190
|
/**
|
|
192
191
|
* The cache layers the run used, so the legend reads `local 1= 2↓ 4↑ ·
|
|
193
192
|
* remote 3↓ 4↑`. Absent: the plain `N miss` legend.
|
|
@@ -390,10 +389,6 @@ export function formatSummarySection(
|
|
|
390
389
|
}
|
|
391
390
|
}
|
|
392
391
|
const timeParts = [total]
|
|
393
|
-
// What the cache paid back, beside what the run cost: the stored exec
|
|
394
|
-
// time of every hit, never the restore this run spent on it.
|
|
395
|
-
if ((stats.savedMs ?? 0) >= 1)
|
|
396
|
-
timeParts.push(paint(SUCCESS, `${formatElapsed(stats.savedMs!)} saved`, colors))
|
|
397
392
|
// The live run's forecast leads the spread: the one number on the row
|
|
398
393
|
// that is about the future.
|
|
399
394
|
if (stats.etaMs !== undefined)
|
|
@@ -407,6 +402,7 @@ export function formatSummarySection(
|
|
|
407
402
|
if (context.concurrency !== undefined)
|
|
408
403
|
info.push(`${context.concurrency} worker${context.concurrency === 1 ? '' : 's'}`)
|
|
409
404
|
info.push(context.remoteCacheEnabled ? 'local + remote cache' : 'local cache')
|
|
405
|
+
if (context.cacheScope !== undefined) info.push(`scope ${context.cacheScope}`)
|
|
410
406
|
// The plugin's hand on the run: a task a policy held with a worker
|
|
411
407
|
// free would otherwise have started; without the line nothing says
|
|
412
408
|
// the policy acted. The duration is the waits summed, task-seconds
|
|
@@ -489,9 +485,6 @@ export function formatRunSummary(
|
|
|
489
485
|
const durations = outcomes
|
|
490
486
|
.filter((o) => (o.status === 'success' || o.status === 'failed') && !isGroupTask(o.node))
|
|
491
487
|
.map((o) => o.durationMs)
|
|
492
|
-
let savedMs = 0
|
|
493
|
-
for (const o of outcomes)
|
|
494
|
-
if (isCacheHit(o.status) && !isGroupTask(o.node)) savedMs += o.storedDurationMs ?? 0
|
|
495
488
|
const heldOutcomes = outcomes.filter((o) => o.admissionHeldMs !== undefined)
|
|
496
489
|
const notRun = outcomes.filter((o) => neverStarted(o) && !isGroupTask(o.node)).length
|
|
497
490
|
const emptyGroups = outcomes.every((o) => isGroupTask(o.node))
|
|
@@ -512,7 +505,6 @@ export function formatRunSummary(
|
|
|
512
505
|
notSaved: outcomes.filter((o) => !isGroupTask(o.node) && ranUnsaved(o)).length,
|
|
513
506
|
miss: t.total - t.skipped - (t.cachedLocal + t.cachedRemote) - noCache,
|
|
514
507
|
noCache,
|
|
515
|
-
savedMs,
|
|
516
508
|
spread:
|
|
517
509
|
durations.length > 0
|
|
518
510
|
? {
|