@vzn/vx 0.0.643 → 0.0.645
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +15 -23
- package/package.json +5 -5
- package/schemas/info.json +0 -11
- package/schemas/init.json +5 -0
- package/skills/vx/SKILL.md +1 -1
- package/src/cache/archive.ts +89 -2
- package/src/cache/cache.ts +396 -176
- package/src/cache/file-hashes.ts +2 -14
- package/src/cache/fold-memo.ts +7 -3
- package/src/cache/index.ts +4 -2
- package/src/cache/key-fold.ts +4 -1
- package/src/cache/layer.ts +14 -10
- package/src/cache/layered-cache.ts +157 -41
- package/src/cache/plugin-cache.ts +40 -44
- package/src/cache/remote-lane.ts +59 -25
- package/src/cache/scoped-cache.ts +3 -80
- package/src/cache/tar-stream.ts +12 -7
- package/src/cli/format.ts +1 -2
- package/src/cli/help.ts +10 -4
- package/src/cli/info.ts +2 -19
- package/src/cli/init.ts +121 -20
- package/src/cli/plan-format.ts +6 -2
- package/src/cli/plugin-commands.ts +4 -5
- package/src/cli/plugin-templates.ts +1 -1
- package/src/cli/run.ts +3 -1
- package/src/cli/upgrade.ts +39 -9
- package/src/config.ts +13 -10
- package/src/exec/sandbox-paths.ts +2 -2
- package/src/graph/scheduler.ts +3 -1
- package/src/graph/task-graph.ts +15 -11
- package/src/index.ts +2 -2
- package/src/orchestrator/affected-tasks.ts +0 -18
- package/src/orchestrator/doctor.ts +0 -17
- package/src/orchestrator/execute-task.ts +1 -1
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/options.ts +6 -8
- package/src/orchestrator/plan.ts +7 -1
- package/src/orchestrator/plugin-host.ts +6 -5
- package/src/orchestrator/plugin.ts +12 -5
- package/src/orchestrator/prepare.ts +16 -46
- package/src/orchestrator/projects.ts +8 -1
- package/src/orchestrator/remote-prefetch.ts +11 -4
- package/src/orchestrator/run-context.ts +48 -5
- package/src/orchestrator/run-settings.ts +3 -0
- package/src/orchestrator/run.ts +10 -26
- package/src/orchestrator/task-hash.ts +1 -3
- package/src/report/logger.ts +0 -4
- package/src/report/summary.ts +14 -13
- package/src/settings.ts +38 -15
- package/src/util/index.ts +1 -0
- package/src/util/paths.ts +42 -33
- package/src/util/size.ts +4 -5
- package/src/workspace/affected.ts +0 -39
- package/src/workspace/config-schema.ts +0 -36
- package/src/workspace/index.ts +0 -1
|
@@ -26,20 +26,18 @@ import {
|
|
|
26
26
|
writesRunStore,
|
|
27
27
|
type PluginCache,
|
|
28
28
|
type ResolvedRemote,
|
|
29
|
-
BranchScope,
|
|
30
29
|
applyGitEnumeration,
|
|
31
30
|
gitPathspecs,
|
|
32
31
|
MAX_SCOPED_PATHSPECS,
|
|
33
32
|
startGitEnumeration,
|
|
34
33
|
lazyGitEnumeration,
|
|
35
34
|
measureFsClockLag,
|
|
35
|
+
DEFAULT_STORE,
|
|
36
36
|
} from '../cache/index.js'
|
|
37
37
|
import {
|
|
38
38
|
buildPackageGraph,
|
|
39
39
|
computeNestedProjectDirs,
|
|
40
40
|
findWorkspaceRoot,
|
|
41
|
-
type AffectedChanges,
|
|
42
|
-
branchChanges,
|
|
43
41
|
configuredBaseBranch,
|
|
44
42
|
type LoadReads,
|
|
45
43
|
PROJECT_CONFIG_FILENAMES,
|
|
@@ -79,17 +77,17 @@ import {
|
|
|
79
77
|
lockReader,
|
|
80
78
|
type LoadedProjects,
|
|
81
79
|
} from './projects.js'
|
|
82
|
-
import { keptByAffected
|
|
80
|
+
import { keptByAffected } from './affected-tasks.js'
|
|
83
81
|
import { applyGraphStage } from './graph-stage.js'
|
|
84
82
|
import { keyExcludedDependencies } from './excluded-keys.js'
|
|
85
83
|
import { FingerprintWatch } from './fingerprint-watch.js'
|
|
86
84
|
import type { VxPlugin } from './plugin.js'
|
|
87
85
|
import { askIgnoredOutputs, gatedOutputs } from './stable-keys.js'
|
|
88
|
-
import { currentBranch, resolveCacheScope } from './run-context.js'
|
|
86
|
+
import { cacheRefs, currentBranch, resolveCacheScope } from './run-context.js'
|
|
89
87
|
import { createHashCache, type HashCache } from './task-hash.js'
|
|
90
88
|
import { type Logger, warnOf } from '../report/index.js'
|
|
91
89
|
import type { RunOptions } from './options.js'
|
|
92
|
-
import { runRules } from '../settings.js'
|
|
90
|
+
import { resolveSetting, runRules } from '../settings.js'
|
|
93
91
|
|
|
94
92
|
export interface PreparedRun {
|
|
95
93
|
workspaceRoot: string
|
|
@@ -263,7 +261,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
263
261
|
options.cacheScope ??
|
|
264
262
|
resolveCacheScope(
|
|
265
263
|
workspaceRoot,
|
|
266
|
-
branchFacts.scope,
|
|
264
|
+
branchFacts.trusted === true ? null : branchFacts.scope,
|
|
267
265
|
plugins.some((p) => p.cache !== undefined),
|
|
268
266
|
baseBranch,
|
|
269
267
|
)
|
|
@@ -289,33 +287,6 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
289
287
|
reused !== undefined
|
|
290
288
|
? Promise.resolve(reused.projects)
|
|
291
289
|
: discoverProjects(workspaceRoot, readLock, plugins, cacheDir, warnOf(log), git)
|
|
292
|
-
// What the branch changed decides which tasks read the scope (owner,
|
|
293
|
-
// 2026-10-17). Only a scoped run's first remote call asks git: a run the
|
|
294
|
-
// local cache answers makes no remote call, and diffing beside discovery
|
|
295
|
-
// cost it 20-60 ms at 1k-5k projects.
|
|
296
|
-
const scopeChanges: (() => Promise<AffectedChanges>) | undefined =
|
|
297
|
-
cacheScope === undefined
|
|
298
|
-
? undefined
|
|
299
|
-
: options.scopeChanges !== undefined
|
|
300
|
-
? async () => options.scopeChanges!
|
|
301
|
-
: () =>
|
|
302
|
-
branchChanges(
|
|
303
|
-
workspaceRoot,
|
|
304
|
-
options.scopeBase ?? branchFacts.base,
|
|
305
|
-
baseBranch,
|
|
306
|
-
discovering,
|
|
307
|
-
{
|
|
308
|
-
untracked: async () => (await git.start()).untracked,
|
|
309
|
-
// A scoped enumeration lists only its projects: null asks git.
|
|
310
|
-
links: async () => {
|
|
311
|
-
const g = await git.start()
|
|
312
|
-
if (!g.scopes.includes('.')) return null
|
|
313
|
-
const out: string[] = []
|
|
314
|
-
for (const [rel, oid] of g.trusted) if (oid.startsWith('120000:')) out.push(rel)
|
|
315
|
-
return out
|
|
316
|
-
},
|
|
317
|
-
},
|
|
318
|
-
)
|
|
319
290
|
const projectMetas = await discovering
|
|
320
291
|
mark('discover projects')
|
|
321
292
|
|
|
@@ -376,6 +347,12 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
376
347
|
options.artifactCeiling,
|
|
377
348
|
'open',
|
|
378
349
|
await storeRoot,
|
|
350
|
+
{
|
|
351
|
+
embeddings: (resolveSetting('cache.store.embeddings', workspaceConfig).value ??
|
|
352
|
+
DEFAULT_STORE.embeddings) as number,
|
|
353
|
+
decompress: (resolveSetting('cache.store.decompress', workspaceConfig).value ??
|
|
354
|
+
DEFAULT_STORE.decompress) as boolean,
|
|
355
|
+
},
|
|
379
356
|
)
|
|
380
357
|
const unwritable = localCache.readOnlyNotice()
|
|
381
358
|
if (unwritable !== null) warnOf(log)(`[vx] ${unwritable}`)
|
|
@@ -509,9 +486,6 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
509
486
|
// cache plugin.
|
|
510
487
|
let cache: CacheLayer
|
|
511
488
|
const remotes: ResolvedRemote[] = []
|
|
512
|
-
// A scope only picks remote keys (owner, 2026-10-17): the local store
|
|
513
|
-
// keeps every branch's entries under the task keys.
|
|
514
|
-
const branchScope = cacheScope === undefined ? undefined : new BranchScope(cacheScope)
|
|
515
489
|
if (options.remoteCache) {
|
|
516
490
|
remotes.push({
|
|
517
491
|
remote: options.remoteCache,
|
|
@@ -522,16 +496,19 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
522
496
|
}
|
|
523
497
|
try {
|
|
524
498
|
cache = options.remoteCache
|
|
525
|
-
? new LayeredCache(localCache, remotes,
|
|
499
|
+
? new LayeredCache(localCache, remotes, options.uploadConcurrency)
|
|
526
500
|
: await resolveCache(plugins, {
|
|
527
501
|
workspaceRoot,
|
|
528
502
|
cacheDir,
|
|
529
503
|
warn: warnOf(log),
|
|
530
504
|
scope: cacheScope,
|
|
505
|
+
refs: cacheRefs(workspaceRoot, baseBranch),
|
|
531
506
|
localCache,
|
|
532
507
|
policy,
|
|
533
508
|
remotes,
|
|
534
|
-
...(
|
|
509
|
+
...(options.uploadConcurrency !== undefined
|
|
510
|
+
? { uploadConcurrency: options.uploadConcurrency }
|
|
511
|
+
: {}),
|
|
535
512
|
})
|
|
536
513
|
} catch (err) {
|
|
537
514
|
// A cache plugin that throws or returns something off-contract is
|
|
@@ -763,13 +740,6 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
763
740
|
if (hasHook(plugins, 'key')) {
|
|
764
741
|
await applyKeyHooks(plugins, nodes, { workspaceRoot, cacheDir, warn: warnOf(log) })
|
|
765
742
|
}
|
|
766
|
-
// The final inputs decide which tasks the branch changed: they read
|
|
767
|
-
// the scope, every other task the default key.
|
|
768
|
-
if (branchScope !== undefined && scopeChanges !== undefined) {
|
|
769
|
-
branchScope.routeTasks(async () =>
|
|
770
|
-
keyedByChange(nodes, await scopeChanges(), projects, packageGraph),
|
|
771
|
-
)
|
|
772
|
-
}
|
|
773
743
|
refuseFiltersThroughGroups(nodes)
|
|
774
744
|
const exclude = options.excludeDependencies
|
|
775
745
|
let keyOnly: ReadonlyMap<string, TaskNode> = new Map()
|
|
@@ -71,7 +71,14 @@ export async function loadWorkspacePlugins(
|
|
|
71
71
|
const setBranch = (facts: BranchFacts): void => {
|
|
72
72
|
if (facts.scope !== undefined && !isCacheScope(facts.scope))
|
|
73
73
|
throw new UserError(`setBranch: ${cacheScopeError('scope')}`)
|
|
74
|
-
if (facts.
|
|
74
|
+
if (facts.trusted === true) {
|
|
75
|
+
delete branch.scope
|
|
76
|
+
branch.trusted = true
|
|
77
|
+
}
|
|
78
|
+
if (facts.scope !== undefined) {
|
|
79
|
+
branch.scope = facts.scope
|
|
80
|
+
delete branch.trusted
|
|
81
|
+
}
|
|
75
82
|
if (facts.base !== undefined) branch.base = facts.base
|
|
76
83
|
}
|
|
77
84
|
await applyConfigHooks(plugins, workspaceConfig, { workspaceRoot, warn, setBranch }, (plugin) =>
|
|
@@ -37,7 +37,7 @@ export interface PrefetchArgs {
|
|
|
37
37
|
nestedDirsByProject: Map<string, string[]>
|
|
38
38
|
gitFilesCache: GitFilesCache
|
|
39
39
|
hashCache: HashCache
|
|
40
|
-
/** Cap on concurrent in-flight prefetches
|
|
40
|
+
/** Cap on concurrent in-flight prefetches (`cache.concurrency.download`). */
|
|
41
41
|
concurrency: number
|
|
42
42
|
/** When false (remote reads disabled), no prefetch fires. */
|
|
43
43
|
remoteRead: boolean
|
|
@@ -75,10 +75,14 @@ export function startRemotePrefetch(args: PrefetchArgs): Promise<void> {
|
|
|
75
75
|
// a still-in-flight prefetch can never ingest into a closed DB. Errors
|
|
76
76
|
// degrade inside LayeredCache.prefetch; the catch keeps the handle
|
|
77
77
|
// from ever rejecting.
|
|
78
|
-
|
|
78
|
+
const started = Promise.withResolvers<void>()
|
|
79
|
+
args.cache.probeFirst?.(started.promise)
|
|
80
|
+
return runPrefetch(args, started.resolve)
|
|
81
|
+
.catch(() => {})
|
|
82
|
+
.finally(started.resolve)
|
|
79
83
|
}
|
|
80
84
|
|
|
81
|
-
async function runPrefetch(args: PrefetchArgs): Promise<void> {
|
|
85
|
+
async function runPrefetch(args: PrefetchArgs, probing: () => void): Promise<void> {
|
|
82
86
|
const stableKeys = await deriveStableKeys(args)
|
|
83
87
|
if (stableKeys.length === 0) return
|
|
84
88
|
|
|
@@ -94,7 +98,10 @@ async function runPrefetch(args: PrefetchArgs): Promise<void> {
|
|
|
94
98
|
let toPrefetch = held === null ? stableKeys : stableKeys.filter((k) => !held.has(k.hash))
|
|
95
99
|
if (toPrefetch.length === 0) return
|
|
96
100
|
const uniqueHashes = [...new Set(toPrefetch.map((k) => k.hash))]
|
|
97
|
-
const
|
|
101
|
+
const asked = args.cache.remoteHasMany?.(uniqueHashes)
|
|
102
|
+
// Every lookup that waited on the probe now finds its hash in it.
|
|
103
|
+
probing()
|
|
104
|
+
const present = (await asked) ?? null
|
|
98
105
|
if (present !== null) {
|
|
99
106
|
args.cache.markRemoteAbsent?.(uniqueHashes.filter((h) => !present.has(h)))
|
|
100
107
|
// The pre-mark above already answers a pull for an absent hash from
|
|
@@ -553,10 +553,51 @@ export function localBranchScope(
|
|
|
553
553
|
return branchCacheScope(repo, name)
|
|
554
554
|
}
|
|
555
555
|
|
|
556
|
-
/**
|
|
556
|
+
/**
|
|
557
|
+
* The refs a remote reads, in order (2026-10-20): the run's own branch,
|
|
558
|
+
* then the branch it merges into, then the default branch, each once. On
|
|
559
|
+
* CI from its env (a GitHub pull request's head branch, a merge queue's
|
|
560
|
+
* queue ref, a GitLab pipeline's ref), else from git: the checked-out
|
|
561
|
+
* branch, then the default (`origin/HEAD`). A remote plugin keys or scopes
|
|
562
|
+
* its entries by them (`scope` on turboCache/nxCache); core keys nothing
|
|
563
|
+
* by them.
|
|
564
|
+
*/
|
|
565
|
+
export function cacheRefs(
|
|
566
|
+
workspaceRoot: string,
|
|
567
|
+
baseBranch: string | undefined,
|
|
568
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
569
|
+
): string[] {
|
|
570
|
+
const set = (v: string | undefined): string | undefined =>
|
|
571
|
+
v !== undefined && v.trim() !== '' ? v.trim() : undefined
|
|
572
|
+
const head = (b: string | undefined) => (b === undefined ? undefined : `refs/heads/${b}`)
|
|
573
|
+
let own: string | undefined
|
|
574
|
+
let base: string | undefined
|
|
575
|
+
if (env['GITHUB_ACTIONS'] === 'true') {
|
|
576
|
+
own = head(set(env['GITHUB_HEAD_REF'])) ?? set(env['GITHUB_REF'])
|
|
577
|
+
base = head(set(env['GITHUB_BASE_REF']))
|
|
578
|
+
const queue = /^refs\/heads\/gh-readonly-queue\/(.+)\/pr-\d+-[0-9a-f]+$/.exec(own ?? '')
|
|
579
|
+
if (queue !== null) base = head(queue[1])
|
|
580
|
+
} else if (env['GITLAB_CI'] === 'true') {
|
|
581
|
+
const tag = set(env['CI_COMMIT_TAG'])
|
|
582
|
+
own = tag !== undefined ? `refs/tags/${tag}` : head(set(env['CI_COMMIT_REF_NAME']))
|
|
583
|
+
base = head(set(env['CI_MERGE_REQUEST_TARGET_BRANCH_NAME']))
|
|
584
|
+
} else {
|
|
585
|
+
own = head(
|
|
586
|
+
(readHeadDirect(workspaceRoot) ?? captureGitContext(workspaceRoot, null, env)).branch ??
|
|
587
|
+
undefined,
|
|
588
|
+
)
|
|
589
|
+
}
|
|
590
|
+
const trunk = head(baseBranch ?? captureDefaultBranch(env, workspaceRoot) ?? undefined)
|
|
591
|
+
return [...new Set([own, base, trunk].filter((r): r is string => r !== undefined))]
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
/**
|
|
595
|
+
* The repository's path on its host (`owner/repo`; a GitLab subgroup's
|
|
596
|
+
* `group/sub/repo` whole, as `gitlab()` names it); undefined for a local path.
|
|
597
|
+
*/
|
|
557
598
|
function repoOfUrl(url: string): string | undefined {
|
|
558
599
|
const parts = normalizeRemoteUrl(url).split('/')
|
|
559
|
-
return parts.length < 3 ? undefined : parts.slice(
|
|
600
|
+
return parts.length < 3 ? undefined : parts.slice(1).join('/')
|
|
560
601
|
}
|
|
561
602
|
|
|
562
603
|
/** `.git/config`'s sections, each a key → value map (last wins); empty when it cannot be read. */
|
|
@@ -586,12 +627,13 @@ function gitConfigSections(workspaceRoot: string): Map<string, Map<string, strin
|
|
|
586
627
|
|
|
587
628
|
/**
|
|
588
629
|
* The run's cache scope (owner, 2026-10-17): `VX_CACHE_SCOPE` (empty: none),
|
|
589
|
-
* else a plugin's (`setBranch`, `github()` on Actions
|
|
590
|
-
*
|
|
630
|
+
* else a plugin's (`setBranch`, `github()` on Actions; null: the plugin
|
|
631
|
+
* named the run trusted), else, with a cache plugin to share it, the
|
|
632
|
+
* branch's from git. Undefined: no scope. It keys plugin entries only.
|
|
591
633
|
*/
|
|
592
634
|
export function resolveCacheScope(
|
|
593
635
|
workspaceRoot: string,
|
|
594
|
-
pluginScope: string | undefined,
|
|
636
|
+
pluginScope: string | null | undefined,
|
|
595
637
|
hasRemote: boolean,
|
|
596
638
|
baseBranch: string | undefined,
|
|
597
639
|
env: NodeJS.ProcessEnv = process.env,
|
|
@@ -602,6 +644,7 @@ export function resolveCacheScope(
|
|
|
602
644
|
if (!isCacheScope(named)) throw cacheScopeEnvError()
|
|
603
645
|
return named
|
|
604
646
|
}
|
|
647
|
+
if (pluginScope === null) return undefined
|
|
605
648
|
return pluginScope ?? (hasRemote ? localBranchScope(workspaceRoot, baseBranch, env) : undefined)
|
|
606
649
|
}
|
|
607
650
|
|
|
@@ -14,6 +14,9 @@ const FROM_OPTIONS = {
|
|
|
14
14
|
retries: 'run.retries',
|
|
15
15
|
continueMode: 'run.continue',
|
|
16
16
|
download: 'cache.download',
|
|
17
|
+
downloadConcurrency: 'cache.concurrency.download',
|
|
18
|
+
uploadConcurrency: 'cache.concurrency.upload',
|
|
19
|
+
restoreConcurrency: 'cache.concurrency.restore',
|
|
17
20
|
} as const satisfies Partial<Record<keyof RunOptions, string>>
|
|
18
21
|
|
|
19
22
|
/**
|
package/src/orchestrator/run.ts
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
type CachePolicy,
|
|
12
12
|
cachesNothing,
|
|
13
13
|
formatCachePolicy,
|
|
14
|
+
networkConcurrency,
|
|
14
15
|
stopRuntimeProbes,
|
|
15
16
|
sweepStaging,
|
|
16
17
|
} from '../cache/index.js'
|
|
@@ -259,6 +260,10 @@ function refuseRunOptions(options: RunOptions): void {
|
|
|
259
260
|
const { concurrency, retries, timeout } = options
|
|
260
261
|
if (concurrency !== undefined && !(Number.isInteger(concurrency) && concurrency > 0))
|
|
261
262
|
refuse('concurrency', concurrency, 'a positive integer')
|
|
263
|
+
for (const field of ['downloadConcurrency', 'uploadConcurrency', 'restoreConcurrency'] as const) {
|
|
264
|
+
const v = options[field]
|
|
265
|
+
if (v !== undefined && !(Number.isInteger(v) && v > 0)) refuse(field, v, 'a positive integer')
|
|
266
|
+
}
|
|
262
267
|
if (retries !== undefined && !(Number.isInteger(retries) && retries >= 0))
|
|
263
268
|
refuse('retries', retries, 'a non-negative integer')
|
|
264
269
|
if (
|
|
@@ -618,7 +623,7 @@ async function runOnBus(
|
|
|
618
623
|
// `vx cache clean` can evict entries the run just hit. Once-only
|
|
619
624
|
// because the normal path closes before the persistent-task wait —
|
|
620
625
|
// holding the handle open for a dev server's whole lifetime would be
|
|
621
|
-
// worse
|
|
626
|
+
// worse.
|
|
622
627
|
let cacheClosed = false
|
|
623
628
|
const closeCache = async (): Promise<void> => {
|
|
624
629
|
if (cacheClosed) return
|
|
@@ -805,6 +810,8 @@ async function runOnBus(
|
|
|
805
810
|
localWrite: policy.localWrite,
|
|
806
811
|
remote: remoteCacheEnabled && (policy.remoteRead || policy.remoteWrite),
|
|
807
812
|
traffic: prepared.localCache.traffic,
|
|
813
|
+
downloads: prepared.localCache.downloads,
|
|
814
|
+
uploads: prepared.localCache.uploads,
|
|
808
815
|
notDownloaded: () => deferredOutputs.pending().length,
|
|
809
816
|
},
|
|
810
817
|
concurrency,
|
|
@@ -871,7 +878,7 @@ async function runOnBus(
|
|
|
871
878
|
nestedDirsByProject,
|
|
872
879
|
gitFilesCache,
|
|
873
880
|
hashCache,
|
|
874
|
-
concurrency,
|
|
881
|
+
concurrency: options.downloadConcurrency ?? networkConcurrency(),
|
|
875
882
|
remoteRead: policy.remoteRead,
|
|
876
883
|
remoteWrite: policy.remoteWrite,
|
|
877
884
|
}),
|
|
@@ -1088,6 +1095,7 @@ async function runOnBus(
|
|
|
1088
1095
|
const outcomes = await runGraph({
|
|
1089
1096
|
nodes,
|
|
1090
1097
|
concurrency,
|
|
1098
|
+
restoreConcurrency: options.restoreConcurrency ?? 2 * concurrency,
|
|
1091
1099
|
settledOf: (o) => deferredSaves.get(o.node.id),
|
|
1092
1100
|
...(hasPooledExecutor(executors) ? { poolOf: poolOfPlacement(placements) } : {}),
|
|
1093
1101
|
...(admit !== undefined ? { admit } : {}),
|
|
@@ -1349,9 +1357,6 @@ async function runOnBus(
|
|
|
1349
1357
|
// The miss path's output-directory snapshots first (miss-save.ts).
|
|
1350
1358
|
await step('vx:snapshots', () => recordOutputDirSnapshots(cache, outputDirSnapshots))
|
|
1351
1359
|
mark('output dir snapshots')
|
|
1352
|
-
// Not on a stopped run: one stopped while it waited on another run's
|
|
1353
|
-
// lock never held it, and its prune evicted under that run (item 858).
|
|
1354
|
-
if (!stop.signal.aborted) await step('vx:retention', () => applyCacheRetention(prepared))
|
|
1355
1360
|
// A plugin hears the run until its teardown and nothing after: released
|
|
1356
1361
|
// only in the finally, its handlers heard a kept server through the
|
|
1357
1362
|
// whole keep-alive wait below (C-66). Idempotent; the finally's stay.
|
|
@@ -1488,27 +1493,6 @@ async function runOnBus(
|
|
|
1488
1493
|
}
|
|
1489
1494
|
}
|
|
1490
1495
|
|
|
1491
|
-
/**
|
|
1492
|
-
* `cache.retention`, after every save and upload of this run
|
|
1493
|
-
* has landed (so nothing this run wrote is mid-flight) and before the cache
|
|
1494
|
-
* closes. Housekeeping, not the run's work: silent, and a failure is never
|
|
1495
|
-
* a failed run. Declared nowhere → one property read.
|
|
1496
|
-
*/
|
|
1497
|
-
async function applyCacheRetention(prepared: PreparedRun): Promise<void> {
|
|
1498
|
-
const ws = prepared.workspaceConfig
|
|
1499
|
-
const maxAgeMs = resolveSetting('cache.retention.olderThan', ws).value as number | undefined
|
|
1500
|
-
const maxBytes = resolveSetting('cache.retention.maxSize', ws).value as number | undefined
|
|
1501
|
-
if (maxAgeMs === undefined && maxBytes === undefined) return
|
|
1502
|
-
try {
|
|
1503
|
-
await prepared.localCache.evictIfDue({
|
|
1504
|
-
...(maxAgeMs !== undefined ? { maxAgeMs } : {}),
|
|
1505
|
-
...(maxBytes !== undefined ? { maxBytes } : {}),
|
|
1506
|
-
})
|
|
1507
|
-
} catch {
|
|
1508
|
-
// Housekeeping says nothing (owner): the next due run tries again.
|
|
1509
|
-
}
|
|
1510
|
-
}
|
|
1511
|
-
|
|
1512
1496
|
/** Why an `--affected` run has nothing to run, said alike by the run and its `--dry` plan. */
|
|
1513
1497
|
function noneAffected(prepared: PreparedRun, options: RunOptions): string {
|
|
1514
1498
|
// A changed project declares the task, so `--affected`'s task walk is
|
|
@@ -97,9 +97,7 @@ export interface ComputeHashArgs {
|
|
|
97
97
|
*/
|
|
98
98
|
export async function computeTaskHash(args: ComputeHashArgs): Promise<string> {
|
|
99
99
|
if (isKeyless(args.node.config)) return keylessHash(args.node.id)
|
|
100
|
-
|
|
101
|
-
args.cache.noteKey?.(args.node.id, key)
|
|
102
|
-
return key
|
|
100
|
+
return taskHash(args)
|
|
103
101
|
}
|
|
104
102
|
|
|
105
103
|
async function taskHash(args: ComputeHashArgs): Promise<string> {
|
package/src/report/logger.ts
CHANGED
|
@@ -382,7 +382,6 @@ export function defaultLogger(
|
|
|
382
382
|
let saved = 0
|
|
383
383
|
let notSaved = 0
|
|
384
384
|
let skippedCount = 0
|
|
385
|
-
let savedMs = 0
|
|
386
385
|
// Each finished task's cache bucket by its key: its remote traffic moves it.
|
|
387
386
|
const buckets = new Map<string, CacheBucket>()
|
|
388
387
|
// The graph is done and vx's own closing steps run: the region stays,
|
|
@@ -530,7 +529,6 @@ export function defaultLogger(
|
|
|
530
529
|
notSaved,
|
|
531
530
|
miss: succeeded + failed - noCache,
|
|
532
531
|
noCache,
|
|
533
|
-
savedMs,
|
|
534
532
|
left: total - done - stopped - notRun,
|
|
535
533
|
...(etaMs !== undefined && !tail && !stopping ? { etaMs } : {}),
|
|
536
534
|
buckets,
|
|
@@ -1028,12 +1026,10 @@ export function defaultLogger(
|
|
|
1028
1026
|
case 'cache-hit':
|
|
1029
1027
|
if (outcome.restored === false) upToDate++
|
|
1030
1028
|
else restoredLocal++
|
|
1031
|
-
savedMs += outcome.storedDurationMs ?? 0
|
|
1032
1029
|
break
|
|
1033
1030
|
case 'cache-hit-remote':
|
|
1034
1031
|
if (outcome.restored === false) upToDate++
|
|
1035
1032
|
else restoredRemote++
|
|
1036
|
-
savedMs += outcome.storedDurationMs ?? 0
|
|
1037
1033
|
break
|
|
1038
1034
|
case 'skipped':
|
|
1039
1035
|
skippedCount++
|
package/src/report/summary.ts
CHANGED
|
@@ -5,12 +5,12 @@
|
|
|
5
5
|
|
|
6
6
|
import type { TaskOutcome } from '../graph/index.js'
|
|
7
7
|
import { tallyOutcomes } from './tally.js'
|
|
8
|
-
import { isCacheHit } from '../graph/index.js'
|
|
9
8
|
import { outcomeLabel, projectOutcome, ranNoCache } from './events.js'
|
|
10
9
|
import { isGroupTask } from '../graph/index.js'
|
|
11
10
|
import {
|
|
12
11
|
barWidth,
|
|
13
12
|
ERROR,
|
|
13
|
+
formatBytes,
|
|
14
14
|
formatElapsed,
|
|
15
15
|
gradientRule,
|
|
16
16
|
paint,
|
|
@@ -106,8 +106,6 @@ export interface SummaryStats {
|
|
|
106
106
|
held?: { count: number; sumMs: number }
|
|
107
107
|
/** Live section only: the forecast of what is left, from history. */
|
|
108
108
|
etaMs?: number
|
|
109
|
-
/** The hits' stored exec times summed: the work the cache skipped. */
|
|
110
|
-
savedMs?: number
|
|
111
109
|
/** A run of groups only: their names, so `0 tasks` says why. */
|
|
112
110
|
emptyGroups?: readonly string[]
|
|
113
111
|
/** Each finished task's cache bucket by its key and its id (its run bytes), so its remote traffic can move it. */
|
|
@@ -206,6 +204,9 @@ export interface RunContext {
|
|
|
206
204
|
readonly inFlight: number
|
|
207
205
|
readonly keys: ReadonlyMap<string, { uploading: number; failures: readonly string[] }>
|
|
208
206
|
}
|
|
207
|
+
/** Bytes the run's remote hits pulled, and its landed uploads pushed (live refs). */
|
|
208
|
+
downloads?: { readonly bytes: number }
|
|
209
|
+
uploads?: { readonly bytes: number }
|
|
209
210
|
/** Tasks whose outputs are still in the remote store. */
|
|
210
211
|
notDownloaded: () => number
|
|
211
212
|
}
|
|
@@ -338,9 +339,17 @@ export function formatSummarySection(
|
|
|
338
339
|
]
|
|
339
340
|
// The layer words are dim; only the counts carry colour (M, 2026-10-15).
|
|
340
341
|
if (local.length > 0) cacheParts.push(`${dim('local')} ${local.join(' ')}`)
|
|
342
|
+
// Each transfer count is followed by its bytes, dim: what the
|
|
343
|
+
// remote moved is the cost a hit or an upload paid.
|
|
344
|
+
const sized = (count: number, glyph: string, bytes: number | undefined): string[] =>
|
|
345
|
+
count === 0
|
|
346
|
+
? []
|
|
347
|
+
: bytes === undefined || bytes === 0
|
|
348
|
+
? n(count, glyph, REMOTE)
|
|
349
|
+
: [...n(count, glyph, REMOTE), dim(formatBytes(bytes))]
|
|
341
350
|
const remote = [
|
|
342
|
-
...
|
|
343
|
-
...
|
|
351
|
+
...sized(b.restoredRemote, '\u2193', layers.downloads?.bytes),
|
|
352
|
+
...sized(b.uploaded, '\u2191', layers.uploads?.bytes),
|
|
344
353
|
...n(b.remoteFailed, '\u2717', ERROR),
|
|
345
354
|
...n(b.uploading, '\u21e1', ACCENT),
|
|
346
355
|
]
|
|
@@ -392,10 +401,6 @@ export function formatSummarySection(
|
|
|
392
401
|
}
|
|
393
402
|
}
|
|
394
403
|
const timeParts = [total]
|
|
395
|
-
// What the cache paid back, beside what the run cost: the stored exec
|
|
396
|
-
// time of every hit, never the restore this run spent on it.
|
|
397
|
-
if ((stats.savedMs ?? 0) >= 1)
|
|
398
|
-
timeParts.push(paint(SUCCESS, `${formatElapsed(stats.savedMs!)} saved`, colors))
|
|
399
404
|
// The live run's forecast leads the spread: the one number on the row
|
|
400
405
|
// that is about the future.
|
|
401
406
|
if (stats.etaMs !== undefined)
|
|
@@ -492,9 +497,6 @@ export function formatRunSummary(
|
|
|
492
497
|
const durations = outcomes
|
|
493
498
|
.filter((o) => (o.status === 'success' || o.status === 'failed') && !isGroupTask(o.node))
|
|
494
499
|
.map((o) => o.durationMs)
|
|
495
|
-
let savedMs = 0
|
|
496
|
-
for (const o of outcomes)
|
|
497
|
-
if (isCacheHit(o.status) && !isGroupTask(o.node)) savedMs += o.storedDurationMs ?? 0
|
|
498
500
|
const heldOutcomes = outcomes.filter((o) => o.admissionHeldMs !== undefined)
|
|
499
501
|
const notRun = outcomes.filter((o) => neverStarted(o) && !isGroupTask(o.node)).length
|
|
500
502
|
const emptyGroups = outcomes.every((o) => isGroupTask(o.node))
|
|
@@ -515,7 +517,6 @@ export function formatRunSummary(
|
|
|
515
517
|
notSaved: outcomes.filter((o) => !isGroupTask(o.node) && ranUnsaved(o)).length,
|
|
516
518
|
miss: t.total - t.skipped - (t.cachedLocal + t.cachedRemote) - noCache,
|
|
517
519
|
noCache,
|
|
518
|
-
savedMs,
|
|
519
520
|
spread:
|
|
520
521
|
durations.length > 0
|
|
521
522
|
? {
|
package/src/settings.ts
CHANGED
|
@@ -17,7 +17,8 @@ export type SettingKind =
|
|
|
17
17
|
| { readonly type: 'concurrency' }
|
|
18
18
|
// `zero`: 0 is a value ("wait for nothing not already done"), not a typo.
|
|
19
19
|
| { readonly type: 'duration'; readonly timer?: true; readonly zero?: true }
|
|
20
|
-
|
|
20
|
+
// `off`: `false` is a value, 0 bytes (the thing the size bounds is off).
|
|
21
|
+
| { readonly type: 'size'; readonly off?: true }
|
|
21
22
|
| { readonly type: 'bool' }
|
|
22
23
|
| { readonly type: 'enum'; readonly values: readonly string[] }
|
|
23
24
|
| { readonly type: 'string' }
|
|
@@ -115,18 +116,6 @@ export const SETTINGS: readonly Setting[] = [
|
|
|
115
116
|
doc: 'Where the cache lives, relative to the workspace root; named, it holds the whole cache.',
|
|
116
117
|
flag: { verb: 'run', name: '--cache-dir' },
|
|
117
118
|
},
|
|
118
|
-
{
|
|
119
|
-
path: 'cache.retention.maxSize',
|
|
120
|
-
kind: { type: 'size' },
|
|
121
|
-
default: 'none',
|
|
122
|
-
doc: "Evict least-recently-used entries at each run's end until the cache is under this (`'10GB'`).",
|
|
123
|
-
},
|
|
124
|
-
{
|
|
125
|
-
path: 'cache.retention.olderThan',
|
|
126
|
-
kind: { type: 'duration' },
|
|
127
|
-
default: 'none',
|
|
128
|
-
doc: "Evict entries unused for this long at each run's end (`'30d'`).",
|
|
129
|
-
},
|
|
130
119
|
{
|
|
131
120
|
path: 'cache.policy',
|
|
132
121
|
kind: { type: 'string' },
|
|
@@ -138,9 +127,30 @@ export const SETTINGS: readonly Setting[] = [
|
|
|
138
127
|
path: 'cache.scope',
|
|
139
128
|
kind: { type: 'string' },
|
|
140
129
|
default: "the branch's, `<owner>/<repo>:<branch>`, with a remote cache off `baseBranch`",
|
|
141
|
-
doc: "
|
|
130
|
+
doc: "The branch scope plugin entries are keyed under (`'vznjs/vx:feat-x'`); task keys never carry it. Empty: no scope.",
|
|
142
131
|
config: false,
|
|
143
132
|
},
|
|
133
|
+
{
|
|
134
|
+
path: 'cache.concurrency.download',
|
|
135
|
+
kind: { type: 'int', min: 1 },
|
|
136
|
+
default: 'min(96, max(cores × 3, 64))',
|
|
137
|
+
doc: 'Remote downloads at once.',
|
|
138
|
+
flag: { verb: 'run', name: '--cache-concurrency-download' },
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
path: 'cache.concurrency.upload',
|
|
142
|
+
kind: { type: 'int', min: 1 },
|
|
143
|
+
default: 'min(96, max(cores × 3, 64))',
|
|
144
|
+
doc: 'Uploads at once, per remote.',
|
|
145
|
+
flag: { verb: 'run', name: '--cache-concurrency-upload' },
|
|
146
|
+
},
|
|
147
|
+
{
|
|
148
|
+
path: 'cache.concurrency.restore',
|
|
149
|
+
kind: { type: 'int', min: 1 },
|
|
150
|
+
default: 'twice `run.concurrency`',
|
|
151
|
+
doc: 'Local hits restoring outputs at once.',
|
|
152
|
+
flag: { verb: 'run', name: '--cache-concurrency-restore' },
|
|
153
|
+
},
|
|
144
154
|
{
|
|
145
155
|
path: 'cache.download',
|
|
146
156
|
kind: { type: 'enum', values: ['all', 'toplevel', 'none'] },
|
|
@@ -170,6 +180,18 @@ export const SETTINGS: readonly Setting[] = [
|
|
|
170
180
|
task: 'cache.outputs.clean',
|
|
171
181
|
env: false,
|
|
172
182
|
},
|
|
183
|
+
{
|
|
184
|
+
path: 'cache.store.embeddings',
|
|
185
|
+
kind: { type: 'size', off: true },
|
|
186
|
+
default: "'32KiB'",
|
|
187
|
+
doc: 'Keep an entry up to this size inside the cache index, one zstd tar, rather than as files; `false` keeps none.',
|
|
188
|
+
},
|
|
189
|
+
{
|
|
190
|
+
path: 'cache.store.decompress',
|
|
191
|
+
kind: { type: 'bool' },
|
|
192
|
+
default: 'true',
|
|
193
|
+
doc: 'Keep a larger entry as its plain files, which a hit copies or clones out; off, as one `.tar.zst`.',
|
|
194
|
+
},
|
|
173
195
|
{
|
|
174
196
|
path: 'cache.upload.failures',
|
|
175
197
|
kind: { type: 'enum', values: ['ci', 'local', 'always'] },
|
|
@@ -250,7 +272,7 @@ export function kindHint(kind: SettingKind): string {
|
|
|
250
272
|
? "a duration like '500ms', '30s' or '5m', at most 24 days"
|
|
251
273
|
: "a duration like '45s', '12h' or '30d'"
|
|
252
274
|
case 'size':
|
|
253
|
-
return "a size like '500MB' or '10GB'"
|
|
275
|
+
return kind.off === true ? "false, or a size like '32KiB'" : "a size like '500MB' or '10GB'"
|
|
254
276
|
case 'bool':
|
|
255
277
|
return 'true or false'
|
|
256
278
|
case 'enum':
|
|
@@ -298,6 +320,7 @@ export function parseSettingValue(kind: SettingKind, raw: unknown): unknown {
|
|
|
298
320
|
return kind.timer === true && n > MAX_TIMEOUT_MS ? null : n
|
|
299
321
|
}
|
|
300
322
|
case 'size':
|
|
323
|
+
if (kind.off === true && (raw === false || raw === 'false')) return 0
|
|
301
324
|
return typeof raw === 'string' ? parseSize(raw) : null
|
|
302
325
|
case 'bool':
|
|
303
326
|
if (typeof raw === 'boolean') return raw
|