@vzn/vx 0.0.643 → 0.0.644
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/schemas/info.json +0 -11
- package/skills/vx/SKILL.md +1 -1
- package/src/cache/archive.ts +153 -2
- package/src/cache/cache.ts +416 -176
- package/src/cache/file-hashes.ts +2 -14
- package/src/cache/fold-memo.ts +7 -3
- package/src/cache/index.ts +3 -0
- package/src/cache/key-fold.ts +4 -1
- package/src/cache/layer.ts +9 -4
- package/src/cache/layered-cache.ts +55 -3
- package/src/cache/plugin-cache.ts +33 -32
- package/src/cache/remote-lane.ts +12 -7
- package/src/cache/tar-stream.ts +12 -7
- package/src/cli/format.ts +1 -2
- package/src/cli/help.ts +6 -0
- package/src/cli/info.ts +2 -19
- package/src/cli/plugin-commands.ts +1 -1
- package/src/cli/run.ts +3 -0
- package/src/cli/upgrade.ts +38 -8
- package/src/config.ts +15 -10
- package/src/exec/sandbox-paths.ts +2 -2
- package/src/graph/scheduler.ts +3 -1
- package/src/graph/task-graph.ts +15 -11
- package/src/index.ts +1 -1
- package/src/orchestrator/doctor.ts +0 -17
- package/src/orchestrator/execute-task.ts +1 -1
- package/src/orchestrator/options.ts +6 -0
- package/src/orchestrator/plugin-host.ts +6 -2
- package/src/orchestrator/plugin.ts +4 -1
- package/src/orchestrator/prepare.ts +16 -3
- package/src/orchestrator/projects.ts +8 -1
- package/src/orchestrator/remote-prefetch.ts +11 -4
- package/src/orchestrator/run-context.ts +10 -5
- package/src/orchestrator/run-settings.ts +3 -0
- package/src/orchestrator/run.ts +8 -26
- package/src/report/logger.ts +0 -4
- package/src/report/summary.ts +0 -11
- package/src/settings.ts +43 -14
- package/src/util/index.ts +1 -0
- package/src/util/paths.ts +42 -33
- package/src/util/size.ts +4 -5
- package/src/workspace/config-schema.ts +0 -36
package/src/config.ts
CHANGED
|
@@ -79,15 +79,6 @@ export interface CacheSettings {
|
|
|
79
79
|
* `.vx/cache` and its entries in the user's shared store.
|
|
80
80
|
*/
|
|
81
81
|
dir?: string
|
|
82
|
-
/**
|
|
83
|
-
* Evict from the local cache at the end of every run, whatever its
|
|
84
|
-
* policy (a remote hit is ingested under any), by the same policy
|
|
85
|
-
* `vx cache clean` takes as flags: entries not used for `olderThan`
|
|
86
|
-
* (`'30d'`, `'12h'`), then least-recently-used entries until the cache
|
|
87
|
-
* is under `maxSize` (`'10GB'`, `'500MB'`). Either or both. Omitted →
|
|
88
|
-
* nothing is evicted unless you run `vx cache clean`.
|
|
89
|
-
*/
|
|
90
|
-
retention?: { olderThan?: Duration; maxSize?: string }
|
|
91
82
|
/**
|
|
92
83
|
* Which layers a run reads and writes (`'local:rw,remote:r'`; a layer it
|
|
93
84
|
* leaves out stays on); `--no-cache` and `--force` still apply on top.
|
|
@@ -107,6 +98,21 @@ export interface CacheSettings {
|
|
|
107
98
|
* both (`'always'`). Only a run whose policy writes a remote keeps any.
|
|
108
99
|
*/
|
|
109
100
|
upload?: { failures?: 'ci' | 'local' | 'always' }
|
|
101
|
+
/**
|
|
102
|
+
* How the local cache keeps an entry. `embeddings` (default `'32KiB'`):
|
|
103
|
+
* an entry up to this size inside the index; `false`, none. `decompress` (default true): a larger one as its
|
|
104
|
+
* plain files, which a hit copies or clones out with no decode; off, as
|
|
105
|
+
* one `.tar.zst`. `verify` (default `'stat'`): what a hit on plain files
|
|
106
|
+
* checks first, each file's size and mtime, every byte (`'hash'`), or
|
|
107
|
+
* nothing (`'none'`).
|
|
108
|
+
*/
|
|
109
|
+
store?: { embeddings?: false | string; decompress?: boolean; verify?: 'stat' | 'hash' | 'none' }
|
|
110
|
+
/**
|
|
111
|
+
* Cache transfers at once. `download` and `upload` (per remote) default
|
|
112
|
+
* to min(96, max(cores × 3, 64)); `restore`, local hits writing their
|
|
113
|
+
* outputs, to twice `run.concurrency`.
|
|
114
|
+
*/
|
|
115
|
+
concurrency?: { download?: number; upload?: number; restore?: number }
|
|
110
116
|
}
|
|
111
117
|
|
|
112
118
|
/** `WorkspaceConfig.lock`. */
|
|
@@ -786,7 +792,6 @@ export function defineWorkspace<T extends WorkspaceConfig>(
|
|
|
786
792
|
rules?: Known<At<At<T, 'run'>, 'rules'>, WorkspaceRules>
|
|
787
793
|
}
|
|
788
794
|
cache?: Known<At<T, 'cache'>, CacheSettings> & {
|
|
789
|
-
retention?: Known<At<At<T, 'cache'>, 'retention'>, NonNullable<CacheSettings['retention']>>
|
|
790
795
|
inputStability?: Known<At<At<T, 'cache'>, 'inputStability'>, InputStability>
|
|
791
796
|
}
|
|
792
797
|
lock?: Known<At<T, 'lock'>, LockSettings>
|
|
@@ -16,8 +16,8 @@ import path from 'node:path'
|
|
|
16
16
|
* move that spelling into the scan, which finds nothing before the task has
|
|
17
17
|
* written, and turn a working grant into `Read-only file system`. The two
|
|
18
18
|
* predicates answer different questions: whether a declaration must be
|
|
19
|
-
* MATCHED against other declarations (495; `GLOB_WILDCARDS
|
|
20
|
-
*
|
|
19
|
+
* MATCHED against other declarations (495; `GLOB_WILDCARDS`), and whether a
|
|
20
|
+
* grant can be mounted (here).
|
|
21
21
|
* Item 577 gave each question one spelling.
|
|
22
22
|
*/
|
|
23
23
|
export const MOUNT_WILDCARDS = /[*?[\]]/
|
package/src/graph/scheduler.ts
CHANGED
|
@@ -239,6 +239,8 @@ export type ContinueMode = 'never' | 'deps-ok' | 'always'
|
|
|
239
239
|
export interface ScheduleOptions {
|
|
240
240
|
nodes: Map<string, TaskNode>
|
|
241
241
|
concurrency: number
|
|
242
|
+
/** Restore-tier tasks at once; twice `concurrency` unset. */
|
|
243
|
+
restoreConcurrency?: number
|
|
242
244
|
/**
|
|
243
245
|
* An aborted run dispatches nothing further: every task not yet
|
|
244
246
|
* started completes `aborted` (the ones in flight are being killed by
|
|
@@ -604,7 +606,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
|
|
|
604
606
|
// the two lanes share its one slot, where a lane each let an 80 MB
|
|
605
607
|
// restore run inside another task's execution (item 1103).
|
|
606
608
|
let activeRestore = 0
|
|
607
|
-
const restoreConcurrency = 2 * concurrency
|
|
609
|
+
const restoreConcurrency = options.restoreConcurrency ?? 2 * concurrency
|
|
608
610
|
const serial = concurrency === 1
|
|
609
611
|
const execRoom = (): boolean => (serial ? active + activeRestore < 1 : active < concurrency)
|
|
610
612
|
const restoreRoom = (): boolean =>
|
package/src/graph/task-graph.ts
CHANGED
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
checkIgnoreSync,
|
|
5
5
|
ignoreProbes,
|
|
6
6
|
isLiteralPattern,
|
|
7
|
+
literalPath,
|
|
7
8
|
listedSync,
|
|
8
9
|
normalizeGlob,
|
|
9
10
|
relPosix,
|
|
@@ -915,15 +916,17 @@ export function outputsOverlap(rawA: string, rawB: string): boolean {
|
|
|
915
916
|
// A built path proves overlap only between globs as written: a literal's
|
|
916
917
|
// `/**` twin reads it as a directory, and `dist/types.d.ts` met
|
|
917
918
|
// `dist/**/*.js` at `dist/types.d.ts/x.js`, a file no one can write.
|
|
918
|
-
const bothGlobs =
|
|
919
|
+
const bothGlobs = literalPath(rawA) === null && literalPath(rawB) === null
|
|
919
920
|
for (const a of asTrees([rawA])) {
|
|
920
921
|
for (const b of asTrees([rawB])) {
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
if (
|
|
925
|
-
} else if (
|
|
926
|
-
if (taskGlob(
|
|
922
|
+
const la = literalPath(a)
|
|
923
|
+
const lb = literalPath(b)
|
|
924
|
+
if (la !== null && lb !== null) {
|
|
925
|
+
if (la === lb) return true
|
|
926
|
+
} else if (la !== null) {
|
|
927
|
+
if (taskGlob(b).match(la)) return true
|
|
928
|
+
} else if (lb !== null) {
|
|
929
|
+
if (taskGlob(a).match(lb)) return true
|
|
927
930
|
} else if (a === b || covers(a, b) || covers(b, a) || (bothGlobs && witnessed(a, b)))
|
|
928
931
|
return true
|
|
929
932
|
}
|
|
@@ -1202,13 +1205,13 @@ function collideAcross(
|
|
|
1202
1205
|
* found no match outside the head this cuts, and thousands once `\` and
|
|
1203
1206
|
* `!` were left out (item 746).
|
|
1204
1207
|
*/
|
|
1205
|
-
const GLOB_HEAD_END = /[*?{}\\!]/
|
|
1208
|
+
const GLOB_HEAD_END = /[*?{}[\]\\!]/
|
|
1206
1209
|
|
|
1207
1210
|
/** `firstSegment`'s answer for a glob that can start anywhere. */
|
|
1208
1211
|
const ANYWHERE = ''
|
|
1209
1212
|
|
|
1210
1213
|
/** Where a glob's first segment ends: a separator, or anything `GLOB_HEAD_END` stops at. */
|
|
1211
|
-
const FIRST_SEGMENT_END = /[*?{}\\!/]/
|
|
1214
|
+
const FIRST_SEGMENT_END = /[*?{}[\]\\!/]/
|
|
1212
1215
|
|
|
1213
1216
|
const firstSegments = new Map<string, string>()
|
|
1214
1217
|
|
|
@@ -1315,8 +1318,9 @@ function overlapCandidates<T>(
|
|
|
1315
1318
|
const s = sideOf?.(tasks[i]!) === true ? 1 : 0
|
|
1316
1319
|
side.push(s)
|
|
1317
1320
|
for (const tree of asTrees(globsOf(tasks[i]!) ?? [])) {
|
|
1318
|
-
|
|
1319
|
-
|
|
1321
|
+
const lit = literalPath(tree)
|
|
1322
|
+
if (lit !== null) {
|
|
1323
|
+
literals.push([i, lit])
|
|
1320
1324
|
continue
|
|
1321
1325
|
}
|
|
1322
1326
|
file(globs[s]!, tree, i)
|
package/src/index.ts
CHANGED
|
@@ -20,7 +20,7 @@ export { PLUGIN_HOOKS, type PluginHook } from './config.js'
|
|
|
20
20
|
// the second, so the cgroup walk has one implementation.
|
|
21
21
|
export { machineMemoryBytes, machineParallelism } from './util/index.js'
|
|
22
22
|
// How core reads a task glob — its spelling, and which characters are
|
|
23
|
-
// wildcards (`[` `]`
|
|
23
|
+
// wildcards (`[` `]` and an escape are) — for a plugin that turns declared
|
|
24
24
|
// outputs into something else (`@vzn/vx-reapi`'s literal `output_paths`).
|
|
25
25
|
export { isLiteralPattern, normalizeGlob, restorableOutput } from './util/index.js'
|
|
26
26
|
|
|
@@ -18,7 +18,6 @@ import {
|
|
|
18
18
|
isUserError,
|
|
19
19
|
machineMemoryBytes,
|
|
20
20
|
machineParallelism,
|
|
21
|
-
parseSize,
|
|
22
21
|
treeBytes,
|
|
23
22
|
} from '../util/index.js'
|
|
24
23
|
import { vxPath } from '../handoff.js'
|
|
@@ -99,8 +98,6 @@ export interface InfoFacts {
|
|
|
99
98
|
string,
|
|
100
99
|
{ value: string | number | boolean | null; source: 'env' | 'workspace' | 'default' }
|
|
101
100
|
>
|
|
102
|
-
/** `cache.retention`, with `maxSize` in bytes; null when none is set. */
|
|
103
|
-
cacheRetention: { olderThan?: string; maxSize?: string; maxBytes?: number } | null
|
|
104
101
|
orphans: { artifacts: number; bytes: number }
|
|
105
102
|
runs24h: number
|
|
106
103
|
hits24h: number
|
|
@@ -261,7 +258,6 @@ async function collectWorkspaceInfo(
|
|
|
261
258
|
cacheEntries: stats.entryCount,
|
|
262
259
|
cacheBytes: stats.totalBytes,
|
|
263
260
|
settings: settingsFact(workspaceConfig),
|
|
264
|
-
cacheRetention: retentionFact(workspaceConfig),
|
|
265
261
|
// A row-less artifact is indexed again only when its task asks for its
|
|
266
262
|
// key; the rest only `vx cache clean` reclaims (after an upgrade's
|
|
267
263
|
// schema reset, most often).
|
|
@@ -395,16 +391,3 @@ function settingsFact(config: WorkspaceConfig | null): InfoFacts['settings'] {
|
|
|
395
391
|
}
|
|
396
392
|
return out
|
|
397
393
|
}
|
|
398
|
-
|
|
399
|
-
function retentionFact(config: WorkspaceConfig | null): InfoFacts['cacheRetention'] {
|
|
400
|
-
const raw = (path: string): string | undefined =>
|
|
401
|
-
process.env[envName(path)] || (atPath(config, path) as string | undefined)
|
|
402
|
-
const olderThan = raw('cache.retention.olderThan')
|
|
403
|
-
const maxSize = raw('cache.retention.maxSize')
|
|
404
|
-
if (olderThan === undefined && maxSize === undefined) return null
|
|
405
|
-
return {
|
|
406
|
-
...(olderThan !== undefined ? { olderThan } : {}),
|
|
407
|
-
// Validated before the run (env) or at load (workspace), so it parses.
|
|
408
|
-
...(maxSize !== undefined ? { maxSize, maxBytes: parseSize(maxSize)! } : {}),
|
|
409
|
-
}
|
|
410
|
-
}
|
|
@@ -335,7 +335,7 @@ async function restoreProbed(
|
|
|
335
335
|
|
|
336
336
|
/**
|
|
337
337
|
* A hit restored, or null when its artifact was removed after the probe (a
|
|
338
|
-
*
|
|
338
|
+
* `vx cache clean`, here or for a workspace sharing the cache directory): a
|
|
339
339
|
* miss. The restore has wiped the declared outputs by then; the run path
|
|
340
340
|
* cleans them again and the task writes them.
|
|
341
341
|
*/
|
|
@@ -130,6 +130,12 @@ export interface RunOptions {
|
|
|
130
130
|
* Locally-placed tasks always write in place and ignore this.
|
|
131
131
|
*/
|
|
132
132
|
download?: 'all' | 'toplevel' | 'none'
|
|
133
|
+
/** Remote downloads at once (`cache.concurrency.download`). */
|
|
134
|
+
downloadConcurrency?: number
|
|
135
|
+
/** Uploads at once, per remote (`cache.concurrency.upload`). */
|
|
136
|
+
uploadConcurrency?: number
|
|
137
|
+
/** Local hits restoring at once (`cache.concurrency.restore`); twice `concurrency` unset. */
|
|
138
|
+
restoreConcurrency?: number
|
|
133
139
|
/**
|
|
134
140
|
* Run intent, derived by the CLI from selection flags: 'broad' iff
|
|
135
141
|
* `--all` / `--filter` / `--affected` was passed, else 'focused'.
|
|
@@ -558,6 +558,8 @@ export interface CacheHost extends CacheContext {
|
|
|
558
558
|
readonly remotes?: ResolvedRemote[]
|
|
559
559
|
/** The run's branch scope, which picks each remote key. */
|
|
560
560
|
readonly branch?: BranchScope
|
|
561
|
+
/** Uploads at once, per remote (`cache.concurrency.upload`). */
|
|
562
|
+
readonly uploadConcurrency?: number
|
|
561
563
|
}
|
|
562
564
|
|
|
563
565
|
/**
|
|
@@ -572,7 +574,7 @@ export async function resolveCache(
|
|
|
572
574
|
plugins: readonly VxPlugin[],
|
|
573
575
|
host: CacheHost,
|
|
574
576
|
): Promise<CacheLayer> {
|
|
575
|
-
const { localCache, policy, remotes, branch, ...ctx } = host
|
|
577
|
+
const { localCache, policy, remotes, branch, uploadConcurrency, ...ctx } = host
|
|
576
578
|
const resolved: ResolvedRemote[] = []
|
|
577
579
|
for (const plugin of plugins) {
|
|
578
580
|
if (plugin.cache === undefined) continue
|
|
@@ -589,7 +591,9 @@ export async function resolveCache(
|
|
|
589
591
|
resolved.push(entry)
|
|
590
592
|
remotes?.push(entry)
|
|
591
593
|
}
|
|
592
|
-
return resolved.length === 0
|
|
594
|
+
return resolved.length === 0
|
|
595
|
+
? localCache
|
|
596
|
+
: new LayeredCache(localCache, resolved, branch, uploadConcurrency)
|
|
593
597
|
}
|
|
594
598
|
|
|
595
599
|
/** The plugin each resolved executor came from, so a hook of it that throws is named. */
|
|
@@ -347,11 +347,14 @@ export interface WorkspaceHookContext {
|
|
|
347
347
|
* cache scope (`'vznjs/vx:feat-x'`; `VX_CACHE_SCOPE` beats it, and it
|
|
348
348
|
* beats the one git names), and `base`, the commit the branch is compared
|
|
349
349
|
* with (a pull request's base), for a bare `--affected` and for which
|
|
350
|
-
* tasks read the scope.
|
|
350
|
+
* tasks read the scope. `trusted`: the run has no scope, whatever git
|
|
351
|
+
* names, and reads and writes only the default keys (a merge queue's run,
|
|
352
|
+
* owner 2026-10-18). A later plugin's field wins.
|
|
351
353
|
*/
|
|
352
354
|
export interface BranchFacts {
|
|
353
355
|
scope?: string
|
|
354
356
|
base?: string
|
|
357
|
+
trusted?: boolean
|
|
355
358
|
}
|
|
356
359
|
|
|
357
360
|
export interface ConfigContext extends WorkspaceHookContext {
|
|
@@ -33,6 +33,8 @@ import {
|
|
|
33
33
|
startGitEnumeration,
|
|
34
34
|
lazyGitEnumeration,
|
|
35
35
|
measureFsClockLag,
|
|
36
|
+
DEFAULT_STORE,
|
|
37
|
+
type StoreOptions,
|
|
36
38
|
} from '../cache/index.js'
|
|
37
39
|
import {
|
|
38
40
|
buildPackageGraph,
|
|
@@ -89,7 +91,7 @@ import { currentBranch, resolveCacheScope } from './run-context.js'
|
|
|
89
91
|
import { createHashCache, type HashCache } from './task-hash.js'
|
|
90
92
|
import { type Logger, warnOf } from '../report/index.js'
|
|
91
93
|
import type { RunOptions } from './options.js'
|
|
92
|
-
import { runRules } from '../settings.js'
|
|
94
|
+
import { resolveSetting, runRules } from '../settings.js'
|
|
93
95
|
|
|
94
96
|
export interface PreparedRun {
|
|
95
97
|
workspaceRoot: string
|
|
@@ -263,7 +265,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
263
265
|
options.cacheScope ??
|
|
264
266
|
resolveCacheScope(
|
|
265
267
|
workspaceRoot,
|
|
266
|
-
branchFacts.scope,
|
|
268
|
+
branchFacts.trusted === true ? null : branchFacts.scope,
|
|
267
269
|
plugins.some((p) => p.cache !== undefined),
|
|
268
270
|
baseBranch,
|
|
269
271
|
)
|
|
@@ -376,6 +378,14 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
376
378
|
options.artifactCeiling,
|
|
377
379
|
'open',
|
|
378
380
|
await storeRoot,
|
|
381
|
+
{
|
|
382
|
+
embeddings: (resolveSetting('cache.store.embeddings', workspaceConfig).value ??
|
|
383
|
+
DEFAULT_STORE.embeddings) as number,
|
|
384
|
+
decompress: (resolveSetting('cache.store.decompress', workspaceConfig).value ??
|
|
385
|
+
DEFAULT_STORE.decompress) as boolean,
|
|
386
|
+
verify: (resolveSetting('cache.store.verify', workspaceConfig).value ??
|
|
387
|
+
DEFAULT_STORE.verify) as StoreOptions['verify'],
|
|
388
|
+
},
|
|
379
389
|
)
|
|
380
390
|
const unwritable = localCache.readOnlyNotice()
|
|
381
391
|
if (unwritable !== null) warnOf(log)(`[vx] ${unwritable}`)
|
|
@@ -522,7 +532,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
522
532
|
}
|
|
523
533
|
try {
|
|
524
534
|
cache = options.remoteCache
|
|
525
|
-
? new LayeredCache(localCache, remotes, branchScope)
|
|
535
|
+
? new LayeredCache(localCache, remotes, branchScope, options.uploadConcurrency)
|
|
526
536
|
: await resolveCache(plugins, {
|
|
527
537
|
workspaceRoot,
|
|
528
538
|
cacheDir,
|
|
@@ -532,6 +542,9 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
532
542
|
policy,
|
|
533
543
|
remotes,
|
|
534
544
|
...(branchScope !== undefined ? { branch: branchScope } : {}),
|
|
545
|
+
...(options.uploadConcurrency !== undefined
|
|
546
|
+
? { uploadConcurrency: options.uploadConcurrency }
|
|
547
|
+
: {}),
|
|
535
548
|
})
|
|
536
549
|
} catch (err) {
|
|
537
550
|
// A cache plugin that throws or returns something off-contract is
|
|
@@ -71,7 +71,14 @@ export async function loadWorkspacePlugins(
|
|
|
71
71
|
const setBranch = (facts: BranchFacts): void => {
|
|
72
72
|
if (facts.scope !== undefined && !isCacheScope(facts.scope))
|
|
73
73
|
throw new UserError(`setBranch: ${cacheScopeError('scope')}`)
|
|
74
|
-
if (facts.
|
|
74
|
+
if (facts.trusted === true) {
|
|
75
|
+
delete branch.scope
|
|
76
|
+
branch.trusted = true
|
|
77
|
+
}
|
|
78
|
+
if (facts.scope !== undefined) {
|
|
79
|
+
branch.scope = facts.scope
|
|
80
|
+
delete branch.trusted
|
|
81
|
+
}
|
|
75
82
|
if (facts.base !== undefined) branch.base = facts.base
|
|
76
83
|
}
|
|
77
84
|
await applyConfigHooks(plugins, workspaceConfig, { workspaceRoot, warn, setBranch }, (plugin) =>
|
|
@@ -37,7 +37,7 @@ export interface PrefetchArgs {
|
|
|
37
37
|
nestedDirsByProject: Map<string, string[]>
|
|
38
38
|
gitFilesCache: GitFilesCache
|
|
39
39
|
hashCache: HashCache
|
|
40
|
-
/** Cap on concurrent in-flight prefetches
|
|
40
|
+
/** Cap on concurrent in-flight prefetches (`cache.concurrency.download`). */
|
|
41
41
|
concurrency: number
|
|
42
42
|
/** When false (remote reads disabled), no prefetch fires. */
|
|
43
43
|
remoteRead: boolean
|
|
@@ -75,10 +75,14 @@ export function startRemotePrefetch(args: PrefetchArgs): Promise<void> {
|
|
|
75
75
|
// a still-in-flight prefetch can never ingest into a closed DB. Errors
|
|
76
76
|
// degrade inside LayeredCache.prefetch; the catch keeps the handle
|
|
77
77
|
// from ever rejecting.
|
|
78
|
-
|
|
78
|
+
const started = Promise.withResolvers<void>()
|
|
79
|
+
args.cache.probeFirst?.(started.promise)
|
|
80
|
+
return runPrefetch(args, started.resolve)
|
|
81
|
+
.catch(() => {})
|
|
82
|
+
.finally(started.resolve)
|
|
79
83
|
}
|
|
80
84
|
|
|
81
|
-
async function runPrefetch(args: PrefetchArgs): Promise<void> {
|
|
85
|
+
async function runPrefetch(args: PrefetchArgs, probing: () => void): Promise<void> {
|
|
82
86
|
const stableKeys = await deriveStableKeys(args)
|
|
83
87
|
if (stableKeys.length === 0) return
|
|
84
88
|
|
|
@@ -94,7 +98,10 @@ async function runPrefetch(args: PrefetchArgs): Promise<void> {
|
|
|
94
98
|
let toPrefetch = held === null ? stableKeys : stableKeys.filter((k) => !held.has(k.hash))
|
|
95
99
|
if (toPrefetch.length === 0) return
|
|
96
100
|
const uniqueHashes = [...new Set(toPrefetch.map((k) => k.hash))]
|
|
97
|
-
const
|
|
101
|
+
const asked = args.cache.remoteHasMany?.(uniqueHashes)
|
|
102
|
+
// Every lookup that waited on the probe now finds its hash in it.
|
|
103
|
+
probing()
|
|
104
|
+
const present = (await asked) ?? null
|
|
98
105
|
if (present !== null) {
|
|
99
106
|
args.cache.markRemoteAbsent?.(uniqueHashes.filter((h) => !present.has(h)))
|
|
100
107
|
// The pre-mark above already answers a pull for an absent hash from
|
|
@@ -553,10 +553,13 @@ export function localBranchScope(
|
|
|
553
553
|
return branchCacheScope(repo, name)
|
|
554
554
|
}
|
|
555
555
|
|
|
556
|
-
/**
|
|
556
|
+
/**
|
|
557
|
+
* The repository's path on its host (`owner/repo`; a GitLab subgroup's
|
|
558
|
+
* `group/sub/repo` whole, as `gitlab()` names it); undefined for a local path.
|
|
559
|
+
*/
|
|
557
560
|
function repoOfUrl(url: string): string | undefined {
|
|
558
561
|
const parts = normalizeRemoteUrl(url).split('/')
|
|
559
|
-
return parts.length < 3 ? undefined : parts.slice(
|
|
562
|
+
return parts.length < 3 ? undefined : parts.slice(1).join('/')
|
|
560
563
|
}
|
|
561
564
|
|
|
562
565
|
/** `.git/config`'s sections, each a key → value map (last wins); empty when it cannot be read. */
|
|
@@ -586,12 +589,13 @@ function gitConfigSections(workspaceRoot: string): Map<string, Map<string, strin
|
|
|
586
589
|
|
|
587
590
|
/**
|
|
588
591
|
* The run's cache scope (owner, 2026-10-17): `VX_CACHE_SCOPE` (empty: none),
|
|
589
|
-
* else a plugin's (`setBranch`, `github()` on Actions
|
|
590
|
-
*
|
|
592
|
+
* else a plugin's (`setBranch`, `github()` on Actions; null: the plugin
|
|
593
|
+
* named the run trusted), else, with a cache plugin to share it, the
|
|
594
|
+
* branch's from git. Undefined: the default keys.
|
|
591
595
|
*/
|
|
592
596
|
export function resolveCacheScope(
|
|
593
597
|
workspaceRoot: string,
|
|
594
|
-
pluginScope: string | undefined,
|
|
598
|
+
pluginScope: string | null | undefined,
|
|
595
599
|
hasRemote: boolean,
|
|
596
600
|
baseBranch: string | undefined,
|
|
597
601
|
env: NodeJS.ProcessEnv = process.env,
|
|
@@ -602,6 +606,7 @@ export function resolveCacheScope(
|
|
|
602
606
|
if (!isCacheScope(named)) throw cacheScopeEnvError()
|
|
603
607
|
return named
|
|
604
608
|
}
|
|
609
|
+
if (pluginScope === null) return undefined
|
|
605
610
|
return pluginScope ?? (hasRemote ? localBranchScope(workspaceRoot, baseBranch, env) : undefined)
|
|
606
611
|
}
|
|
607
612
|
|
|
@@ -14,6 +14,9 @@ const FROM_OPTIONS = {
|
|
|
14
14
|
retries: 'run.retries',
|
|
15
15
|
continueMode: 'run.continue',
|
|
16
16
|
download: 'cache.download',
|
|
17
|
+
downloadConcurrency: 'cache.concurrency.download',
|
|
18
|
+
uploadConcurrency: 'cache.concurrency.upload',
|
|
19
|
+
restoreConcurrency: 'cache.concurrency.restore',
|
|
17
20
|
} as const satisfies Partial<Record<keyof RunOptions, string>>
|
|
18
21
|
|
|
19
22
|
/**
|
package/src/orchestrator/run.ts
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
type CachePolicy,
|
|
12
12
|
cachesNothing,
|
|
13
13
|
formatCachePolicy,
|
|
14
|
+
networkConcurrency,
|
|
14
15
|
stopRuntimeProbes,
|
|
15
16
|
sweepStaging,
|
|
16
17
|
} from '../cache/index.js'
|
|
@@ -259,6 +260,10 @@ function refuseRunOptions(options: RunOptions): void {
|
|
|
259
260
|
const { concurrency, retries, timeout } = options
|
|
260
261
|
if (concurrency !== undefined && !(Number.isInteger(concurrency) && concurrency > 0))
|
|
261
262
|
refuse('concurrency', concurrency, 'a positive integer')
|
|
263
|
+
for (const field of ['downloadConcurrency', 'uploadConcurrency', 'restoreConcurrency'] as const) {
|
|
264
|
+
const v = options[field]
|
|
265
|
+
if (v !== undefined && !(Number.isInteger(v) && v > 0)) refuse(field, v, 'a positive integer')
|
|
266
|
+
}
|
|
262
267
|
if (retries !== undefined && !(Number.isInteger(retries) && retries >= 0))
|
|
263
268
|
refuse('retries', retries, 'a non-negative integer')
|
|
264
269
|
if (
|
|
@@ -618,7 +623,7 @@ async function runOnBus(
|
|
|
618
623
|
// `vx cache clean` can evict entries the run just hit. Once-only
|
|
619
624
|
// because the normal path closes before the persistent-task wait —
|
|
620
625
|
// holding the handle open for a dev server's whole lifetime would be
|
|
621
|
-
// worse
|
|
626
|
+
// worse.
|
|
622
627
|
let cacheClosed = false
|
|
623
628
|
const closeCache = async (): Promise<void> => {
|
|
624
629
|
if (cacheClosed) return
|
|
@@ -871,7 +876,7 @@ async function runOnBus(
|
|
|
871
876
|
nestedDirsByProject,
|
|
872
877
|
gitFilesCache,
|
|
873
878
|
hashCache,
|
|
874
|
-
concurrency,
|
|
879
|
+
concurrency: options.downloadConcurrency ?? networkConcurrency(),
|
|
875
880
|
remoteRead: policy.remoteRead,
|
|
876
881
|
remoteWrite: policy.remoteWrite,
|
|
877
882
|
}),
|
|
@@ -1088,6 +1093,7 @@ async function runOnBus(
|
|
|
1088
1093
|
const outcomes = await runGraph({
|
|
1089
1094
|
nodes,
|
|
1090
1095
|
concurrency,
|
|
1096
|
+
restoreConcurrency: options.restoreConcurrency ?? 2 * concurrency,
|
|
1091
1097
|
settledOf: (o) => deferredSaves.get(o.node.id),
|
|
1092
1098
|
...(hasPooledExecutor(executors) ? { poolOf: poolOfPlacement(placements) } : {}),
|
|
1093
1099
|
...(admit !== undefined ? { admit } : {}),
|
|
@@ -1349,9 +1355,6 @@ async function runOnBus(
|
|
|
1349
1355
|
// The miss path's output-directory snapshots first (miss-save.ts).
|
|
1350
1356
|
await step('vx:snapshots', () => recordOutputDirSnapshots(cache, outputDirSnapshots))
|
|
1351
1357
|
mark('output dir snapshots')
|
|
1352
|
-
// Not on a stopped run: one stopped while it waited on another run's
|
|
1353
|
-
// lock never held it, and its prune evicted under that run (item 858).
|
|
1354
|
-
if (!stop.signal.aborted) await step('vx:retention', () => applyCacheRetention(prepared))
|
|
1355
1358
|
// A plugin hears the run until its teardown and nothing after: released
|
|
1356
1359
|
// only in the finally, its handlers heard a kept server through the
|
|
1357
1360
|
// whole keep-alive wait below (C-66). Idempotent; the finally's stay.
|
|
@@ -1488,27 +1491,6 @@ async function runOnBus(
|
|
|
1488
1491
|
}
|
|
1489
1492
|
}
|
|
1490
1493
|
|
|
1491
|
-
/**
|
|
1492
|
-
* `cache.retention`, after every save and upload of this run
|
|
1493
|
-
* has landed (so nothing this run wrote is mid-flight) and before the cache
|
|
1494
|
-
* closes. Housekeeping, not the run's work: silent, and a failure is never
|
|
1495
|
-
* a failed run. Declared nowhere → one property read.
|
|
1496
|
-
*/
|
|
1497
|
-
async function applyCacheRetention(prepared: PreparedRun): Promise<void> {
|
|
1498
|
-
const ws = prepared.workspaceConfig
|
|
1499
|
-
const maxAgeMs = resolveSetting('cache.retention.olderThan', ws).value as number | undefined
|
|
1500
|
-
const maxBytes = resolveSetting('cache.retention.maxSize', ws).value as number | undefined
|
|
1501
|
-
if (maxAgeMs === undefined && maxBytes === undefined) return
|
|
1502
|
-
try {
|
|
1503
|
-
await prepared.localCache.evictIfDue({
|
|
1504
|
-
...(maxAgeMs !== undefined ? { maxAgeMs } : {}),
|
|
1505
|
-
...(maxBytes !== undefined ? { maxBytes } : {}),
|
|
1506
|
-
})
|
|
1507
|
-
} catch {
|
|
1508
|
-
// Housekeeping says nothing (owner): the next due run tries again.
|
|
1509
|
-
}
|
|
1510
|
-
}
|
|
1511
|
-
|
|
1512
1494
|
/** Why an `--affected` run has nothing to run, said alike by the run and its `--dry` plan. */
|
|
1513
1495
|
function noneAffected(prepared: PreparedRun, options: RunOptions): string {
|
|
1514
1496
|
// A changed project declares the task, so `--affected`'s task walk is
|
package/src/report/logger.ts
CHANGED
|
@@ -382,7 +382,6 @@ export function defaultLogger(
|
|
|
382
382
|
let saved = 0
|
|
383
383
|
let notSaved = 0
|
|
384
384
|
let skippedCount = 0
|
|
385
|
-
let savedMs = 0
|
|
386
385
|
// Each finished task's cache bucket by its key: its remote traffic moves it.
|
|
387
386
|
const buckets = new Map<string, CacheBucket>()
|
|
388
387
|
// The graph is done and vx's own closing steps run: the region stays,
|
|
@@ -530,7 +529,6 @@ export function defaultLogger(
|
|
|
530
529
|
notSaved,
|
|
531
530
|
miss: succeeded + failed - noCache,
|
|
532
531
|
noCache,
|
|
533
|
-
savedMs,
|
|
534
532
|
left: total - done - stopped - notRun,
|
|
535
533
|
...(etaMs !== undefined && !tail && !stopping ? { etaMs } : {}),
|
|
536
534
|
buckets,
|
|
@@ -1028,12 +1026,10 @@ export function defaultLogger(
|
|
|
1028
1026
|
case 'cache-hit':
|
|
1029
1027
|
if (outcome.restored === false) upToDate++
|
|
1030
1028
|
else restoredLocal++
|
|
1031
|
-
savedMs += outcome.storedDurationMs ?? 0
|
|
1032
1029
|
break
|
|
1033
1030
|
case 'cache-hit-remote':
|
|
1034
1031
|
if (outcome.restored === false) upToDate++
|
|
1035
1032
|
else restoredRemote++
|
|
1036
|
-
savedMs += outcome.storedDurationMs ?? 0
|
|
1037
1033
|
break
|
|
1038
1034
|
case 'skipped':
|
|
1039
1035
|
skippedCount++
|
package/src/report/summary.ts
CHANGED
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
|
|
6
6
|
import type { TaskOutcome } from '../graph/index.js'
|
|
7
7
|
import { tallyOutcomes } from './tally.js'
|
|
8
|
-
import { isCacheHit } from '../graph/index.js'
|
|
9
8
|
import { outcomeLabel, projectOutcome, ranNoCache } from './events.js'
|
|
10
9
|
import { isGroupTask } from '../graph/index.js'
|
|
11
10
|
import {
|
|
@@ -106,8 +105,6 @@ export interface SummaryStats {
|
|
|
106
105
|
held?: { count: number; sumMs: number }
|
|
107
106
|
/** Live section only: the forecast of what is left, from history. */
|
|
108
107
|
etaMs?: number
|
|
109
|
-
/** The hits' stored exec times summed: the work the cache skipped. */
|
|
110
|
-
savedMs?: number
|
|
111
108
|
/** A run of groups only: their names, so `0 tasks` says why. */
|
|
112
109
|
emptyGroups?: readonly string[]
|
|
113
110
|
/** Each finished task's cache bucket by its key and its id (its run bytes), so its remote traffic can move it. */
|
|
@@ -392,10 +389,6 @@ export function formatSummarySection(
|
|
|
392
389
|
}
|
|
393
390
|
}
|
|
394
391
|
const timeParts = [total]
|
|
395
|
-
// What the cache paid back, beside what the run cost: the stored exec
|
|
396
|
-
// time of every hit, never the restore this run spent on it.
|
|
397
|
-
if ((stats.savedMs ?? 0) >= 1)
|
|
398
|
-
timeParts.push(paint(SUCCESS, `${formatElapsed(stats.savedMs!)} saved`, colors))
|
|
399
392
|
// The live run's forecast leads the spread: the one number on the row
|
|
400
393
|
// that is about the future.
|
|
401
394
|
if (stats.etaMs !== undefined)
|
|
@@ -492,9 +485,6 @@ export function formatRunSummary(
|
|
|
492
485
|
const durations = outcomes
|
|
493
486
|
.filter((o) => (o.status === 'success' || o.status === 'failed') && !isGroupTask(o.node))
|
|
494
487
|
.map((o) => o.durationMs)
|
|
495
|
-
let savedMs = 0
|
|
496
|
-
for (const o of outcomes)
|
|
497
|
-
if (isCacheHit(o.status) && !isGroupTask(o.node)) savedMs += o.storedDurationMs ?? 0
|
|
498
488
|
const heldOutcomes = outcomes.filter((o) => o.admissionHeldMs !== undefined)
|
|
499
489
|
const notRun = outcomes.filter((o) => neverStarted(o) && !isGroupTask(o.node)).length
|
|
500
490
|
const emptyGroups = outcomes.every((o) => isGroupTask(o.node))
|
|
@@ -515,7 +505,6 @@ export function formatRunSummary(
|
|
|
515
505
|
notSaved: outcomes.filter((o) => !isGroupTask(o.node) && ranUnsaved(o)).length,
|
|
516
506
|
miss: t.total - t.skipped - (t.cachedLocal + t.cachedRemote) - noCache,
|
|
517
507
|
noCache,
|
|
518
|
-
savedMs,
|
|
519
508
|
spread:
|
|
520
509
|
durations.length > 0
|
|
521
510
|
? {
|