@vzn/vx 0.0.643 → 0.0.644
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/schemas/info.json +0 -11
- package/skills/vx/SKILL.md +1 -1
- package/src/cache/archive.ts +153 -2
- package/src/cache/cache.ts +416 -176
- package/src/cache/file-hashes.ts +2 -14
- package/src/cache/fold-memo.ts +7 -3
- package/src/cache/index.ts +3 -0
- package/src/cache/key-fold.ts +4 -1
- package/src/cache/layer.ts +9 -4
- package/src/cache/layered-cache.ts +55 -3
- package/src/cache/plugin-cache.ts +33 -32
- package/src/cache/remote-lane.ts +12 -7
- package/src/cache/tar-stream.ts +12 -7
- package/src/cli/format.ts +1 -2
- package/src/cli/help.ts +6 -0
- package/src/cli/info.ts +2 -19
- package/src/cli/plugin-commands.ts +1 -1
- package/src/cli/run.ts +3 -0
- package/src/cli/upgrade.ts +38 -8
- package/src/config.ts +15 -10
- package/src/exec/sandbox-paths.ts +2 -2
- package/src/graph/scheduler.ts +3 -1
- package/src/graph/task-graph.ts +15 -11
- package/src/index.ts +1 -1
- package/src/orchestrator/doctor.ts +0 -17
- package/src/orchestrator/execute-task.ts +1 -1
- package/src/orchestrator/options.ts +6 -0
- package/src/orchestrator/plugin-host.ts +6 -2
- package/src/orchestrator/plugin.ts +4 -1
- package/src/orchestrator/prepare.ts +16 -3
- package/src/orchestrator/projects.ts +8 -1
- package/src/orchestrator/remote-prefetch.ts +11 -4
- package/src/orchestrator/run-context.ts +10 -5
- package/src/orchestrator/run-settings.ts +3 -0
- package/src/orchestrator/run.ts +8 -26
- package/src/report/logger.ts +0 -4
- package/src/report/summary.ts +0 -11
- package/src/settings.ts +43 -14
- package/src/util/index.ts +1 -0
- package/src/util/paths.ts +42 -33
- package/src/util/size.ts +4 -5
- package/src/workspace/config-schema.ts +0 -36
package/src/cache/file-hashes.ts
CHANGED
|
@@ -14,9 +14,6 @@ import { fileIdentity, repoFacts } from './git-inputs.js'
|
|
|
14
14
|
import { FILE_HASH_RACY_MS, isIndexFull, racyWindowMs } from './layer.js'
|
|
15
15
|
import { isPermissionError, UserError } from '../util/index.js'
|
|
16
16
|
|
|
17
|
-
const FILE_HASHES_SWEPT_AT = 'file_hashes_swept_at'
|
|
18
|
-
const DAY_MS = 24 * 60 * 60 * 1000
|
|
19
|
-
|
|
20
17
|
export class FileHashStore {
|
|
21
18
|
private readonly selectFileHash: ReturnType<Database['prepare']>
|
|
22
19
|
private readonly upsertFileHash: ReturnType<Database['prepare']>
|
|
@@ -62,22 +59,13 @@ export class FileHashStore {
|
|
|
62
59
|
* path) keeps its row forever otherwise, and `cache.db` grows with every
|
|
63
60
|
* path ever hashed. A hit does not refresh `seen_at` (a write per file
|
|
64
61
|
* per run), so a file unchanged for the window loses its row too and is
|
|
65
|
-
* read once more; a dropped row is only a memo miss.
|
|
66
|
-
* 8 ms at 100,000 rows, so it runs once a day, on its own clock in
|
|
67
|
-
* `schema_meta` (item 1082).
|
|
62
|
+
* read once more; a dropped row is only a memo miss.
|
|
68
63
|
*/
|
|
69
|
-
pruneOlderThan(cutoff: number
|
|
70
|
-
const last = this.db
|
|
71
|
-
.prepare('SELECT value FROM schema_meta WHERE key = ?')
|
|
72
|
-
.get(FILE_HASHES_SWEPT_AT) as { value: string } | null
|
|
73
|
-
if (last !== null && now - Number(last.value) < DAY_MS) return
|
|
64
|
+
pruneOlderThan(cutoff: number): void {
|
|
74
65
|
this.db.transaction(() => {
|
|
75
66
|
this.db.prepare('DELETE FROM file_hashes WHERE seen_at < ?').run(cutoff)
|
|
76
67
|
this.db.prepare('DELETE FROM blob_sizes WHERE seen_at < ?').run(cutoff)
|
|
77
68
|
this.db.prepare('DELETE FROM blob_verdicts WHERE seen_at < ?').run(cutoff)
|
|
78
|
-
this.db
|
|
79
|
-
.prepare('INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)')
|
|
80
|
-
.run(FILE_HASHES_SWEPT_AT, String(now))
|
|
81
69
|
})()
|
|
82
70
|
}
|
|
83
71
|
|
package/src/cache/fold-memo.ts
CHANGED
|
@@ -43,16 +43,20 @@ export class FoldMemo {
|
|
|
43
43
|
if (this.write) this.dirty.set(taskId, { probe, key })
|
|
44
44
|
}
|
|
45
45
|
|
|
46
|
-
/** Write the folds this run made
|
|
47
|
-
flush(
|
|
46
|
+
/** Write the folds this run made. */
|
|
47
|
+
flush(now: number = Date.now()): void {
|
|
48
48
|
if (this.dirty.size === 0) return
|
|
49
49
|
const upsert = this.db.prepare(
|
|
50
50
|
'INSERT OR REPLACE INTO fold_memo(task_id, probe, key, seen_at) VALUES (?, ?, ?, ?)',
|
|
51
51
|
)
|
|
52
52
|
this.db.transaction(() => {
|
|
53
53
|
for (const [taskId, r] of this.dirty) upsert.run(taskId, r.probe, r.key, now)
|
|
54
|
-
this.db.prepare('DELETE FROM fold_memo WHERE seen_at < ?').run(cutoff)
|
|
55
54
|
})()
|
|
56
55
|
this.dirty.clear()
|
|
57
56
|
}
|
|
57
|
+
|
|
58
|
+
/** Rows no run has rewritten since `cutoff`. */
|
|
59
|
+
prune(cutoff: number): void {
|
|
60
|
+
this.db.prepare('DELETE FROM fold_memo WHERE seen_at < ?').run(cutoff)
|
|
61
|
+
}
|
|
58
62
|
}
|
package/src/cache/index.ts
CHANGED
|
@@ -8,6 +8,8 @@ export {
|
|
|
8
8
|
applyCacheFlags,
|
|
9
9
|
ArtifactVanishedError,
|
|
10
10
|
Cache,
|
|
11
|
+
DEFAULT_STORE,
|
|
12
|
+
type StoreOptions,
|
|
11
13
|
type CacheFlags,
|
|
12
14
|
type CacheEntry,
|
|
13
15
|
type CacheKeyInput,
|
|
@@ -85,6 +87,7 @@ export {
|
|
|
85
87
|
type RemoteKey,
|
|
86
88
|
type RemoteOpenContext,
|
|
87
89
|
} from './layered-cache.js'
|
|
90
|
+
export { networkConcurrency } from './remote-lane.js'
|
|
88
91
|
export type { ResolvedRemote } from './remote-lane.js'
|
|
89
92
|
export { BranchScope } from './scoped-cache.js'
|
|
90
93
|
export {
|
package/src/cache/key-fold.ts
CHANGED
|
@@ -117,7 +117,10 @@ import type { CacheKeyInput } from './layer.js'
|
|
|
117
117
|
// v46: vx's names are reserved only at the project root; a v45 artifact
|
|
118
118
|
// left out a build's own `dist/.git/**`, `dist/.vx/**` and nested
|
|
119
119
|
// `vx.lock.json` (Growth's parity, 2026-10-17).
|
|
120
|
-
|
|
120
|
+
// v47: the local store keeps plain files (`cache.store`).
|
|
121
|
+
// v48: a task glob is standard again (M, 2026-10-18): `app/[id]/page.js` is
|
|
122
|
+
// a class, and a v47 entry under its unchanged key holds the literal route.
|
|
123
|
+
export const CACHE_VERSION = 'vx-cache-v48'
|
|
121
124
|
|
|
122
125
|
/**
|
|
123
126
|
* The digest a key folds for an input gone between its enumeration and its
|
package/src/cache/layer.ts
CHANGED
|
@@ -493,8 +493,8 @@ export class CorruptArtifactError extends Error {
|
|
|
493
493
|
|
|
494
494
|
/**
|
|
495
495
|
* `restoreOutputs` found no artifact where the probe found one: it was
|
|
496
|
-
* removed in between — a `vx cache clean` in another shell, or
|
|
497
|
-
*
|
|
496
|
+
* removed in between — a `vx cache clean` in another shell, or one for
|
|
497
|
+
* another workspace sharing the cache directory. The
|
|
498
498
|
* bytes are not wrong, they are gone, so the entry is a MISS and the task
|
|
499
499
|
* runs (execute-task.ts); nothing about the cache needs throwing away.
|
|
500
500
|
* Also thrown, with its `reason`, for an artifact whose bytes ARE wrong
|
|
@@ -510,8 +510,7 @@ export class ArtifactVanishedError extends Error {
|
|
|
510
510
|
cause?: unknown,
|
|
511
511
|
) {
|
|
512
512
|
super(
|
|
513
|
-
reason ??
|
|
514
|
-
`its cache artifact ${hash} vanished before the restore (a prune or another run's retention)`,
|
|
513
|
+
reason ?? `its cache artifact ${hash} vanished before the restore (a vx cache clean)`,
|
|
515
514
|
cause === undefined ? undefined : { cause },
|
|
516
515
|
)
|
|
517
516
|
this.name = 'ArtifactVanishedError'
|
|
@@ -576,6 +575,12 @@ export interface CacheLayer {
|
|
|
576
575
|
* `hasRemote`.
|
|
577
576
|
*/
|
|
578
577
|
probeBeforeUpload?(hashes: Promise<readonly string[]>): void
|
|
578
|
+
/**
|
|
579
|
+
* The run's batch probe starts when `started` settles: until then a pull
|
|
580
|
+
* waits, so a lookup that lands first does not GET a key the probe is
|
|
581
|
+
* about to call absent. Only meaningful with `hasRemote`.
|
|
582
|
+
*/
|
|
583
|
+
probeFirst?(started: Promise<void>): void
|
|
579
584
|
/** `taskId`'s key is `hash`: a scoped run picks each remote key by its task (`BranchScope`). */
|
|
580
585
|
noteKey?(taskId: string, hash: string): void
|
|
581
586
|
/**
|
|
@@ -175,6 +175,17 @@ export class LayeredCache implements CacheLayer {
|
|
|
175
175
|
*/
|
|
176
176
|
private readonly inflight = new Map<string, Promise<boolean>>()
|
|
177
177
|
|
|
178
|
+
/**
|
|
179
|
+
* Hashes a batch probe is asking about, each resolving whether a pull may
|
|
180
|
+
* still find it (false: no remote holds it). A `get` that lands while the
|
|
181
|
+
* probe is out waits for its answer instead of spending a GET on a key
|
|
182
|
+
* the probe is about to call absent.
|
|
183
|
+
*/
|
|
184
|
+
private readonly probing = new Map<string, Promise<boolean>>()
|
|
185
|
+
|
|
186
|
+
/** Settles once the run's batch probe has started (`probeFirst`); a pull waits on it. */
|
|
187
|
+
private probeStarted: Promise<void> | null = null
|
|
188
|
+
|
|
178
189
|
/**
|
|
179
190
|
* Hashes whose local artifact was materialized FROM a remote this run
|
|
180
191
|
* (by `prefetch` or `get`'s read-through). A later `get` finds them as
|
|
@@ -192,9 +203,11 @@ export class LayeredCache implements CacheLayer {
|
|
|
192
203
|
remotes: readonly ResolvedRemote[],
|
|
193
204
|
/** The run's branch scope: it picks each remote key; the local store never sees it. */
|
|
194
205
|
private readonly branch?: BranchScope,
|
|
206
|
+
uploadConcurrency?: number,
|
|
195
207
|
) {
|
|
196
208
|
this.lanes = remotes.map(
|
|
197
|
-
(r) =>
|
|
209
|
+
(r) =>
|
|
210
|
+
new RemoteLane(r, local.traffic, branch?.remote(r.remote) ?? r.remote, uploadConcurrency),
|
|
198
211
|
)
|
|
199
212
|
}
|
|
200
213
|
|
|
@@ -238,6 +251,27 @@ export class LayeredCache implements CacheLayer {
|
|
|
238
251
|
*/
|
|
239
252
|
async remoteHasMany(hashes: readonly string[]): Promise<Set<string> | null> {
|
|
240
253
|
if (this.readable.length === 0) return null
|
|
254
|
+
const answer = this.probeMany(hashes)
|
|
255
|
+
const waiting = hashes.filter((h) => !this.inflight.has(h) && !this.probing.has(h))
|
|
256
|
+
for (const h of waiting) {
|
|
257
|
+
this.probing.set(
|
|
258
|
+
h,
|
|
259
|
+
answer.then(
|
|
260
|
+
(found) => {
|
|
261
|
+
this.probing.delete(h)
|
|
262
|
+
return found === null || found.has(h)
|
|
263
|
+
},
|
|
264
|
+
() => {
|
|
265
|
+
this.probing.delete(h)
|
|
266
|
+
return true
|
|
267
|
+
},
|
|
268
|
+
),
|
|
269
|
+
)
|
|
270
|
+
}
|
|
271
|
+
return answer
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
private async probeMany(hashes: readonly string[]): Promise<Set<string> | null> {
|
|
241
275
|
let out: Set<string> | null = new Set()
|
|
242
276
|
for (const lane of this.readable) {
|
|
243
277
|
const found = await lane.hasMany(hashes)
|
|
@@ -261,6 +295,17 @@ export class LayeredCache implements CacheLayer {
|
|
|
261
295
|
}
|
|
262
296
|
}
|
|
263
297
|
|
|
298
|
+
probeFirst(started: Promise<void>): void {
|
|
299
|
+
const gate = started.then(
|
|
300
|
+
() => {},
|
|
301
|
+
() => {},
|
|
302
|
+
)
|
|
303
|
+
this.probeStarted = gate
|
|
304
|
+
void gate.then(() => {
|
|
305
|
+
if (this.probeStarted === gate) this.probeStarted = null
|
|
306
|
+
})
|
|
307
|
+
}
|
|
308
|
+
|
|
264
309
|
probeBeforeUpload(hashes: Promise<readonly string[]>): void {
|
|
265
310
|
for (const lane of this.lanes) lane.probeBeforeUpload(hashes)
|
|
266
311
|
}
|
|
@@ -331,6 +376,11 @@ export class LayeredCache implements CacheLayer {
|
|
|
331
376
|
private pullFromRemote(hash: string, ctx?: CacheGetContext): Promise<boolean> {
|
|
332
377
|
const existing = this.inflight.get(hash)
|
|
333
378
|
if (existing) return existing
|
|
379
|
+
const probed = this.probing.get(hash)
|
|
380
|
+
if (probed !== undefined)
|
|
381
|
+
return probed.then((found) => (found ? this.pullFromRemote(hash, ctx) : false))
|
|
382
|
+
if (this.probeStarted !== null)
|
|
383
|
+
return this.probeStarted.then(() => this.pullFromRemote(hash, ctx))
|
|
334
384
|
const p = this.doPullFromRemote(hash, ctx)
|
|
335
385
|
this.inflight.set(hash, p)
|
|
336
386
|
return p
|
|
@@ -457,8 +507,10 @@ export class LayeredCache implements CacheLayer {
|
|
|
457
507
|
return
|
|
458
508
|
}
|
|
459
509
|
}
|
|
460
|
-
const body = (): { body: Blob; release?: () => Promise<void> } =>
|
|
461
|
-
packed !== undefined
|
|
510
|
+
const body = (): Promise<{ body: Blob; release?: () => Promise<void> }> =>
|
|
511
|
+
packed !== undefined
|
|
512
|
+
? Promise.resolve({ body: new Blob([packed]) })
|
|
513
|
+
: this.local.pinArtifact(hash)
|
|
462
514
|
for (const lane of writable) lane.upload(hash, durationMs, body, this.local.uploads)
|
|
463
515
|
}
|
|
464
516
|
|
|
@@ -19,17 +19,21 @@ export const PLUGIN_ENTRY_MAX_BYTES = 1024 * 1024
|
|
|
19
19
|
|
|
20
20
|
/** What a plugin's `ctx.cache` offers. Keys are the plugin's own names. */
|
|
21
21
|
export interface PluginCache {
|
|
22
|
-
/**
|
|
23
|
-
|
|
22
|
+
/**
|
|
23
|
+
* The bytes stored under `key`, from the first layer holding them; null
|
|
24
|
+
* when none does. On a scoped run a remote is asked for the branch's
|
|
25
|
+
* entry, or with `scoped` false the base branch's: one request either way.
|
|
26
|
+
*/
|
|
27
|
+
get(key: string, scoped?: boolean): Promise<Uint8Array | null>
|
|
24
28
|
/** Store `bytes` under `key` in every layer the policy writes, where it is not already; empty bytes store nothing. */
|
|
25
29
|
put(key: string, bytes: Uint8Array): Promise<void>
|
|
26
30
|
/** Whether some layer holds `key`; null when no layer could answer. */
|
|
27
|
-
has(key: string): Promise<boolean | null>
|
|
31
|
+
has(key: string, scoped?: boolean): Promise<boolean | null>
|
|
28
32
|
/**
|
|
29
33
|
* The subset of `keys` some layer holds, in one probe per layer; null
|
|
30
34
|
* when a readable remote has no batch probe: ask `get` one at a time.
|
|
31
35
|
*/
|
|
32
|
-
hasMany(keys: readonly string[]): Promise<Set<string> | null>
|
|
36
|
+
hasMany(keys: readonly string[], scoped?: boolean): Promise<Set<string> | null>
|
|
33
37
|
/**
|
|
34
38
|
* What core kept of a run under `name` (a `log` or `outputs` name its
|
|
35
39
|
* `RunReport` hands `teardown`), from the first readable remote holding
|
|
@@ -88,8 +92,9 @@ async function capped(body: unknown, max = PLUGIN_ENTRY_MAX_BYTES): Promise<Uint
|
|
|
88
92
|
/**
|
|
89
93
|
* `plugin`'s view of the cache. The local store never knows a scope
|
|
90
94
|
* (owner, 2026-10-17): it keeps every entry under its name's key. A scoped
|
|
91
|
-
* run
|
|
92
|
-
*
|
|
95
|
+
* run writes a remote only under the scope's key, and reads one key: the
|
|
96
|
+
* scope's, or with `scoped` false the default (M, 2026-10-18: no request
|
|
97
|
+
* twice).
|
|
93
98
|
*/
|
|
94
99
|
export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache {
|
|
95
100
|
const { db, policy, scope } = host
|
|
@@ -97,9 +102,9 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
97
102
|
const writers = host.remotes.filter((r) => r.write)
|
|
98
103
|
const at: RemoteKey = scope === undefined ? {} : { scope }
|
|
99
104
|
const atOf = (key: string, name: string): RemoteKey => (key === entryKey(plugin, name) ? {} : at)
|
|
100
|
-
const
|
|
105
|
+
const readKey = (name: string, scoped: boolean): string => {
|
|
101
106
|
const key = entryKey(plugin, name)
|
|
102
|
-
return scope === undefined ?
|
|
107
|
+
return scope === undefined || !scoped ? key : scopedKey(scope, key)
|
|
103
108
|
}
|
|
104
109
|
const writeKey = (name: string): string => {
|
|
105
110
|
const key = entryKey(plugin, name)
|
|
@@ -143,23 +148,21 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
143
148
|
}
|
|
144
149
|
|
|
145
150
|
return {
|
|
146
|
-
async get(name) {
|
|
151
|
+
async get(name, scoped = true) {
|
|
147
152
|
const local = entryKey(plugin, name)
|
|
148
153
|
const hit = localGet(local)
|
|
149
154
|
if (hit !== null) return hit
|
|
150
|
-
const
|
|
155
|
+
const key = readKey(name, scoped)
|
|
151
156
|
for (const { remote } of readers) {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
// a failing remote is a miss there
|
|
162
|
-
}
|
|
157
|
+
try {
|
|
158
|
+
const got = await remote.get(key, atOf(key, name))
|
|
159
|
+
if (got === null) continue
|
|
160
|
+
const bytes = await capped(got.body)
|
|
161
|
+
if (bytes === null) continue
|
|
162
|
+
if (policy.localWrite) localPut(local, bytes)
|
|
163
|
+
return bytes
|
|
164
|
+
} catch {
|
|
165
|
+
// a failing remote is a miss there
|
|
163
166
|
}
|
|
164
167
|
}
|
|
165
168
|
return null
|
|
@@ -178,26 +181,24 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
178
181
|
}
|
|
179
182
|
},
|
|
180
183
|
|
|
181
|
-
async has(name) {
|
|
184
|
+
async has(name, scoped = true) {
|
|
182
185
|
if (localHas([entryKey(plugin, name)]).size > 0) return true
|
|
183
|
-
const
|
|
186
|
+
const key = readKey(name, scoped)
|
|
184
187
|
let answered = policy.localRead
|
|
185
188
|
for (const { remote } of readers) {
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
// no answer from this remote
|
|
192
|
-
}
|
|
189
|
+
try {
|
|
190
|
+
if (await remote.has(key, atOf(key, name))) return true
|
|
191
|
+
answered = true
|
|
192
|
+
} catch {
|
|
193
|
+
// no answer from this remote
|
|
193
194
|
}
|
|
194
195
|
}
|
|
195
196
|
return answered ? false : null
|
|
196
197
|
},
|
|
197
198
|
|
|
198
|
-
async hasMany(names) {
|
|
199
|
+
async hasMany(names, scoped = true) {
|
|
199
200
|
const byKey = new Map<string, string>()
|
|
200
|
-
for (const name of names)
|
|
201
|
+
for (const name of names) byKey.set(readKey(name, scoped), name)
|
|
201
202
|
const keys = [...byKey.keys()]
|
|
202
203
|
const found = new Set<string>()
|
|
203
204
|
for (const key of localHas(names.map((n) => entryKey(plugin, n)))) found.add(byKey.get(key)!)
|
package/src/cache/remote-lane.ts
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
// class with the endpoint named. The order across remotes, the local store
|
|
5
5
|
// and provenance are `LayeredCache`'s; nothing here knows another remote.
|
|
6
6
|
|
|
7
|
+
import { machineParallelism } from '../util/index.js'
|
|
7
8
|
import type { RemoteKeyTraffic, RemoteTraffic } from './cache.js'
|
|
8
9
|
import type { RemoteCacheLayer } from './layered-cache.js'
|
|
9
10
|
|
|
@@ -20,11 +21,13 @@ export interface ResolvedRemote {
|
|
|
20
21
|
}
|
|
21
22
|
|
|
22
23
|
/**
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
24
|
+
* Requests at once to a remote, by default: pnpm's network concurrency
|
|
25
|
+
* (Bun's is a fixed 64). A request waits on the network, not a core, so
|
|
26
|
+
* the cap follows cores only past 21 of them.
|
|
26
27
|
*/
|
|
27
|
-
|
|
28
|
+
export function networkConcurrency(): number {
|
|
29
|
+
return Math.min(96, Math.max(machineParallelism() * 3, 64))
|
|
30
|
+
}
|
|
28
31
|
|
|
29
32
|
/** The seam call a failure happened in, as the warning names it. */
|
|
30
33
|
export type RemoteOp = 'probe' | 'download' | 'upload'
|
|
@@ -135,6 +138,8 @@ export class RemoteLane {
|
|
|
135
138
|
private readonly traffic: RemoteTraffic,
|
|
136
139
|
/** The wire as this run asks it: a branch scope's view of `resolved.remote`. */
|
|
137
140
|
remote: RemoteCacheLayer = resolved.remote,
|
|
141
|
+
/** Background PUTs at once; excess uploads queue and drain FIFO. */
|
|
142
|
+
private readonly uploadConcurrency: number = networkConcurrency(),
|
|
138
143
|
) {
|
|
139
144
|
this.resolved = resolved
|
|
140
145
|
this.remote = remote
|
|
@@ -255,7 +260,7 @@ export class RemoteLane {
|
|
|
255
260
|
upload(
|
|
256
261
|
hash: string,
|
|
257
262
|
durationMs: number,
|
|
258
|
-
body: () => { body: Blob; release?: () => Promise<void> }
|
|
263
|
+
body: () => Promise<{ body: Blob; release?: () => Promise<void> }>,
|
|
259
264
|
tally: { count: number; bytes: number; ms: number; failed: number },
|
|
260
265
|
): void {
|
|
261
266
|
const key = this.trafficOf(hash)
|
|
@@ -268,7 +273,7 @@ export class RemoteLane {
|
|
|
268
273
|
if (!this.read && (await this.holds(hash))) return
|
|
269
274
|
if (this.stopped) return
|
|
270
275
|
const start = performance.now()
|
|
271
|
-
const got = body()
|
|
276
|
+
const got = await body()
|
|
272
277
|
release = got.release
|
|
273
278
|
await this.remote.put(hash, got.body, { durationMs })
|
|
274
279
|
tally.count++
|
|
@@ -369,7 +374,7 @@ export class RemoteLane {
|
|
|
369
374
|
}
|
|
370
375
|
|
|
371
376
|
private pump(): void {
|
|
372
|
-
while (this.activeUploads <
|
|
377
|
+
while (this.activeUploads < this.uploadConcurrency && this.uploadQueue.length > 0) {
|
|
373
378
|
const job = this.uploadQueue.shift()!
|
|
374
379
|
this.activeUploads++
|
|
375
380
|
// Jobs never reject (each wraps its PUT in the never-fail guard).
|
package/src/cache/tar-stream.ts
CHANGED
|
@@ -122,14 +122,19 @@ class Source {
|
|
|
122
122
|
constructor(private readonly reader: AsyncIterator<Uint8Array>) {}
|
|
123
123
|
|
|
124
124
|
private async fill(): Promise<boolean> {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
this.done
|
|
129
|
-
|
|
125
|
+
// An empty chunk is skipped, never reported as data: the callers read
|
|
126
|
+
// `chunks[0]` after a true (the tar a tree packs back yields them).
|
|
127
|
+
for (;;) {
|
|
128
|
+
if (this.done) return false
|
|
129
|
+
const { done, value } = await this.reader.next()
|
|
130
|
+
if (done === true || value === undefined) {
|
|
131
|
+
this.done = true
|
|
132
|
+
return false
|
|
133
|
+
}
|
|
134
|
+
if (value.byteLength === 0) continue
|
|
135
|
+
this.chunks.push(value)
|
|
136
|
+
return true
|
|
130
137
|
}
|
|
131
|
-
if (value.byteLength > 0) this.chunks.push(value)
|
|
132
|
-
return true
|
|
133
138
|
}
|
|
134
139
|
|
|
135
140
|
/** Exactly `n` bytes, or null at a clean end (nothing buffered), or a TarFormatError mid-entry. */
|
package/src/cli/format.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
// `formatBytes`
|
|
2
|
-
// names what the workspace's `cacheRetention` evicted, and the orchestrator
|
|
1
|
+
// `formatBytes` lives in util/size.ts beside the parsers: the orchestrator
|
|
3
2
|
// may not import the CLI.
|
|
4
3
|
export { formatBytes } from '../util/index.js'
|
package/src/cli/help.ts
CHANGED
|
@@ -163,6 +163,12 @@ export function helpText(pluginCommands: readonly string[] = []): string {
|
|
|
163
163
|
' .vx/cache, entries in ~/.vx/<id>/cache.',
|
|
164
164
|
' --cache-download <mode> all (default) | toplevel | none:',
|
|
165
165
|
' where remote outputs land.',
|
|
166
|
+
' --cache-concurrency-download <n>',
|
|
167
|
+
' Remote downloads at once.',
|
|
168
|
+
' --cache-concurrency-upload <n>',
|
|
169
|
+
' Uploads at once, per remote.',
|
|
170
|
+
' --cache-concurrency-restore <n>',
|
|
171
|
+
' Local hits restoring at once.',
|
|
166
172
|
' --retries <n> Up to <n> more runs of a failed task, for',
|
|
167
173
|
' tasks without their own exec.retries. Not',
|
|
168
174
|
' folded into cache keys.',
|
package/src/cli/info.ts
CHANGED
|
@@ -100,9 +100,6 @@ export function renderInfo(f: InfoFacts, colors: ColorSupport = { enabled: false
|
|
|
100
100
|
...(Object.values(f.settings).some((s) => s.source !== 'default')
|
|
101
101
|
? ([['settings', describeSettings(f.settings)]] as [string, string][])
|
|
102
102
|
: []),
|
|
103
|
-
...(f.cacheRetention !== null
|
|
104
|
-
? ([['cache retention', describeRetention(f.cacheRetention)]] as [string, string][])
|
|
105
|
-
: []),
|
|
106
103
|
// Only when there is something to say.
|
|
107
104
|
...(f.orphans.artifacts > 0
|
|
108
105
|
? ([
|
|
@@ -128,14 +125,9 @@ export function renderInfo(f: InfoFacts, colors: ColorSupport = { enabled: false
|
|
|
128
125
|
return ui.report('vx info', rows).lines(colors).join('\n')
|
|
129
126
|
}
|
|
130
127
|
|
|
131
|
-
/** `812 (1.2 GB)
|
|
128
|
+
/** `812 (1.2 GB)`. */
|
|
132
129
|
function describeEntries(f: InfoFacts): string {
|
|
133
|
-
|
|
134
|
-
const size =
|
|
135
|
-
max === undefined
|
|
136
|
-
? formatBytes(f.cacheBytes)
|
|
137
|
-
: `${formatBytes(f.cacheBytes)} of ${formatBytes(max)}, ${Math.round((f.cacheBytes / max) * 100)}%`
|
|
138
|
-
return `${f.cacheEntries} (${size})`
|
|
130
|
+
return `${f.cacheEntries} (${formatBytes(f.cacheBytes)})`
|
|
139
131
|
}
|
|
140
132
|
|
|
141
133
|
/**
|
|
@@ -152,15 +144,6 @@ function describeSettings(settings: InfoFacts['settings']): string {
|
|
|
152
144
|
.join(', ')
|
|
153
145
|
}
|
|
154
146
|
|
|
155
|
-
/** What a run's end evicts: `entries unused for 30d, and the oldest entries past 10G`. */
|
|
156
|
-
function describeRetention(r: NonNullable<InfoFacts['cacheRetention']>): string {
|
|
157
|
-
const parts = [
|
|
158
|
-
...(r.olderThan !== undefined ? [`entries unused for ${r.olderThan}`] : []),
|
|
159
|
-
...(r.maxSize !== undefined ? [`the oldest entries past ${r.maxSize}`] : []),
|
|
160
|
-
]
|
|
161
|
-
return `each run evicts ${parts.join(', and ')}`
|
|
162
|
-
}
|
|
163
|
-
|
|
164
147
|
/** A path under the home directory as `~/…`: shorter, and no user name in a pasted report. */
|
|
165
148
|
function tildePath(p: string): string {
|
|
166
149
|
const home = os.homedir()
|
|
@@ -144,7 +144,7 @@ async function verbCache(
|
|
|
144
144
|
])
|
|
145
145
|
const scope = resolveCacheScope(
|
|
146
146
|
ws.workspaceRoot,
|
|
147
|
-
ws.branch.scope,
|
|
147
|
+
ws.branch.trusted === true ? null : ws.branch.scope,
|
|
148
148
|
ws.plugins.some((p) => p.cache !== undefined),
|
|
149
149
|
configuredBaseBranch(ws.workspaceConfig),
|
|
150
150
|
)
|
package/src/cli/run.ts
CHANGED
|
@@ -109,6 +109,9 @@ const SETTING_OPTIONS = [
|
|
|
109
109
|
['run.retries', 'retries'],
|
|
110
110
|
['run.outputLogs', 'outputLogs'],
|
|
111
111
|
['cache.download', 'download'],
|
|
112
|
+
['cache.concurrency.download', 'downloadConcurrency'],
|
|
113
|
+
['cache.concurrency.upload', 'uploadConcurrency'],
|
|
114
|
+
['cache.concurrency.restore', 'restoreConcurrency'],
|
|
112
115
|
] as const satisfies readonly (readonly [string, keyof RunOptions])[]
|
|
113
116
|
|
|
114
117
|
/** `verb` is the one being parsed for: `vx watch` reads `vx run`'s flags, and
|
package/src/cli/upgrade.ts
CHANGED
|
@@ -21,6 +21,7 @@ import { access, chmod, chown, link, mkdtemp, rename, rm, stat } from 'node:fs/p
|
|
|
21
21
|
import path from 'node:path'
|
|
22
22
|
import { flagHint, seeHelp } from './help.js'
|
|
23
23
|
import { UserError, sayNotice, ui } from '../util/index.js'
|
|
24
|
+
import { findWorkspaceRoot } from '../workspace/index.js'
|
|
24
25
|
import { VERSION } from '../version.js'
|
|
25
26
|
|
|
26
27
|
const REGISTRY = 'https://registry.npmjs.org'
|
|
@@ -596,11 +597,11 @@ export async function upgradeManaged(
|
|
|
596
597
|
version: string,
|
|
597
598
|
current = VERSION,
|
|
598
599
|
step = '',
|
|
599
|
-
): Promise<
|
|
600
|
+
): Promise<boolean> {
|
|
600
601
|
const plugins = install.plugins ?? []
|
|
601
602
|
if (version === current && plugins.every((p) => installedVersion(install.cwd, p) === version)) {
|
|
602
603
|
ui.print('out', ui.status('info', 'vx upgrade', `${step}already at ${current}`))
|
|
603
|
-
return
|
|
604
|
+
return false
|
|
604
605
|
}
|
|
605
606
|
const argv = upgradeArgv(install, version)
|
|
606
607
|
// PATH as it is now: `Bun.which` alone reads the one the process started with.
|
|
@@ -636,6 +637,7 @@ export async function upgradeManaged(
|
|
|
636
637
|
}`,
|
|
637
638
|
)
|
|
638
639
|
}
|
|
640
|
+
return true
|
|
639
641
|
}
|
|
640
642
|
|
|
641
643
|
/**
|
|
@@ -648,7 +650,7 @@ async function upgradeBinary(
|
|
|
648
650
|
current = VERSION,
|
|
649
651
|
step = '',
|
|
650
652
|
key: SigningKey = NPM_SIGNING_KEY,
|
|
651
|
-
): Promise<
|
|
653
|
+
): Promise<boolean> {
|
|
652
654
|
const pkg = platformPackage()
|
|
653
655
|
let installed: string | undefined
|
|
654
656
|
const replaced = await replaceBinary(
|
|
@@ -679,6 +681,7 @@ async function upgradeBinary(
|
|
|
679
681
|
? ui.status('ok', 'vx upgrade', `${step}installed ${installed}`)
|
|
680
682
|
: ui.status('info', 'vx upgrade', `${step}already at ${current}`),
|
|
681
683
|
)
|
|
684
|
+
return replaced
|
|
682
685
|
}
|
|
683
686
|
|
|
684
687
|
/** The version `bin --version` answers, without the hand-off note a handed-off run adds. */
|
|
@@ -738,9 +741,9 @@ export async function upgradeOnlyGlobal(
|
|
|
738
741
|
}
|
|
739
742
|
|
|
740
743
|
/** `run`, its refusal naming `what` after the verb: a run that moves two installs says which failed. */
|
|
741
|
-
async function named(what: string, run: () => Promise<
|
|
744
|
+
async function named<T>(what: string, run: () => Promise<T>): Promise<T> {
|
|
742
745
|
try {
|
|
743
|
-
await run()
|
|
746
|
+
return await run()
|
|
744
747
|
} catch (err) {
|
|
745
748
|
if (!(err instanceof UserError)) throw err
|
|
746
749
|
throw new UserError(err.message.replace(/^vx upgrade: /, `vx upgrade: ${what}: `), err.code)
|
|
@@ -787,19 +790,46 @@ export async function upgradeCmd(argv: readonly string[]): Promise<number> {
|
|
|
787
790
|
const from = handedOffFrom
|
|
788
791
|
if (from !== undefined) {
|
|
789
792
|
const version = await targetVersion(tag)
|
|
790
|
-
await named('project vx', () =>
|
|
793
|
+
const moved = await named('project vx', () =>
|
|
791
794
|
install === null
|
|
792
795
|
? upgradeBinary(dest, version, VERSION, 'project ')
|
|
793
796
|
: upgradeManaged(install, version, VERSION, 'project '),
|
|
794
797
|
)
|
|
795
798
|
await upgradeGlobal(from, version)
|
|
799
|
+
if (moved) await relock(newVx(install, dest))
|
|
796
800
|
return 0
|
|
797
801
|
}
|
|
798
|
-
|
|
799
|
-
|
|
802
|
+
const moved =
|
|
803
|
+
install !== null
|
|
804
|
+
? await upgradeManaged(install, await targetVersion(tag))
|
|
805
|
+
: await upgradeBinary(dest, versionSpec(tag))
|
|
806
|
+
if (moved) await relock(newVx(install, dest))
|
|
800
807
|
return 0
|
|
801
808
|
}
|
|
802
809
|
|
|
810
|
+
/**
|
|
811
|
+
* The vx an upgrade just installed: the replaced binary, or the project's
|
|
812
|
+
* `node_modules/.bin/vx` (a manager's store path names the old version).
|
|
813
|
+
*/
|
|
814
|
+
const newVx = (install: ManagedInstall | null, dest: string): string =>
|
|
815
|
+
install === null ? dest : path.join(install.cwd, 'node_modules', '.bin', 'vx')
|
|
816
|
+
|
|
817
|
+
/**
|
|
818
|
+
* Lock the workspace with the vx just installed at `bin` (M, 2026-10-18):
|
|
819
|
+
* a release may record the lock differently. `-g` never re-locks. A lock
|
|
820
|
+
* that fails leaves the upgrade done and says what to run.
|
|
821
|
+
*/
|
|
822
|
+
export async function relock(bin: string, cwd = process.cwd()): Promise<void> {
|
|
823
|
+
const root = await findWorkspaceRoot(cwd).catch(() => null)
|
|
824
|
+
if (root === null) return
|
|
825
|
+
const proc = Bun.spawnSync({
|
|
826
|
+
cmd: [bin, 'lock'],
|
|
827
|
+
cwd: root,
|
|
828
|
+
stdio: ['inherit', 'inherit', 'inherit'],
|
|
829
|
+
})
|
|
830
|
+
if (proc.exitCode !== 0) sayNotice('warn', 'vx upgrade', 'vx lock failed: run vx lock')
|
|
831
|
+
}
|
|
832
|
+
|
|
803
833
|
/**
|
|
804
834
|
* `bin --version` when it answers as vx within `timeoutMs`, else null. The
|
|
805
835
|
* answer decides whether a replaced binary is rolled back, so it is
|