@vzn/vx 0.0.644 → 0.0.645
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +15 -23
- package/package.json +5 -5
- package/schemas/init.json +5 -0
- package/src/cache/archive.ts +12 -76
- package/src/cache/cache.ts +6 -26
- package/src/cache/index.ts +1 -2
- package/src/cache/layer.ts +5 -6
- package/src/cache/layered-cache.ts +106 -42
- package/src/cache/plugin-cache.ts +9 -14
- package/src/cache/remote-lane.ts +47 -18
- package/src/cache/scoped-cache.ts +3 -80
- package/src/cli/help.ts +4 -4
- package/src/cli/init.ts +121 -20
- package/src/cli/plan-format.ts +6 -2
- package/src/cli/plugin-commands.ts +3 -4
- package/src/cli/plugin-templates.ts +1 -1
- package/src/cli/run.ts +0 -1
- package/src/cli/upgrade.ts +1 -1
- package/src/config.ts +2 -4
- package/src/index.ts +1 -1
- package/src/orchestrator/affected-tasks.ts +0 -18
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/options.ts +0 -8
- package/src/orchestrator/plan.ts +7 -1
- package/src/orchestrator/plugin-host.ts +2 -5
- package/src/orchestrator/plugin.ts +11 -7
- package/src/orchestrator/prepare.ts +4 -47
- package/src/orchestrator/run-context.ts +39 -1
- package/src/orchestrator/run.ts +2 -0
- package/src/orchestrator/task-hash.ts +1 -3
- package/src/report/summary.ts +14 -2
- package/src/settings.ts +1 -7
- package/src/workspace/affected.ts +0 -39
- package/src/workspace/index.ts +0 -1
|
@@ -36,7 +36,6 @@ import type {
|
|
|
36
36
|
} from './cache.js'
|
|
37
37
|
import type { OutputDirRow } from './cache.js'
|
|
38
38
|
import { RemoteLane, type ResolvedRemote } from './remote-lane.js'
|
|
39
|
-
import type { BranchScope } from './scoped-cache.js'
|
|
40
39
|
import type { PluginSecrets } from '../util/index.js'
|
|
41
40
|
|
|
42
41
|
const STOPPED: unique symbol = Symbol('stopped')
|
|
@@ -46,7 +45,11 @@ const STOPPED: unique symbol = Symbol('stopped')
|
|
|
46
45
|
* fails at once, removes its temp, and releases the connection; the cache
|
|
47
46
|
* then closes under no write.
|
|
48
47
|
*/
|
|
49
|
-
function endsOnStop(
|
|
48
|
+
function endsOnStop(
|
|
49
|
+
body: Blob | Response,
|
|
50
|
+
stopped: Promise<typeof STOPPED>,
|
|
51
|
+
onChunk: (bytes: number) => void = () => {},
|
|
52
|
+
): Blob | Response {
|
|
50
53
|
if (body instanceof Blob || body.body === null) return body
|
|
51
54
|
const reader = body.body.getReader()
|
|
52
55
|
let stop = false
|
|
@@ -61,13 +64,28 @@ function endsOnStop(body: Blob | Response, stopped: Promise<typeof STOPPED>): Bl
|
|
|
61
64
|
const next = await reader.read()
|
|
62
65
|
if (stop) controller.error(new Error('the run was stopped'))
|
|
63
66
|
else if (next.done) controller.close()
|
|
64
|
-
else
|
|
67
|
+
else {
|
|
68
|
+
onChunk(next.value.byteLength)
|
|
69
|
+
controller.enqueue(next.value)
|
|
70
|
+
}
|
|
65
71
|
},
|
|
66
72
|
cancel: (reason) => reader.cancel(reason),
|
|
67
73
|
})
|
|
68
74
|
return new Response(stream, { headers: body.headers })
|
|
69
75
|
}
|
|
70
76
|
|
|
77
|
+
const judged = (i: ArtifactInfo): i is { size: number; durationMs: number } =>
|
|
78
|
+
i.size !== undefined && i.durationMs !== undefined
|
|
79
|
+
|
|
80
|
+
/** How often a task waiting on its download weighs running instead. */
|
|
81
|
+
const JUDGE_EVERY_MS = 25
|
|
82
|
+
|
|
83
|
+
interface Download {
|
|
84
|
+
readonly start: number
|
|
85
|
+
received: number
|
|
86
|
+
abandon(): void
|
|
87
|
+
}
|
|
88
|
+
|
|
71
89
|
/** What a remote's `open` is handed: the run's CI verdict and its plugin's keychain. */
|
|
72
90
|
export interface RemoteOpenContext extends PluginSecrets {
|
|
73
91
|
/** The run is on CI: nobody may be asked. */
|
|
@@ -77,13 +95,13 @@ export interface RemoteOpenContext extends PluginSecrets {
|
|
|
77
95
|
}
|
|
78
96
|
|
|
79
97
|
/**
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
* go straight to that store. turboCache(), nxCache() and vx-reapi ignore it.
|
|
98
|
+
* What a remote knows of an artifact it holds, before it is downloaded:
|
|
99
|
+
* the body's size in bytes and the producing task's duration, each when
|
|
100
|
+
* the wire carries it (Turbo's batch query and `HEAD` do).
|
|
84
101
|
*/
|
|
85
|
-
export interface
|
|
86
|
-
readonly
|
|
102
|
+
export interface ArtifactInfo {
|
|
103
|
+
readonly size?: number
|
|
104
|
+
readonly durationMs?: number
|
|
87
105
|
}
|
|
88
106
|
|
|
89
107
|
/**
|
|
@@ -122,17 +140,21 @@ export interface RemoteCacheLayer {
|
|
|
122
140
|
* plugin's keychain and may ask on the terminal.
|
|
123
141
|
*/
|
|
124
142
|
open?(ctx: RemoteOpenContext): Promise<void>
|
|
125
|
-
/** Existence probe: `--dry` and `--force` ask it per key when the remote cannot batch. */
|
|
126
|
-
has(hash: string, at?: RemoteKey): Promise<boolean>
|
|
127
143
|
/**
|
|
128
|
-
*
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
* `--force` in the background to skip uploads of held keys. Without it a
|
|
132
|
-
* run GETs each key directly (no `has`). Never throws for control flow:
|
|
133
|
-
* `null` means "no batch info; use per-hash". `at[i]` is `hashes[i]`'s.
|
|
144
|
+
* Existence probe: what the remote knows of the stored artifact, or null
|
|
145
|
+
* when it holds none. `--dry` and `--force` ask it per key when the
|
|
146
|
+
* remote cannot batch.
|
|
134
147
|
*/
|
|
135
|
-
|
|
148
|
+
has(hash: string): Promise<ArtifactInfo | null>
|
|
149
|
+
/**
|
|
150
|
+
* Optional batch existence probe: given N hashes, the stored ones and what
|
|
151
|
+
* the remote knows of each, in ONE round-trip. Its presence says the wire
|
|
152
|
+
* can batch: a run asks it once for its keys, `--dry` before its per-key
|
|
153
|
+
* answers, and `--force` in the background to skip uploads of held keys.
|
|
154
|
+
* Without it a run GETs each key directly (no `has`). Never throws for
|
|
155
|
+
* control flow: `null` means "no batch info; use per-hash".
|
|
156
|
+
*/
|
|
157
|
+
hasMany?(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null>
|
|
136
158
|
/**
|
|
137
159
|
* Fetch an artifact; `null` = miss. `body` is read once, by core,
|
|
138
160
|
* straight into the local cache — an HTTP wire returns its `fetch`
|
|
@@ -142,10 +164,7 @@ export interface RemoteCacheLayer {
|
|
|
142
164
|
* nothing and the read is a miss, so a layer can check bytes as they pass
|
|
143
165
|
* (a digest, a signature) and refuse at the end.
|
|
144
166
|
*/
|
|
145
|
-
get(
|
|
146
|
-
hash: string,
|
|
147
|
-
at?: RemoteKey,
|
|
148
|
-
): Promise<{ body: Blob | Response; durationMs: number | undefined } | null>
|
|
167
|
+
get(hash: string): Promise<{ body: Blob | Response; durationMs: number | undefined } | null>
|
|
149
168
|
/**
|
|
150
169
|
* Store an artifact (fire-and-forget from LayeredCache's perspective).
|
|
151
170
|
* `body` is file-backed (`Bun.file`) when the local store holds the
|
|
@@ -154,7 +173,7 @@ export interface RemoteCacheLayer {
|
|
|
154
173
|
* key the remote holds keeps the stored bytes. Core never asks `has`
|
|
155
174
|
* first; a wire that would overwrite checks inside.
|
|
156
175
|
*/
|
|
157
|
-
put(hash: string, body: Blob, meta: { durationMs: number }
|
|
176
|
+
put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void>
|
|
158
177
|
}
|
|
159
178
|
|
|
160
179
|
export class LayeredCache implements CacheLayer {
|
|
@@ -196,23 +215,18 @@ export class LayeredCache implements CacheLayer {
|
|
|
196
215
|
* its download and ingest took.
|
|
197
216
|
*/
|
|
198
217
|
private readonly remoteSourced = new Map<string, number>()
|
|
218
|
+
/** What the batch probe said of each held key: its size and stored duration. */
|
|
219
|
+
private readonly infos = new Map<string, ArtifactInfo>()
|
|
220
|
+
/** Bodies being read now, by key: how far along, and how to give one up. */
|
|
221
|
+
private readonly downloads = new Map<string, Download>()
|
|
199
222
|
|
|
200
223
|
/** Remotes in declared order: the first readable one holding a hash answers it. */
|
|
201
224
|
constructor(
|
|
202
225
|
readonly local: Cache,
|
|
203
226
|
remotes: readonly ResolvedRemote[],
|
|
204
|
-
/** The run's branch scope: it picks each remote key; the local store never sees it. */
|
|
205
|
-
private readonly branch?: BranchScope,
|
|
206
227
|
uploadConcurrency?: number,
|
|
207
228
|
) {
|
|
208
|
-
this.lanes = remotes.map(
|
|
209
|
-
(r) =>
|
|
210
|
-
new RemoteLane(r, local.traffic, branch?.remote(r.remote) ?? r.remote, uploadConcurrency),
|
|
211
|
-
)
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
noteKey(taskId: string, hash: string): void {
|
|
215
|
-
this.branch?.noteKey(taskId, hash)
|
|
229
|
+
this.lanes = remotes.map((r) => new RemoteLane(r, local.traffic, uploadConcurrency))
|
|
216
230
|
}
|
|
217
231
|
|
|
218
232
|
// Asked, not kept: a remote whose `open` failed leaves the run after this is built.
|
|
@@ -249,9 +263,12 @@ export class LayeredCache implements CacheLayer {
|
|
|
249
263
|
* absences either way, so a hash only a sibling holds costs it no GET.
|
|
250
264
|
* Never throws.
|
|
251
265
|
*/
|
|
252
|
-
async remoteHasMany(hashes: readonly string[]): Promise<
|
|
266
|
+
async remoteHasMany(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
|
|
253
267
|
if (this.readable.length === 0) return null
|
|
254
|
-
const answer = this.probeMany(hashes)
|
|
268
|
+
const answer = this.probeMany(hashes).then((found) => {
|
|
269
|
+
if (found !== null) for (const [h, info] of found) this.infos.set(h, info)
|
|
270
|
+
return found
|
|
271
|
+
})
|
|
255
272
|
const waiting = hashes.filter((h) => !this.inflight.has(h) && !this.probing.has(h))
|
|
256
273
|
for (const h of waiting) {
|
|
257
274
|
this.probing.set(
|
|
@@ -271,12 +288,12 @@ export class LayeredCache implements CacheLayer {
|
|
|
271
288
|
return answer
|
|
272
289
|
}
|
|
273
290
|
|
|
274
|
-
private async probeMany(hashes: readonly string[]): Promise<
|
|
275
|
-
let out:
|
|
291
|
+
private async probeMany(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
|
|
292
|
+
let out: Map<string, ArtifactInfo> | null = new Map()
|
|
276
293
|
for (const lane of this.readable) {
|
|
277
294
|
const found = await lane.hasMany(hashes)
|
|
278
295
|
if (found === null) out = null
|
|
279
|
-
else if (out !== null) for (const h of found) out.
|
|
296
|
+
else if (out !== null) for (const [h, info] of found) if (!out.has(h)) out.set(h, info)
|
|
280
297
|
}
|
|
281
298
|
return out
|
|
282
299
|
}
|
|
@@ -328,7 +345,7 @@ export class LayeredCache implements CacheLayer {
|
|
|
328
345
|
// remote walk per key: a settled `false` means no remote had it
|
|
329
346
|
// (no second probe), and a settled/in-flight `true` means the
|
|
330
347
|
// artifact was/will-be ingested locally — re-read below.
|
|
331
|
-
const ingested = await this.pullFromRemote(hash, ctx)
|
|
348
|
+
const ingested = await this.awaitOrRun(hash, this.pullFromRemote(hash, ctx))
|
|
332
349
|
if (!ingested) return null
|
|
333
350
|
|
|
334
351
|
// Read past the local READ gate: `ingest` is deliberately ungated, so
|
|
@@ -350,6 +367,39 @@ export class LayeredCache implements CacheLayer {
|
|
|
350
367
|
return this.asRemote(hash, materialized)
|
|
351
368
|
}
|
|
352
369
|
|
|
370
|
+
/**
|
|
371
|
+
* A pull the task now waits on, given up when the rest of its body would
|
|
372
|
+
* take longer than running the task (its stored duration, doubled for
|
|
373
|
+
* the save a run adds): the task then runs here, a miss. Only a key the
|
|
374
|
+
* batch probe sized and timed is judged, from its own body's pace so far.
|
|
375
|
+
* The facts are read each tick: the probe can answer after the task asks.
|
|
376
|
+
*/
|
|
377
|
+
private async awaitOrRun(hash: string, pull: Promise<boolean>): Promise<boolean> {
|
|
378
|
+
if (this.infos.has(hash) && !judged(this.infos.get(hash)!)) return pull
|
|
379
|
+
for (;;) {
|
|
380
|
+
const done = await Promise.race([pull, Bun.sleep(JUDGE_EVERY_MS).then(() => undefined)])
|
|
381
|
+
if (done !== undefined) return done
|
|
382
|
+
const info = this.infos.get(hash)
|
|
383
|
+
if (info === undefined) continue
|
|
384
|
+
if (!judged(info)) return pull
|
|
385
|
+
const budget = info.durationMs * 2
|
|
386
|
+
const dl = this.downloads.get(hash)
|
|
387
|
+
if (dl === undefined) continue
|
|
388
|
+
const elapsed = performance.now() - dl.start
|
|
389
|
+
const left =
|
|
390
|
+
dl.received === 0
|
|
391
|
+
? elapsed > budget
|
|
392
|
+
? Infinity
|
|
393
|
+
: 0
|
|
394
|
+
: ((info.size - dl.received) * elapsed) / dl.received
|
|
395
|
+
if (left > budget) {
|
|
396
|
+
dl.abandon()
|
|
397
|
+
await pull
|
|
398
|
+
return false
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
353
403
|
private asRemote(hash: string, entry: CacheEntry): CacheEntry {
|
|
354
404
|
const fetchMs = this.remoteSourced.get(hash)
|
|
355
405
|
return fetchMs === undefined ? entry : { ...entry, source: 'remote', fetchMs }
|
|
@@ -360,7 +410,7 @@ export class LayeredCache implements CacheLayer {
|
|
|
360
410
|
// a remote cache. Remote errors degrade to a miss (never throw).
|
|
361
411
|
async has(hash: string): Promise<'local' | 'remote' | null> {
|
|
362
412
|
if ((await this.local.has(hash)) === 'local') return 'local'
|
|
363
|
-
for (const lane of this.readable) if (await lane.has(hash)) return 'remote'
|
|
413
|
+
for (const lane of this.readable) if ((await lane.has(hash)) !== null) return 'remote'
|
|
364
414
|
return null
|
|
365
415
|
}
|
|
366
416
|
|
|
@@ -434,10 +484,19 @@ export class LayeredCache implements CacheLayer {
|
|
|
434
484
|
durationMs: durationOf(remoteBody.durationMs),
|
|
435
485
|
...(ctx?.outputs !== undefined ? { outputs: ctx.outputs } : {}),
|
|
436
486
|
}
|
|
487
|
+
let abandon = (): void => {}
|
|
488
|
+
const abandoned = new Promise<typeof STOPPED>((resolve) => (abandon = () => resolve(STOPPED)))
|
|
489
|
+
const dl: Download = { start: performance.now(), received: 0, abandon }
|
|
490
|
+
this.downloads.set(hash, dl)
|
|
491
|
+
const ends = Promise.race([stopped, abandoned])
|
|
437
492
|
try {
|
|
438
|
-
const ingest = this.local.ingest(
|
|
493
|
+
const ingest = this.local.ingest(
|
|
494
|
+
hash,
|
|
495
|
+
endsOnStop(remoteBody.body, ends, (n) => (dl.received += n)),
|
|
496
|
+
meta,
|
|
497
|
+
)
|
|
439
498
|
ingest.catch(() => undefined)
|
|
440
|
-
if ((await Promise.race([ingest,
|
|
499
|
+
if ((await Promise.race([ingest, ends])) === STOPPED) return false
|
|
441
500
|
} catch (err) {
|
|
442
501
|
// The bytes came off the network — a corrupt/truncated remote
|
|
443
502
|
// artifact must degrade to a cache miss (task re-executes), not
|
|
@@ -445,8 +504,13 @@ export class LayeredCache implements CacheLayer {
|
|
|
445
504
|
// down: `Cache.restoreOutputs` drops it (A-52).
|
|
446
505
|
lane.report('download', hash, err)
|
|
447
506
|
return false
|
|
507
|
+
} finally {
|
|
508
|
+
this.downloads.delete(hash)
|
|
448
509
|
}
|
|
449
510
|
this.remoteSourced.set(hash, Math.round(performance.now() - fetchStart))
|
|
511
|
+
this.local.downloads.count++
|
|
512
|
+
this.local.downloads.bytes +=
|
|
513
|
+
remoteBody.body instanceof Blob ? remoteBody.body.size : dl.received
|
|
450
514
|
return true
|
|
451
515
|
}
|
|
452
516
|
|
|
@@ -8,7 +8,6 @@
|
|
|
8
8
|
|
|
9
9
|
import type { Database } from 'bun:sqlite'
|
|
10
10
|
import { xxh3hex } from '../util/index.js'
|
|
11
|
-
import type { RemoteKey } from './layered-cache.js'
|
|
12
11
|
import type { CachePolicy } from './policy.js'
|
|
13
12
|
import type { ResolvedRemote } from './remote-lane.js'
|
|
14
13
|
import { scopedKey } from './scoped-cache.js'
|
|
@@ -100,8 +99,6 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
100
99
|
const { db, policy, scope } = host
|
|
101
100
|
const readers = host.remotes.filter((r) => r.read)
|
|
102
101
|
const writers = host.remotes.filter((r) => r.write)
|
|
103
|
-
const at: RemoteKey = scope === undefined ? {} : { scope }
|
|
104
|
-
const atOf = (key: string, name: string): RemoteKey => (key === entryKey(plugin, name) ? {} : at)
|
|
105
102
|
const readKey = (name: string, scoped: boolean): string => {
|
|
106
103
|
const key = entryKey(plugin, name)
|
|
107
104
|
return scope === undefined || !scoped ? key : scopedKey(scope, key)
|
|
@@ -155,7 +152,7 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
155
152
|
const key = readKey(name, scoped)
|
|
156
153
|
for (const { remote } of readers) {
|
|
157
154
|
try {
|
|
158
|
-
const got = await remote.get(key
|
|
155
|
+
const got = await remote.get(key)
|
|
159
156
|
if (got === null) continue
|
|
160
157
|
const bytes = await capped(got.body)
|
|
161
158
|
if (bytes === null) continue
|
|
@@ -174,7 +171,7 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
174
171
|
if (policy.localWrite) localPut(entryKey(plugin, name), bytes)
|
|
175
172
|
for (const { remote } of writers) {
|
|
176
173
|
try {
|
|
177
|
-
await remote.put(key, new Blob([bytes]), { durationMs: 0
|
|
174
|
+
await remote.put(key, new Blob([bytes]), { durationMs: 0 })
|
|
178
175
|
} catch {
|
|
179
176
|
// the other layers still keep it
|
|
180
177
|
}
|
|
@@ -187,7 +184,7 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
187
184
|
let answered = policy.localRead
|
|
188
185
|
for (const { remote } of readers) {
|
|
189
186
|
try {
|
|
190
|
-
if (await remote.has(key
|
|
187
|
+
if (await remote.has(key)) return true
|
|
191
188
|
answered = true
|
|
192
189
|
} catch {
|
|
193
190
|
// no answer from this remote
|
|
@@ -207,12 +204,10 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
207
204
|
if (ask.length === 0) break
|
|
208
205
|
if (remote.hasMany === undefined) return null
|
|
209
206
|
try {
|
|
210
|
-
const got: unknown = await remote.hasMany(
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
if (!(got instanceof Set)) return null
|
|
215
|
-
for (const key of got) if (byKey.has(key as string)) found.add(byKey.get(key as string)!)
|
|
207
|
+
const got: unknown = await remote.hasMany(ask)
|
|
208
|
+
if (!(got instanceof Map)) return null
|
|
209
|
+
for (const key of got.keys())
|
|
210
|
+
if (byKey.has(key as string)) found.add(byKey.get(key as string)!)
|
|
216
211
|
} catch {
|
|
217
212
|
return null
|
|
218
213
|
}
|
|
@@ -222,9 +217,9 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
222
217
|
|
|
223
218
|
async getRun(name) {
|
|
224
219
|
for (const { remote } of readers) {
|
|
225
|
-
for (const
|
|
220
|
+
for (const key of runReadKeys(name, scope)) {
|
|
226
221
|
try {
|
|
227
|
-
const got = await remote.get(key
|
|
222
|
+
const got = await remote.get(key)
|
|
228
223
|
if (got === null) continue
|
|
229
224
|
const bytes = await capped(got.body, Infinity)
|
|
230
225
|
if (bytes !== null) return bytes
|
package/src/cache/remote-lane.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
import { machineParallelism } from '../util/index.js'
|
|
8
8
|
import type { RemoteKeyTraffic, RemoteTraffic } from './cache.js'
|
|
9
|
-
import type { RemoteCacheLayer } from './layered-cache.js'
|
|
9
|
+
import type { ArtifactInfo, RemoteCacheLayer } from './layered-cache.js'
|
|
10
10
|
|
|
11
11
|
/** One remote, resolved: its wire and what the run's policy and its `access` let it do. */
|
|
12
12
|
export interface ResolvedRemote {
|
|
@@ -48,6 +48,14 @@ function invalidRemoteResult(what: string): Error {
|
|
|
48
48
|
)
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
+
/** A plugin's `ArtifactInfo`, keeping only the fields that are finite, non-negative numbers; null when it is no object. */
|
|
52
|
+
function artifactInfo(value: unknown): ArtifactInfo | null {
|
|
53
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) return null
|
|
54
|
+
const { size, durationMs } = value as Record<string, unknown>
|
|
55
|
+
const ok = (n: unknown): n is number => typeof n === 'number' && Number.isFinite(n) && n >= 0
|
|
56
|
+
return { ...(ok(size) ? { size } : {}), ...(ok(durationMs) ? { durationMs } : {}) }
|
|
57
|
+
}
|
|
58
|
+
|
|
51
59
|
function isBody(value: unknown): value is Blob | Response {
|
|
52
60
|
return value instanceof Blob || value instanceof Response
|
|
53
61
|
}
|
|
@@ -106,14 +114,16 @@ export class RemoteLane {
|
|
|
106
114
|
*/
|
|
107
115
|
private readonly absent = new Set<string>()
|
|
108
116
|
/** What a `hasMany` said this remote holds, as `absent` records the rest. */
|
|
109
|
-
private readonly held = new
|
|
117
|
+
private readonly held = new Map<string, ArtifactInfo>()
|
|
110
118
|
|
|
111
119
|
/**
|
|
112
120
|
* A remote the run writes but does not read (`--force`): which of the
|
|
113
121
|
* run's up-front keys it already holds, from one `hasMany`. `found` is
|
|
114
122
|
* null when the probe failed.
|
|
115
123
|
*/
|
|
116
|
-
private presence:
|
|
124
|
+
private presence:
|
|
125
|
+
| Promise<{ asked: Set<string>; found: Map<string, ArtifactInfo> | null } | null>
|
|
126
|
+
| undefined
|
|
117
127
|
|
|
118
128
|
private readonly uploadQueue: Array<() => Promise<void>> = []
|
|
119
129
|
private activeUploads = 0
|
|
@@ -136,13 +146,11 @@ export class RemoteLane {
|
|
|
136
146
|
constructor(
|
|
137
147
|
resolved: ResolvedRemote,
|
|
138
148
|
private readonly traffic: RemoteTraffic,
|
|
139
|
-
/** The wire as this run asks it: a branch scope's view of `resolved.remote`. */
|
|
140
|
-
remote: RemoteCacheLayer = resolved.remote,
|
|
141
149
|
/** Background PUTs at once; excess uploads queue and drain FIFO. */
|
|
142
150
|
private readonly uploadConcurrency: number = networkConcurrency(),
|
|
143
151
|
) {
|
|
144
152
|
this.resolved = resolved
|
|
145
|
-
this.remote = remote
|
|
153
|
+
this.remote = resolved.remote
|
|
146
154
|
this.onError = resolved.onError
|
|
147
155
|
this.endpoint = printableEndpoint(resolved.remote.endpoint)
|
|
148
156
|
}
|
|
@@ -159,15 +167,28 @@ export class RemoteLane {
|
|
|
159
167
|
* Existence probe; a failure is a miss. A key a `hasMany` this run
|
|
160
168
|
* answered for is not asked again: `--dry` batches its keys first.
|
|
161
169
|
*/
|
|
162
|
-
async has(hash: string): Promise<
|
|
163
|
-
|
|
164
|
-
if (
|
|
170
|
+
async has(hash: string): Promise<ArtifactInfo | null> {
|
|
171
|
+
const held = this.held.get(hash)
|
|
172
|
+
if (held !== undefined) return held
|
|
173
|
+
if (this.absent.has(hash)) return null
|
|
174
|
+
let found: unknown
|
|
165
175
|
try {
|
|
166
|
-
|
|
176
|
+
found = await this.remote.has(hash)
|
|
167
177
|
} catch (err) {
|
|
168
178
|
this.report('probe', hash, err)
|
|
169
|
-
return
|
|
179
|
+
return null
|
|
170
180
|
}
|
|
181
|
+
if (found === null || found === undefined) return null
|
|
182
|
+
const info = artifactInfo(found)
|
|
183
|
+
if (info === null)
|
|
184
|
+
this.report(
|
|
185
|
+
'probe',
|
|
186
|
+
hash,
|
|
187
|
+
invalidRemoteResult(
|
|
188
|
+
`has() resolved ${describeValue(found)} (expected { size, durationMs } or null)`,
|
|
189
|
+
),
|
|
190
|
+
)
|
|
191
|
+
return info
|
|
171
192
|
}
|
|
172
193
|
|
|
173
194
|
/**
|
|
@@ -175,7 +196,7 @@ export class RemoteLane {
|
|
|
175
196
|
* rest as absent; `null` when this remote cannot batch or the probe
|
|
176
197
|
* failed, which the caller reads as "no batch info".
|
|
177
198
|
*/
|
|
178
|
-
async hasMany(hashes: readonly string[]): Promise<
|
|
199
|
+
async hasMany(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
|
|
179
200
|
if (this.remote.hasMany === undefined) return null
|
|
180
201
|
let found: unknown
|
|
181
202
|
try {
|
|
@@ -185,19 +206,26 @@ export class RemoteLane {
|
|
|
185
206
|
return null
|
|
186
207
|
}
|
|
187
208
|
if (found === null || found === undefined) return null
|
|
188
|
-
if (!(found instanceof
|
|
209
|
+
if (!(found instanceof Map)) {
|
|
189
210
|
this.report(
|
|
190
211
|
'probe',
|
|
191
212
|
batchOf(hashes),
|
|
192
213
|
invalidRemoteResult(
|
|
193
|
-
`hasMany() resolved ${describeValue(found)} (expected a
|
|
214
|
+
`hasMany() resolved ${describeValue(found)} (expected a Map of the hashes present, or null)`,
|
|
194
215
|
),
|
|
195
216
|
)
|
|
196
217
|
return null
|
|
197
218
|
}
|
|
198
|
-
const
|
|
199
|
-
for (const h of hashes)
|
|
200
|
-
|
|
219
|
+
const out = new Map<string, ArtifactInfo>()
|
|
220
|
+
for (const h of hashes) {
|
|
221
|
+
const info = found.has(h) ? (artifactInfo(found.get(h)) ?? {}) : undefined
|
|
222
|
+
if (info === undefined) this.absent.add(h)
|
|
223
|
+
else {
|
|
224
|
+
this.held.set(h, info)
|
|
225
|
+
out.set(h, info)
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
return out
|
|
201
229
|
}
|
|
202
230
|
|
|
203
231
|
/** The artifact's body, or null on a miss, a known absence, a failure or a malformed answer. */
|
|
@@ -253,7 +281,7 @@ export class RemoteLane {
|
|
|
253
281
|
const probed = await this.presence
|
|
254
282
|
if (probed !== undefined && probed !== null && probed.found !== null && probed.asked.has(hash))
|
|
255
283
|
return probed.found.has(hash)
|
|
256
|
-
return this.has(hash)
|
|
284
|
+
return (await this.has(hash)) !== null
|
|
257
285
|
}
|
|
258
286
|
|
|
259
287
|
/** Queue a PUT in this remote's bounded pool; `body` is asked for when the job starts. */
|
|
@@ -270,6 +298,7 @@ export class RemoteLane {
|
|
|
270
298
|
let release: (() => Promise<void>) | undefined
|
|
271
299
|
try {
|
|
272
300
|
if (this.stopped) return
|
|
301
|
+
if (this.held.has(hash)) return
|
|
273
302
|
if (!this.read && (await this.holds(hash))) return
|
|
274
303
|
if (this.stopped) return
|
|
275
304
|
const start = performance.now()
|
|
@@ -1,87 +1,10 @@
|
|
|
1
|
-
// A
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// reads the scope's key, every other task the default key, and a scoped run
|
|
5
|
-
// writes every remote under the scope's key. An artifact is packed under its
|
|
6
|
-
// task key wherever it is stored, so a remote entry ingests under that key.
|
|
1
|
+
// A scope keys only what plugins store themselves (ctx.cache, run records):
|
|
2
|
+
// task keys reach every remote plain, and a remote decides which branch a
|
|
3
|
+
// read may see (VZN, 2026-10-20).
|
|
7
4
|
|
|
8
5
|
import { xxh3hex } from '../util/index.js'
|
|
9
|
-
import type { RemoteCacheLayer, RemoteKey } from './layered-cache.js'
|
|
10
6
|
|
|
11
7
|
/** The key a scope stores `hash` under: task-key shaped, 16 hex, so every wire stores it unchanged. */
|
|
12
8
|
export function scopedKey(scope: string, hash: string): string {
|
|
13
9
|
return xxh3hex(`vx-cache-scope\0${scope}\0${hash}`)
|
|
14
10
|
}
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* Which remote key each task asks for, in a run with a scope. The routes
|
|
18
|
-
* come from the branch's diff, which only a remote call awaits: a run the
|
|
19
|
-
* local cache answers never waits on git.
|
|
20
|
-
*/
|
|
21
|
-
export class BranchScope {
|
|
22
|
-
/** Finds the tasks that read the scope, at the first remote call; null: every task does. */
|
|
23
|
-
private route: (() => Promise<ReadonlySet<string>>) | null = null
|
|
24
|
-
private routes: Promise<ReadonlySet<string>> | null = null
|
|
25
|
-
/** Task key → its task. */
|
|
26
|
-
private readonly taskOf = new Map<string, string>()
|
|
27
|
-
|
|
28
|
-
constructor(readonly scope: string) {}
|
|
29
|
-
|
|
30
|
-
/** Only the tasks `ids` finds read the scope; set before any remote call. */
|
|
31
|
-
routeTasks(ids: () => Promise<ReadonlySet<string>>): void {
|
|
32
|
-
this.route = ids
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
noteKey(taskId: string, hash: string): void {
|
|
36
|
-
this.taskOf.set(hash, taskId)
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
/** The remote key `hash` is read under, and the scope it is read in (none: the default key). */
|
|
40
|
-
private async readKey(hash: string): Promise<{ key: string; at: RemoteKey }> {
|
|
41
|
-
const id = this.taskOf.get(hash)
|
|
42
|
-
if (this.route !== null) this.routes ??= this.route()
|
|
43
|
-
const tasks = this.routes === null ? null : await this.routes
|
|
44
|
-
return tasks !== null && id !== undefined && !tasks.has(id)
|
|
45
|
-
? { key: hash, at: {} }
|
|
46
|
-
: { key: scopedKey(this.scope, hash), at: { scope: this.scope } }
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
/**
|
|
50
|
-
* `inner` as this run asks it: each task key under its route's key; every
|
|
51
|
-
* write under the scope's. Each call also names its scope, for a backend
|
|
52
|
-
* that keeps one store per scope; no wire vx ships sends it.
|
|
53
|
-
*/
|
|
54
|
-
remote(inner: RemoteCacheLayer): RemoteCacheLayer {
|
|
55
|
-
const read = (h: string) => this.readKey(h)
|
|
56
|
-
const at: RemoteKey = { scope: this.scope }
|
|
57
|
-
const hasMany = inner.hasMany
|
|
58
|
-
return {
|
|
59
|
-
...(inner.endpoint !== undefined ? { endpoint: inner.endpoint } : {}),
|
|
60
|
-
...(inner.open !== undefined ? { open: (ctx) => inner.open!(ctx) } : {}),
|
|
61
|
-
has: async (h) => {
|
|
62
|
-
const r = await read(h)
|
|
63
|
-
return inner.has(r.key, r.at)
|
|
64
|
-
},
|
|
65
|
-
...(hasMany !== undefined
|
|
66
|
-
? {
|
|
67
|
-
hasMany: async (hashes) => {
|
|
68
|
-
const reads = await Promise.all(hashes.map(read))
|
|
69
|
-
const keys = reads.map((r) => r.key)
|
|
70
|
-
const found = await hasMany.call(
|
|
71
|
-
inner,
|
|
72
|
-
keys,
|
|
73
|
-
reads.map((r) => r.at),
|
|
74
|
-
)
|
|
75
|
-
if (!(found instanceof Set)) return found
|
|
76
|
-
return new Set(hashes.filter((_, i) => found.has(keys[i]!)))
|
|
77
|
-
},
|
|
78
|
-
}
|
|
79
|
-
: {}),
|
|
80
|
-
get: async (h) => {
|
|
81
|
-
const r = await read(h)
|
|
82
|
-
return inner.get(r.key, r.at)
|
|
83
|
-
},
|
|
84
|
-
put: (h, body, meta) => inner.put(scopedKey(this.scope, h), body, { ...meta, ...at }),
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
}
|
package/src/cli/help.ts
CHANGED
|
@@ -264,10 +264,10 @@ export function helpText(pluginCommands: readonly string[] = []): string {
|
|
|
264
264
|
' --cache-dir <path> Read the cache a run with the same flag uses.',
|
|
265
265
|
'',
|
|
266
266
|
'Migration:',
|
|
267
|
-
' vx init
|
|
268
|
-
'
|
|
269
|
-
'
|
|
270
|
-
'
|
|
267
|
+
' vx init Install vx globally and as the root devDependency, then',
|
|
268
|
+
' write .vx/workspace.ts and an empty root vx.config.ts. In a',
|
|
269
|
+
' Turbo or Nx repo: runs @vzn/vx-migrate, which maps turbo.json',
|
|
270
|
+
' or the Nx graph to vx (asking native or keep).',
|
|
271
271
|
' --dry Print the generated files instead of writing them.',
|
|
272
272
|
' --format json With --dry: the plan as one JSON document (schemas/init.json).',
|
|
273
273
|
' --force Overwrite existing vx.config.* files.',
|