@vzn/vx 0.0.642 → 0.0.644
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/schemas/info.json +0 -11
- package/skills/vx/SKILL.md +1 -1
- package/src/cache/archive.ts +153 -2
- package/src/cache/cache.ts +417 -183
- package/src/cache/file-hashes.ts +2 -14
- package/src/cache/fold-memo.ts +7 -3
- package/src/cache/index.ts +10 -2
- package/src/cache/key-fold.ts +4 -1
- package/src/cache/layer.ts +11 -4
- package/src/cache/layered-cache.ts +82 -8
- package/src/cache/plugin-cache.ts +51 -43
- package/src/cache/remote-lane.ts +18 -13
- package/src/cache/run-store.ts +4 -1
- package/src/cache/scoped-cache.ts +74 -239
- package/src/cache/tar-stream.ts +12 -7
- package/src/cli/format.ts +1 -2
- package/src/cli/help.ts +8 -4
- package/src/cli/info.ts +2 -19
- package/src/cli/plugin-commands.ts +11 -6
- package/src/cli/run.ts +7 -12
- package/src/cli/select.ts +7 -9
- package/src/cli/upgrade.ts +38 -8
- package/src/cli/workspace-config.ts +8 -2
- package/src/config.ts +22 -26
- package/src/exec/sandbox-paths.ts +2 -2
- package/src/graph/scheduler.ts +3 -1
- package/src/graph/task-graph.ts +15 -11
- package/src/index.ts +5 -1
- package/src/orchestrator/affected-tasks.ts +23 -2
- package/src/orchestrator/doctor.ts +0 -17
- package/src/orchestrator/execute-task.ts +1 -1
- package/src/orchestrator/index.ts +4 -0
- package/src/orchestrator/options.ts +15 -1
- package/src/orchestrator/plugin-host.ts +11 -4
- package/src/orchestrator/plugin.ts +23 -2
- package/src/orchestrator/prepare.ts +81 -18
- package/src/orchestrator/projects.ts +26 -5
- package/src/orchestrator/remote-prefetch.ts +11 -4
- package/src/orchestrator/run-context.ts +85 -0
- package/src/orchestrator/run-settings.ts +3 -0
- package/src/orchestrator/run.ts +9 -26
- package/src/orchestrator/task-hash.ts +6 -0
- package/src/report/logger.ts +0 -4
- package/src/report/summary.ts +3 -11
- package/src/settings.ts +54 -25
- package/src/util/index.ts +1 -0
- package/src/util/paths.ts +42 -33
- package/src/util/size.ts +4 -5
- package/src/workspace/affected.ts +120 -63
- package/src/workspace/config-schema.ts +44 -52
- package/src/workspace/index.ts +3 -0
package/src/cache/file-hashes.ts
CHANGED
|
@@ -14,9 +14,6 @@ import { fileIdentity, repoFacts } from './git-inputs.js'
|
|
|
14
14
|
import { FILE_HASH_RACY_MS, isIndexFull, racyWindowMs } from './layer.js'
|
|
15
15
|
import { isPermissionError, UserError } from '../util/index.js'
|
|
16
16
|
|
|
17
|
-
const FILE_HASHES_SWEPT_AT = 'file_hashes_swept_at'
|
|
18
|
-
const DAY_MS = 24 * 60 * 60 * 1000
|
|
19
|
-
|
|
20
17
|
export class FileHashStore {
|
|
21
18
|
private readonly selectFileHash: ReturnType<Database['prepare']>
|
|
22
19
|
private readonly upsertFileHash: ReturnType<Database['prepare']>
|
|
@@ -62,22 +59,13 @@ export class FileHashStore {
|
|
|
62
59
|
* path) keeps its row forever otherwise, and `cache.db` grows with every
|
|
63
60
|
* path ever hashed. A hit does not refresh `seen_at` (a write per file
|
|
64
61
|
* per run), so a file unchanged for the window loses its row too and is
|
|
65
|
-
* read once more; a dropped row is only a memo miss.
|
|
66
|
-
* 8 ms at 100,000 rows, so it runs once a day, on its own clock in
|
|
67
|
-
* `schema_meta` (item 1082).
|
|
62
|
+
* read once more; a dropped row is only a memo miss.
|
|
68
63
|
*/
|
|
69
|
-
pruneOlderThan(cutoff: number
|
|
70
|
-
const last = this.db
|
|
71
|
-
.prepare('SELECT value FROM schema_meta WHERE key = ?')
|
|
72
|
-
.get(FILE_HASHES_SWEPT_AT) as { value: string } | null
|
|
73
|
-
if (last !== null && now - Number(last.value) < DAY_MS) return
|
|
64
|
+
pruneOlderThan(cutoff: number): void {
|
|
74
65
|
this.db.transaction(() => {
|
|
75
66
|
this.db.prepare('DELETE FROM file_hashes WHERE seen_at < ?').run(cutoff)
|
|
76
67
|
this.db.prepare('DELETE FROM blob_sizes WHERE seen_at < ?').run(cutoff)
|
|
77
68
|
this.db.prepare('DELETE FROM blob_verdicts WHERE seen_at < ?').run(cutoff)
|
|
78
|
-
this.db
|
|
79
|
-
.prepare('INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)')
|
|
80
|
-
.run(FILE_HASHES_SWEPT_AT, String(now))
|
|
81
69
|
})()
|
|
82
70
|
}
|
|
83
71
|
|
package/src/cache/fold-memo.ts
CHANGED
|
@@ -43,16 +43,20 @@ export class FoldMemo {
|
|
|
43
43
|
if (this.write) this.dirty.set(taskId, { probe, key })
|
|
44
44
|
}
|
|
45
45
|
|
|
46
|
-
/** Write the folds this run made
|
|
47
|
-
flush(
|
|
46
|
+
/** Write the folds this run made. */
|
|
47
|
+
flush(now: number = Date.now()): void {
|
|
48
48
|
if (this.dirty.size === 0) return
|
|
49
49
|
const upsert = this.db.prepare(
|
|
50
50
|
'INSERT OR REPLACE INTO fold_memo(task_id, probe, key, seen_at) VALUES (?, ?, ?, ?)',
|
|
51
51
|
)
|
|
52
52
|
this.db.transaction(() => {
|
|
53
53
|
for (const [taskId, r] of this.dirty) upsert.run(taskId, r.probe, r.key, now)
|
|
54
|
-
this.db.prepare('DELETE FROM fold_memo WHERE seen_at < ?').run(cutoff)
|
|
55
54
|
})()
|
|
56
55
|
this.dirty.clear()
|
|
57
56
|
}
|
|
57
|
+
|
|
58
|
+
/** Rows no run has rewritten since `cutoff`. */
|
|
59
|
+
prune(cutoff: number): void {
|
|
60
|
+
this.db.prepare('DELETE FROM fold_memo WHERE seen_at < ?').run(cutoff)
|
|
61
|
+
}
|
|
58
62
|
}
|
package/src/cache/index.ts
CHANGED
|
@@ -8,6 +8,8 @@ export {
|
|
|
8
8
|
applyCacheFlags,
|
|
9
9
|
ArtifactVanishedError,
|
|
10
10
|
Cache,
|
|
11
|
+
DEFAULT_STORE,
|
|
12
|
+
type StoreOptions,
|
|
11
13
|
type CacheFlags,
|
|
12
14
|
type CacheEntry,
|
|
13
15
|
type CacheKeyInput,
|
|
@@ -79,9 +81,15 @@ export {
|
|
|
79
81
|
type GitEnumeration,
|
|
80
82
|
type BlobSizeMemo,
|
|
81
83
|
} from './git-inputs.js'
|
|
82
|
-
export {
|
|
84
|
+
export {
|
|
85
|
+
LayeredCache,
|
|
86
|
+
type RemoteCacheLayer,
|
|
87
|
+
type RemoteKey,
|
|
88
|
+
type RemoteOpenContext,
|
|
89
|
+
} from './layered-cache.js'
|
|
90
|
+
export { networkConcurrency } from './remote-lane.js'
|
|
83
91
|
export type { ResolvedRemote } from './remote-lane.js'
|
|
84
|
-
export {
|
|
92
|
+
export { BranchScope } from './scoped-cache.js'
|
|
85
93
|
export {
|
|
86
94
|
absentPluginCache,
|
|
87
95
|
pluginCache,
|
package/src/cache/key-fold.ts
CHANGED
|
@@ -117,7 +117,10 @@ import type { CacheKeyInput } from './layer.js'
|
|
|
117
117
|
// v46: vx's names are reserved only at the project root; a v45 artifact
|
|
118
118
|
// left out a build's own `dist/.git/**`, `dist/.vx/**` and nested
|
|
119
119
|
// `vx.lock.json` (Growth's parity, 2026-10-17).
|
|
120
|
-
|
|
120
|
+
// v47: the local store keeps plain files (`cache.store`).
|
|
121
|
+
// v48: a task glob is standard again (M, 2026-10-18): `app/[id]/page.js` is
|
|
122
|
+
// a class, and a v47 entry under its unchanged key holds the literal route.
|
|
123
|
+
export const CACHE_VERSION = 'vx-cache-v48'
|
|
121
124
|
|
|
122
125
|
/**
|
|
123
126
|
* The digest a key folds for an input gone between its enumeration and its
|
package/src/cache/layer.ts
CHANGED
|
@@ -493,8 +493,8 @@ export class CorruptArtifactError extends Error {
|
|
|
493
493
|
|
|
494
494
|
/**
|
|
495
495
|
* `restoreOutputs` found no artifact where the probe found one: it was
|
|
496
|
-
* removed in between — a `vx cache clean` in another shell, or
|
|
497
|
-
*
|
|
496
|
+
* removed in between — a `vx cache clean` in another shell, or one for
|
|
497
|
+
* another workspace sharing the cache directory. The
|
|
498
498
|
* bytes are not wrong, they are gone, so the entry is a MISS and the task
|
|
499
499
|
* runs (execute-task.ts); nothing about the cache needs throwing away.
|
|
500
500
|
* Also thrown, with its `reason`, for an artifact whose bytes ARE wrong
|
|
@@ -510,8 +510,7 @@ export class ArtifactVanishedError extends Error {
|
|
|
510
510
|
cause?: unknown,
|
|
511
511
|
) {
|
|
512
512
|
super(
|
|
513
|
-
reason ??
|
|
514
|
-
`its cache artifact ${hash} vanished before the restore (a prune or another run's retention)`,
|
|
513
|
+
reason ?? `its cache artifact ${hash} vanished before the restore (a vx cache clean)`,
|
|
515
514
|
cause === undefined ? undefined : { cause },
|
|
516
515
|
)
|
|
517
516
|
this.name = 'ArtifactVanishedError'
|
|
@@ -576,6 +575,14 @@ export interface CacheLayer {
|
|
|
576
575
|
* `hasRemote`.
|
|
577
576
|
*/
|
|
578
577
|
probeBeforeUpload?(hashes: Promise<readonly string[]>): void
|
|
578
|
+
/**
|
|
579
|
+
* The run's batch probe starts when `started` settles: until then a pull
|
|
580
|
+
* waits, so a lookup that lands first does not GET a key the probe is
|
|
581
|
+
* about to call absent. Only meaningful with `hasRemote`.
|
|
582
|
+
*/
|
|
583
|
+
probeFirst?(started: Promise<void>): void
|
|
584
|
+
/** `taskId`'s key is `hash`: a scoped run picks each remote key by its task (`BranchScope`). */
|
|
585
|
+
noteKey?(taskId: string, hash: string): void
|
|
579
586
|
/**
|
|
580
587
|
* Resolve once every background write-through upload settles. The
|
|
581
588
|
* orchestrator awaits it before `close()` — an upload reading layer state
|
|
@@ -36,6 +36,7 @@ import type {
|
|
|
36
36
|
} from './cache.js'
|
|
37
37
|
import type { OutputDirRow } from './cache.js'
|
|
38
38
|
import { RemoteLane, type ResolvedRemote } from './remote-lane.js'
|
|
39
|
+
import type { BranchScope } from './scoped-cache.js'
|
|
39
40
|
import type { PluginSecrets } from '../util/index.js'
|
|
40
41
|
|
|
41
42
|
const STOPPED: unique symbol = Symbol('stopped')
|
|
@@ -75,6 +76,16 @@ export interface RemoteOpenContext extends PluginSecrets {
|
|
|
75
76
|
warn(message: string): void
|
|
76
77
|
}
|
|
77
78
|
|
|
79
|
+
/**
|
|
80
|
+
* Where a remote call's key lives: `scope` names the branch scope a scoped
|
|
81
|
+
* key was derived in (`scopedKey`), absent for a task's own key. The key
|
|
82
|
+
* alone is enough; a backend that keeps one store per scope may use it to
|
|
83
|
+
* go straight to that store. turboCache(), nxCache() and vx-reapi ignore it.
|
|
84
|
+
*/
|
|
85
|
+
export interface RemoteKey {
|
|
86
|
+
readonly scope?: string
|
|
87
|
+
}
|
|
88
|
+
|
|
78
89
|
/**
|
|
79
90
|
* What a remote cache layer must provide — THE plugin seam for remote
|
|
80
91
|
* caching. Core ships no wire client; a plugin's `cache` hook returns one
|
|
@@ -112,16 +123,16 @@ export interface RemoteCacheLayer {
|
|
|
112
123
|
*/
|
|
113
124
|
open?(ctx: RemoteOpenContext): Promise<void>
|
|
114
125
|
/** Existence probe: `--dry` and `--force` ask it per key when the remote cannot batch. */
|
|
115
|
-
has(hash: string): Promise<boolean>
|
|
126
|
+
has(hash: string, at?: RemoteKey): Promise<boolean>
|
|
116
127
|
/**
|
|
117
128
|
* Optional batch existence probe: given N hashes, return the subset stored
|
|
118
129
|
* remotely in ONE round-trip. Its presence says the wire can batch: a run
|
|
119
130
|
* asks it once for its keys, `--dry` before its per-key answers, and
|
|
120
131
|
* `--force` in the background to skip uploads of held keys. Without it a
|
|
121
132
|
* run GETs each key directly (no `has`). Never throws for control flow:
|
|
122
|
-
* `null` means "no batch info; use per-hash".
|
|
133
|
+
* `null` means "no batch info; use per-hash". `at[i]` is `hashes[i]`'s.
|
|
123
134
|
*/
|
|
124
|
-
hasMany?(hashes: readonly string[]): Promise<Set<string> | null>
|
|
135
|
+
hasMany?(hashes: readonly string[], at?: readonly RemoteKey[]): Promise<Set<string> | null>
|
|
125
136
|
/**
|
|
126
137
|
* Fetch an artifact; `null` = miss. `body` is read once, by core,
|
|
127
138
|
* straight into the local cache — an HTTP wire returns its `fetch`
|
|
@@ -131,7 +142,10 @@ export interface RemoteCacheLayer {
|
|
|
131
142
|
* nothing and the read is a miss, so a layer can check bytes as they pass
|
|
132
143
|
* (a digest, a signature) and refuse at the end.
|
|
133
144
|
*/
|
|
134
|
-
get(
|
|
145
|
+
get(
|
|
146
|
+
hash: string,
|
|
147
|
+
at?: RemoteKey,
|
|
148
|
+
): Promise<{ body: Blob | Response; durationMs: number | undefined } | null>
|
|
135
149
|
/**
|
|
136
150
|
* Store an artifact (fire-and-forget from LayeredCache's perspective).
|
|
137
151
|
* `body` is file-backed (`Bun.file`) when the local store holds the
|
|
@@ -140,7 +154,7 @@ export interface RemoteCacheLayer {
|
|
|
140
154
|
* key the remote holds keeps the stored bytes. Core never asks `has`
|
|
141
155
|
* first; a wire that would overwrite checks inside.
|
|
142
156
|
*/
|
|
143
|
-
put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void>
|
|
157
|
+
put(hash: string, body: Blob, meta: { durationMs: number } & RemoteKey): Promise<void>
|
|
144
158
|
}
|
|
145
159
|
|
|
146
160
|
export class LayeredCache implements CacheLayer {
|
|
@@ -161,6 +175,17 @@ export class LayeredCache implements CacheLayer {
|
|
|
161
175
|
*/
|
|
162
176
|
private readonly inflight = new Map<string, Promise<boolean>>()
|
|
163
177
|
|
|
178
|
+
/**
|
|
179
|
+
* Hashes a batch probe is asking about, each resolving whether a pull may
|
|
180
|
+
* still find it (false: no remote holds it). A `get` that lands while the
|
|
181
|
+
* probe is out waits for its answer instead of spending a GET on a key
|
|
182
|
+
* the probe is about to call absent.
|
|
183
|
+
*/
|
|
184
|
+
private readonly probing = new Map<string, Promise<boolean>>()
|
|
185
|
+
|
|
186
|
+
/** Settles once the run's batch probe has started (`probeFirst`); a pull waits on it. */
|
|
187
|
+
private probeStarted: Promise<void> | null = null
|
|
188
|
+
|
|
164
189
|
/**
|
|
165
190
|
* Hashes whose local artifact was materialized FROM a remote this run
|
|
166
191
|
* (by `prefetch` or `get`'s read-through). A later `get` finds them as
|
|
@@ -176,8 +201,18 @@ export class LayeredCache implements CacheLayer {
|
|
|
176
201
|
constructor(
|
|
177
202
|
readonly local: Cache,
|
|
178
203
|
remotes: readonly ResolvedRemote[],
|
|
204
|
+
/** The run's branch scope: it picks each remote key; the local store never sees it. */
|
|
205
|
+
private readonly branch?: BranchScope,
|
|
206
|
+
uploadConcurrency?: number,
|
|
179
207
|
) {
|
|
180
|
-
this.lanes = remotes.map(
|
|
208
|
+
this.lanes = remotes.map(
|
|
209
|
+
(r) =>
|
|
210
|
+
new RemoteLane(r, local.traffic, branch?.remote(r.remote) ?? r.remote, uploadConcurrency),
|
|
211
|
+
)
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
noteKey(taskId: string, hash: string): void {
|
|
215
|
+
this.branch?.noteKey(taskId, hash)
|
|
181
216
|
}
|
|
182
217
|
|
|
183
218
|
// Asked, not kept: a remote whose `open` failed leaves the run after this is built.
|
|
@@ -216,6 +251,27 @@ export class LayeredCache implements CacheLayer {
|
|
|
216
251
|
*/
|
|
217
252
|
async remoteHasMany(hashes: readonly string[]): Promise<Set<string> | null> {
|
|
218
253
|
if (this.readable.length === 0) return null
|
|
254
|
+
const answer = this.probeMany(hashes)
|
|
255
|
+
const waiting = hashes.filter((h) => !this.inflight.has(h) && !this.probing.has(h))
|
|
256
|
+
for (const h of waiting) {
|
|
257
|
+
this.probing.set(
|
|
258
|
+
h,
|
|
259
|
+
answer.then(
|
|
260
|
+
(found) => {
|
|
261
|
+
this.probing.delete(h)
|
|
262
|
+
return found === null || found.has(h)
|
|
263
|
+
},
|
|
264
|
+
() => {
|
|
265
|
+
this.probing.delete(h)
|
|
266
|
+
return true
|
|
267
|
+
},
|
|
268
|
+
),
|
|
269
|
+
)
|
|
270
|
+
}
|
|
271
|
+
return answer
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
private async probeMany(hashes: readonly string[]): Promise<Set<string> | null> {
|
|
219
275
|
let out: Set<string> | null = new Set()
|
|
220
276
|
for (const lane of this.readable) {
|
|
221
277
|
const found = await lane.hasMany(hashes)
|
|
@@ -239,6 +295,17 @@ export class LayeredCache implements CacheLayer {
|
|
|
239
295
|
}
|
|
240
296
|
}
|
|
241
297
|
|
|
298
|
+
probeFirst(started: Promise<void>): void {
|
|
299
|
+
const gate = started.then(
|
|
300
|
+
() => {},
|
|
301
|
+
() => {},
|
|
302
|
+
)
|
|
303
|
+
this.probeStarted = gate
|
|
304
|
+
void gate.then(() => {
|
|
305
|
+
if (this.probeStarted === gate) this.probeStarted = null
|
|
306
|
+
})
|
|
307
|
+
}
|
|
308
|
+
|
|
242
309
|
probeBeforeUpload(hashes: Promise<readonly string[]>): void {
|
|
243
310
|
for (const lane of this.lanes) lane.probeBeforeUpload(hashes)
|
|
244
311
|
}
|
|
@@ -309,6 +376,11 @@ export class LayeredCache implements CacheLayer {
|
|
|
309
376
|
private pullFromRemote(hash: string, ctx?: CacheGetContext): Promise<boolean> {
|
|
310
377
|
const existing = this.inflight.get(hash)
|
|
311
378
|
if (existing) return existing
|
|
379
|
+
const probed = this.probing.get(hash)
|
|
380
|
+
if (probed !== undefined)
|
|
381
|
+
return probed.then((found) => (found ? this.pullFromRemote(hash, ctx) : false))
|
|
382
|
+
if (this.probeStarted !== null)
|
|
383
|
+
return this.probeStarted.then(() => this.pullFromRemote(hash, ctx))
|
|
312
384
|
const p = this.doPullFromRemote(hash, ctx)
|
|
313
385
|
this.inflight.set(hash, p)
|
|
314
386
|
return p
|
|
@@ -435,8 +507,10 @@ export class LayeredCache implements CacheLayer {
|
|
|
435
507
|
return
|
|
436
508
|
}
|
|
437
509
|
}
|
|
438
|
-
const body = (): { body: Blob; release?: () => Promise<void> } =>
|
|
439
|
-
packed !== undefined
|
|
510
|
+
const body = (): Promise<{ body: Blob; release?: () => Promise<void> }> =>
|
|
511
|
+
packed !== undefined
|
|
512
|
+
? Promise.resolve({ body: new Blob([packed]) })
|
|
513
|
+
: this.local.pinArtifact(hash)
|
|
440
514
|
for (const lane of writable) lane.upload(hash, durationMs, body, this.local.uploads)
|
|
441
515
|
}
|
|
442
516
|
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
import type { Database } from 'bun:sqlite'
|
|
10
10
|
import { xxh3hex } from '../util/index.js'
|
|
11
|
+
import type { RemoteKey } from './layered-cache.js'
|
|
11
12
|
import type { CachePolicy } from './policy.js'
|
|
12
13
|
import type { ResolvedRemote } from './remote-lane.js'
|
|
13
14
|
import { scopedKey } from './scoped-cache.js'
|
|
@@ -18,17 +19,21 @@ export const PLUGIN_ENTRY_MAX_BYTES = 1024 * 1024
|
|
|
18
19
|
|
|
19
20
|
/** What a plugin's `ctx.cache` offers. Keys are the plugin's own names. */
|
|
20
21
|
export interface PluginCache {
|
|
21
|
-
/**
|
|
22
|
-
|
|
22
|
+
/**
|
|
23
|
+
* The bytes stored under `key`, from the first layer holding them; null
|
|
24
|
+
* when none does. On a scoped run a remote is asked for the branch's
|
|
25
|
+
* entry, or with `scoped` false the base branch's: one request either way.
|
|
26
|
+
*/
|
|
27
|
+
get(key: string, scoped?: boolean): Promise<Uint8Array | null>
|
|
23
28
|
/** Store `bytes` under `key` in every layer the policy writes, where it is not already; empty bytes store nothing. */
|
|
24
29
|
put(key: string, bytes: Uint8Array): Promise<void>
|
|
25
30
|
/** Whether some layer holds `key`; null when no layer could answer. */
|
|
26
|
-
has(key: string): Promise<boolean | null>
|
|
31
|
+
has(key: string, scoped?: boolean): Promise<boolean | null>
|
|
27
32
|
/**
|
|
28
33
|
* The subset of `keys` some layer holds, in one probe per layer; null
|
|
29
34
|
* when a readable remote has no batch probe: ask `get` one at a time.
|
|
30
35
|
*/
|
|
31
|
-
hasMany(keys: readonly string[]): Promise<Set<string> | null>
|
|
36
|
+
hasMany(keys: readonly string[], scoped?: boolean): Promise<Set<string> | null>
|
|
32
37
|
/**
|
|
33
38
|
* What core kept of a run under `name` (a `log` or `outputs` name its
|
|
34
39
|
* `RunReport` hands `teardown`), from the first readable remote holding
|
|
@@ -85,16 +90,21 @@ async function capped(body: unknown, max = PLUGIN_ENTRY_MAX_BYTES): Promise<Uint
|
|
|
85
90
|
}
|
|
86
91
|
|
|
87
92
|
/**
|
|
88
|
-
* `plugin`'s view of the cache.
|
|
89
|
-
*
|
|
93
|
+
* `plugin`'s view of the cache. The local store never knows a scope
|
|
94
|
+
* (owner, 2026-10-17): it keeps every entry under its name's key. A scoped
|
|
95
|
+
* run writes a remote only under the scope's key, and reads one key: the
|
|
96
|
+
* scope's, or with `scoped` false the default (M, 2026-10-18: no request
|
|
97
|
+
* twice).
|
|
90
98
|
*/
|
|
91
99
|
export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache {
|
|
92
100
|
const { db, policy, scope } = host
|
|
93
101
|
const readers = host.remotes.filter((r) => r.read)
|
|
94
102
|
const writers = host.remotes.filter((r) => r.write)
|
|
95
|
-
const
|
|
103
|
+
const at: RemoteKey = scope === undefined ? {} : { scope }
|
|
104
|
+
const atOf = (key: string, name: string): RemoteKey => (key === entryKey(plugin, name) ? {} : at)
|
|
105
|
+
const readKey = (name: string, scoped: boolean): string => {
|
|
96
106
|
const key = entryKey(plugin, name)
|
|
97
|
-
return scope === undefined ?
|
|
107
|
+
return scope === undefined || !scoped ? key : scopedKey(scope, key)
|
|
98
108
|
}
|
|
99
109
|
const writeKey = (name: string): string => {
|
|
100
110
|
const key = entryKey(plugin, name)
|
|
@@ -138,24 +148,21 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
138
148
|
}
|
|
139
149
|
|
|
140
150
|
return {
|
|
141
|
-
async get(name) {
|
|
142
|
-
const
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
}
|
|
151
|
+
async get(name, scoped = true) {
|
|
152
|
+
const local = entryKey(plugin, name)
|
|
153
|
+
const hit = localGet(local)
|
|
154
|
+
if (hit !== null) return hit
|
|
155
|
+
const key = readKey(name, scoped)
|
|
147
156
|
for (const { remote } of readers) {
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
// a failing remote is a miss there
|
|
158
|
-
}
|
|
157
|
+
try {
|
|
158
|
+
const got = await remote.get(key, atOf(key, name))
|
|
159
|
+
if (got === null) continue
|
|
160
|
+
const bytes = await capped(got.body)
|
|
161
|
+
if (bytes === null) continue
|
|
162
|
+
if (policy.localWrite) localPut(local, bytes)
|
|
163
|
+
return bytes
|
|
164
|
+
} catch {
|
|
165
|
+
// a failing remote is a miss there
|
|
159
166
|
}
|
|
160
167
|
}
|
|
161
168
|
return null
|
|
@@ -164,45 +171,46 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
164
171
|
async put(name, bytes) {
|
|
165
172
|
if (bytes.byteLength === 0 || bytes.byteLength > PLUGIN_ENTRY_MAX_BYTES) return
|
|
166
173
|
const key = writeKey(name)
|
|
167
|
-
if (policy.localWrite) localPut(
|
|
174
|
+
if (policy.localWrite) localPut(entryKey(plugin, name), bytes)
|
|
168
175
|
for (const { remote } of writers) {
|
|
169
176
|
try {
|
|
170
|
-
await remote.put(key, new Blob([bytes]), { durationMs: 0 })
|
|
177
|
+
await remote.put(key, new Blob([bytes]), { durationMs: 0, ...at })
|
|
171
178
|
} catch {
|
|
172
179
|
// the other layers still keep it
|
|
173
180
|
}
|
|
174
181
|
}
|
|
175
182
|
},
|
|
176
183
|
|
|
177
|
-
async has(name) {
|
|
178
|
-
|
|
179
|
-
|
|
184
|
+
async has(name, scoped = true) {
|
|
185
|
+
if (localHas([entryKey(plugin, name)]).size > 0) return true
|
|
186
|
+
const key = readKey(name, scoped)
|
|
180
187
|
let answered = policy.localRead
|
|
181
188
|
for (const { remote } of readers) {
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
// no answer from this remote
|
|
188
|
-
}
|
|
189
|
+
try {
|
|
190
|
+
if (await remote.has(key, atOf(key, name))) return true
|
|
191
|
+
answered = true
|
|
192
|
+
} catch {
|
|
193
|
+
// no answer from this remote
|
|
189
194
|
}
|
|
190
195
|
}
|
|
191
196
|
return answered ? false : null
|
|
192
197
|
},
|
|
193
198
|
|
|
194
|
-
async hasMany(names) {
|
|
199
|
+
async hasMany(names, scoped = true) {
|
|
195
200
|
const byKey = new Map<string, string>()
|
|
196
|
-
for (const name of names)
|
|
201
|
+
for (const name of names) byKey.set(readKey(name, scoped), name)
|
|
197
202
|
const keys = [...byKey.keys()]
|
|
198
203
|
const found = new Set<string>()
|
|
199
|
-
for (const key of localHas(
|
|
204
|
+
for (const key of localHas(names.map((n) => entryKey(plugin, n)))) found.add(byKey.get(key)!)
|
|
200
205
|
for (const { remote } of readers) {
|
|
201
206
|
const ask = keys.filter((k) => !found.has(byKey.get(k)!))
|
|
202
207
|
if (ask.length === 0) break
|
|
203
208
|
if (remote.hasMany === undefined) return null
|
|
204
209
|
try {
|
|
205
|
-
const got: unknown = await remote.hasMany(
|
|
210
|
+
const got: unknown = await remote.hasMany(
|
|
211
|
+
ask,
|
|
212
|
+
ask.map((k) => atOf(k, byKey.get(k)!)),
|
|
213
|
+
)
|
|
206
214
|
if (!(got instanceof Set)) return null
|
|
207
215
|
for (const key of got) if (byKey.has(key as string)) found.add(byKey.get(key as string)!)
|
|
208
216
|
} catch {
|
|
@@ -214,9 +222,9 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
214
222
|
|
|
215
223
|
async getRun(name) {
|
|
216
224
|
for (const { remote } of readers) {
|
|
217
|
-
for (const key of runReadKeys(name, scope)) {
|
|
225
|
+
for (const [i, key] of runReadKeys(name, scope).entries()) {
|
|
218
226
|
try {
|
|
219
|
-
const got = await remote.get(key)
|
|
227
|
+
const got = await remote.get(key, i === 0 ? {} : at)
|
|
220
228
|
if (got === null) continue
|
|
221
229
|
const bytes = await capped(got.body, Infinity)
|
|
222
230
|
if (bytes !== null) return bytes
|
package/src/cache/remote-lane.ts
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
// class with the endpoint named. The order across remotes, the local store
|
|
5
5
|
// and provenance are `LayeredCache`'s; nothing here knows another remote.
|
|
6
6
|
|
|
7
|
+
import { machineParallelism } from '../util/index.js'
|
|
7
8
|
import type { RemoteKeyTraffic, RemoteTraffic } from './cache.js'
|
|
8
9
|
import type { RemoteCacheLayer } from './layered-cache.js'
|
|
9
10
|
|
|
@@ -20,11 +21,13 @@ export interface ResolvedRemote {
|
|
|
20
21
|
}
|
|
21
22
|
|
|
22
23
|
/**
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
24
|
+
* Requests at once to a remote, by default: pnpm's network concurrency
|
|
25
|
+
* (Bun's is a fixed 64). A request waits on the network, not a core, so
|
|
26
|
+
* the cap follows cores only past 21 of them.
|
|
26
27
|
*/
|
|
27
|
-
|
|
28
|
+
export function networkConcurrency(): number {
|
|
29
|
+
return Math.min(96, Math.max(machineParallelism() * 3, 64))
|
|
30
|
+
}
|
|
28
31
|
|
|
29
32
|
/** The seam call a failure happened in, as the warning names it. */
|
|
30
33
|
export type RemoteOp = 'probe' | 'download' | 'upload'
|
|
@@ -133,9 +136,13 @@ export class RemoteLane {
|
|
|
133
136
|
constructor(
|
|
134
137
|
resolved: ResolvedRemote,
|
|
135
138
|
private readonly traffic: RemoteTraffic,
|
|
139
|
+
/** The wire as this run asks it: a branch scope's view of `resolved.remote`. */
|
|
140
|
+
remote: RemoteCacheLayer = resolved.remote,
|
|
141
|
+
/** Background PUTs at once; excess uploads queue and drain FIFO. */
|
|
142
|
+
private readonly uploadConcurrency: number = networkConcurrency(),
|
|
136
143
|
) {
|
|
137
144
|
this.resolved = resolved
|
|
138
|
-
this.remote =
|
|
145
|
+
this.remote = remote
|
|
139
146
|
this.onError = resolved.onError
|
|
140
147
|
this.endpoint = printableEndpoint(resolved.remote.endpoint)
|
|
141
148
|
}
|
|
@@ -253,11 +260,10 @@ export class RemoteLane {
|
|
|
253
260
|
upload(
|
|
254
261
|
hash: string,
|
|
255
262
|
durationMs: number,
|
|
256
|
-
body: () => { body: Blob; release?: () => Promise<void> }
|
|
263
|
+
body: () => Promise<{ body: Blob; release?: () => Promise<void> }>,
|
|
257
264
|
tally: { count: number; bytes: number; ms: number; failed: number },
|
|
258
265
|
): void {
|
|
259
266
|
const key = this.trafficOf(hash)
|
|
260
|
-
const counted = this.traffic.aliases.get(hash) ?? hash
|
|
261
267
|
key.uploading++
|
|
262
268
|
this.traffic.inFlight++
|
|
263
269
|
this.uploadQueue.push(async () => {
|
|
@@ -267,7 +273,7 @@ export class RemoteLane {
|
|
|
267
273
|
if (!this.read && (await this.holds(hash))) return
|
|
268
274
|
if (this.stopped) return
|
|
269
275
|
const start = performance.now()
|
|
270
|
-
const got = body()
|
|
276
|
+
const got = await body()
|
|
271
277
|
release = got.release
|
|
272
278
|
await this.remote.put(hash, got.body, { durationMs })
|
|
273
279
|
tally.count++
|
|
@@ -284,7 +290,7 @@ export class RemoteLane {
|
|
|
284
290
|
// Nothing sent and nothing failed (the remote held it, or the run
|
|
285
291
|
// stopped first): no traffic, so the footer counts no upload.
|
|
286
292
|
if (key.uploading === 0 && !key.landed && key.failures.length === 0)
|
|
287
|
-
this.traffic.keys.delete(
|
|
293
|
+
this.traffic.keys.delete(hash)
|
|
288
294
|
}
|
|
289
295
|
})
|
|
290
296
|
this.pump()
|
|
@@ -359,17 +365,16 @@ export class RemoteLane {
|
|
|
359
365
|
}
|
|
360
366
|
|
|
361
367
|
private trafficOf(hash: string): RemoteKeyTraffic {
|
|
362
|
-
|
|
363
|
-
let key = this.traffic.keys.get(counted)
|
|
368
|
+
let key = this.traffic.keys.get(hash)
|
|
364
369
|
if (key === undefined) {
|
|
365
370
|
key = { uploading: 0, landed: false, failures: [] }
|
|
366
|
-
this.traffic.keys.set(
|
|
371
|
+
this.traffic.keys.set(hash, key)
|
|
367
372
|
}
|
|
368
373
|
return key
|
|
369
374
|
}
|
|
370
375
|
|
|
371
376
|
private pump(): void {
|
|
372
|
-
while (this.activeUploads <
|
|
377
|
+
while (this.activeUploads < this.uploadConcurrency && this.uploadQueue.length > 0) {
|
|
373
378
|
const job = this.uploadQueue.shift()!
|
|
374
379
|
this.activeUploads++
|
|
375
380
|
// Jobs never reject (each wraps its PUT in the never-fail guard).
|
package/src/cache/run-store.ts
CHANGED
|
@@ -44,7 +44,10 @@ export async function putRunEntry(
|
|
|
44
44
|
for (const { remote, write } of remotes) {
|
|
45
45
|
if (!write) continue
|
|
46
46
|
try {
|
|
47
|
-
await remote.put(key, new Blob([bytes]), {
|
|
47
|
+
await remote.put(key, new Blob([bytes]), {
|
|
48
|
+
durationMs: 0,
|
|
49
|
+
...(scope === undefined ? {} : { scope }),
|
|
50
|
+
})
|
|
48
51
|
kept = true
|
|
49
52
|
} catch {
|
|
50
53
|
// the other remotes still keep it
|