@vzn/vx 0.0.642 → 0.0.644

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/package.json +5 -5
  2. package/schemas/info.json +0 -11
  3. package/skills/vx/SKILL.md +1 -1
  4. package/src/cache/archive.ts +153 -2
  5. package/src/cache/cache.ts +417 -183
  6. package/src/cache/file-hashes.ts +2 -14
  7. package/src/cache/fold-memo.ts +7 -3
  8. package/src/cache/index.ts +10 -2
  9. package/src/cache/key-fold.ts +4 -1
  10. package/src/cache/layer.ts +11 -4
  11. package/src/cache/layered-cache.ts +82 -8
  12. package/src/cache/plugin-cache.ts +51 -43
  13. package/src/cache/remote-lane.ts +18 -13
  14. package/src/cache/run-store.ts +4 -1
  15. package/src/cache/scoped-cache.ts +74 -239
  16. package/src/cache/tar-stream.ts +12 -7
  17. package/src/cli/format.ts +1 -2
  18. package/src/cli/help.ts +8 -4
  19. package/src/cli/info.ts +2 -19
  20. package/src/cli/plugin-commands.ts +11 -6
  21. package/src/cli/run.ts +7 -12
  22. package/src/cli/select.ts +7 -9
  23. package/src/cli/upgrade.ts +38 -8
  24. package/src/cli/workspace-config.ts +8 -2
  25. package/src/config.ts +22 -26
  26. package/src/exec/sandbox-paths.ts +2 -2
  27. package/src/graph/scheduler.ts +3 -1
  28. package/src/graph/task-graph.ts +15 -11
  29. package/src/index.ts +5 -1
  30. package/src/orchestrator/affected-tasks.ts +23 -2
  31. package/src/orchestrator/doctor.ts +0 -17
  32. package/src/orchestrator/execute-task.ts +1 -1
  33. package/src/orchestrator/index.ts +4 -0
  34. package/src/orchestrator/options.ts +15 -1
  35. package/src/orchestrator/plugin-host.ts +11 -4
  36. package/src/orchestrator/plugin.ts +23 -2
  37. package/src/orchestrator/prepare.ts +81 -18
  38. package/src/orchestrator/projects.ts +26 -5
  39. package/src/orchestrator/remote-prefetch.ts +11 -4
  40. package/src/orchestrator/run-context.ts +85 -0
  41. package/src/orchestrator/run-settings.ts +3 -0
  42. package/src/orchestrator/run.ts +9 -26
  43. package/src/orchestrator/task-hash.ts +6 -0
  44. package/src/report/logger.ts +0 -4
  45. package/src/report/summary.ts +3 -11
  46. package/src/settings.ts +54 -25
  47. package/src/util/index.ts +1 -0
  48. package/src/util/paths.ts +42 -33
  49. package/src/util/size.ts +4 -5
  50. package/src/workspace/affected.ts +120 -63
  51. package/src/workspace/config-schema.ts +44 -52
  52. package/src/workspace/index.ts +3 -0
@@ -14,9 +14,6 @@ import { fileIdentity, repoFacts } from './git-inputs.js'
14
14
  import { FILE_HASH_RACY_MS, isIndexFull, racyWindowMs } from './layer.js'
15
15
  import { isPermissionError, UserError } from '../util/index.js'
16
16
 
17
- const FILE_HASHES_SWEPT_AT = 'file_hashes_swept_at'
18
- const DAY_MS = 24 * 60 * 60 * 1000
19
-
20
17
  export class FileHashStore {
21
18
  private readonly selectFileHash: ReturnType<Database['prepare']>
22
19
  private readonly upsertFileHash: ReturnType<Database['prepare']>
@@ -62,22 +59,13 @@ export class FileHashStore {
62
59
  * path) keeps its row forever otherwise, and `cache.db` grows with every
63
60
  * path ever hashed. A hit does not refresh `seen_at` (a write per file
64
61
  * per run), so a file unchanged for the window loses its row too and is
65
- * read once more; a dropped row is only a memo miss. The scan costs
66
- * 8 ms at 100,000 rows, so it runs once a day, on its own clock in
67
- * `schema_meta` (item 1082).
62
+ * read once more; a dropped row is only a memo miss.
68
63
  */
69
- pruneOlderThan(cutoff: number, now: number = Date.now()): void {
70
- const last = this.db
71
- .prepare('SELECT value FROM schema_meta WHERE key = ?')
72
- .get(FILE_HASHES_SWEPT_AT) as { value: string } | null
73
- if (last !== null && now - Number(last.value) < DAY_MS) return
64
+ pruneOlderThan(cutoff: number): void {
74
65
  this.db.transaction(() => {
75
66
  this.db.prepare('DELETE FROM file_hashes WHERE seen_at < ?').run(cutoff)
76
67
  this.db.prepare('DELETE FROM blob_sizes WHERE seen_at < ?').run(cutoff)
77
68
  this.db.prepare('DELETE FROM blob_verdicts WHERE seen_at < ?').run(cutoff)
78
- this.db
79
- .prepare('INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)')
80
- .run(FILE_HASHES_SWEPT_AT, String(now))
81
69
  })()
82
70
  }
83
71
 
@@ -43,16 +43,20 @@ export class FoldMemo {
43
43
  if (this.write) this.dirty.set(taskId, { probe, key })
44
44
  }
45
45
 
46
- /** Write the folds this run made; rows a run has not rewritten for `cutoff` go. */
47
- flush(cutoff: number, now: number = Date.now()): void {
46
+ /** Write the folds this run made. */
47
+ flush(now: number = Date.now()): void {
48
48
  if (this.dirty.size === 0) return
49
49
  const upsert = this.db.prepare(
50
50
  'INSERT OR REPLACE INTO fold_memo(task_id, probe, key, seen_at) VALUES (?, ?, ?, ?)',
51
51
  )
52
52
  this.db.transaction(() => {
53
53
  for (const [taskId, r] of this.dirty) upsert.run(taskId, r.probe, r.key, now)
54
- this.db.prepare('DELETE FROM fold_memo WHERE seen_at < ?').run(cutoff)
55
54
  })()
56
55
  this.dirty.clear()
57
56
  }
57
+
58
+ /** Rows no run has rewritten since `cutoff`. */
59
+ prune(cutoff: number): void {
60
+ this.db.prepare('DELETE FROM fold_memo WHERE seen_at < ?').run(cutoff)
61
+ }
58
62
  }
@@ -8,6 +8,8 @@ export {
8
8
  applyCacheFlags,
9
9
  ArtifactVanishedError,
10
10
  Cache,
11
+ DEFAULT_STORE,
12
+ type StoreOptions,
11
13
  type CacheFlags,
12
14
  type CacheEntry,
13
15
  type CacheKeyInput,
@@ -79,9 +81,15 @@ export {
79
81
  type GitEnumeration,
80
82
  type BlobSizeMemo,
81
83
  } from './git-inputs.js'
82
- export { LayeredCache, type RemoteCacheLayer, type RemoteOpenContext } from './layered-cache.js'
84
+ export {
85
+ LayeredCache,
86
+ type RemoteCacheLayer,
87
+ type RemoteKey,
88
+ type RemoteOpenContext,
89
+ } from './layered-cache.js'
90
+ export { networkConcurrency } from './remote-lane.js'
83
91
  export type { ResolvedRemote } from './remote-lane.js'
84
- export { ScopedCache } from './scoped-cache.js'
92
+ export { BranchScope } from './scoped-cache.js'
85
93
  export {
86
94
  absentPluginCache,
87
95
  pluginCache,
@@ -117,7 +117,10 @@ import type { CacheKeyInput } from './layer.js'
117
117
  // v46: vx's names are reserved only at the project root; a v45 artifact
118
118
  // left out a build's own `dist/.git/**`, `dist/.vx/**` and nested
119
119
  // `vx.lock.json` (Growth's parity, 2026-10-17).
120
- export const CACHE_VERSION = 'vx-cache-v46'
120
+ // v47: the local store keeps plain files (`cache.store`).
121
+ // v48: a task glob is standard again (M, 2026-10-18): `app/[id]/page.js` is
122
+ // a class, and a v47 entry under its unchanged key holds the literal route.
123
+ export const CACHE_VERSION = 'vx-cache-v48'
121
124
 
122
125
  /**
123
126
  * The digest a key folds for an input gone between its enumeration and its
@@ -493,8 +493,8 @@ export class CorruptArtifactError extends Error {
493
493
 
494
494
  /**
495
495
  * `restoreOutputs` found no artifact where the probe found one: it was
496
- * removed in between — a `vx cache clean` in another shell, or the
497
- * `cacheRetention` of another workspace sharing the cache directory. The
496
+ * removed in between — a `vx cache clean` in another shell, or one for
497
+ * another workspace sharing the cache directory. The
498
498
  * bytes are not wrong, they are gone, so the entry is a MISS and the task
499
499
  * runs (execute-task.ts); nothing about the cache needs throwing away.
500
500
  * Also thrown, with its `reason`, for an artifact whose bytes ARE wrong
@@ -510,8 +510,7 @@ export class ArtifactVanishedError extends Error {
510
510
  cause?: unknown,
511
511
  ) {
512
512
  super(
513
- reason ??
514
- `its cache artifact ${hash} vanished before the restore (a prune or another run's retention)`,
513
+ reason ?? `its cache artifact ${hash} vanished before the restore (a vx cache clean)`,
515
514
  cause === undefined ? undefined : { cause },
516
515
  )
517
516
  this.name = 'ArtifactVanishedError'
@@ -576,6 +575,14 @@ export interface CacheLayer {
576
575
  * `hasRemote`.
577
576
  */
578
577
  probeBeforeUpload?(hashes: Promise<readonly string[]>): void
578
+ /**
579
+ * The run's batch probe starts when `started` settles: until then a pull
580
+ * waits, so a lookup that lands first does not GET a key the probe is
581
+ * about to call absent. Only meaningful with `hasRemote`.
582
+ */
583
+ probeFirst?(started: Promise<void>): void
584
+ /** `taskId`'s key is `hash`: a scoped run picks each remote key by its task (`BranchScope`). */
585
+ noteKey?(taskId: string, hash: string): void
579
586
  /**
580
587
  * Resolve once every background write-through upload settles. The
581
588
  * orchestrator awaits it before `close()` — an upload reading layer state
@@ -36,6 +36,7 @@ import type {
36
36
  } from './cache.js'
37
37
  import type { OutputDirRow } from './cache.js'
38
38
  import { RemoteLane, type ResolvedRemote } from './remote-lane.js'
39
+ import type { BranchScope } from './scoped-cache.js'
39
40
  import type { PluginSecrets } from '../util/index.js'
40
41
 
41
42
  const STOPPED: unique symbol = Symbol('stopped')
@@ -75,6 +76,16 @@ export interface RemoteOpenContext extends PluginSecrets {
75
76
  warn(message: string): void
76
77
  }
77
78
 
79
+ /**
80
+ * Where a remote call's key lives: `scope` names the branch scope a scoped
81
+ * key was derived in (`scopedKey`), absent for a task's own key. The key
82
+ * alone is enough; a backend that keeps one store per scope may use it to
83
+ * go straight to that store. turboCache(), nxCache() and vx-reapi ignore it.
84
+ */
85
+ export interface RemoteKey {
86
+ readonly scope?: string
87
+ }
88
+
78
89
  /**
79
90
  * What a remote cache layer must provide — THE plugin seam for remote
80
91
  * caching. Core ships no wire client; a plugin's `cache` hook returns one
@@ -112,16 +123,16 @@ export interface RemoteCacheLayer {
112
123
  */
113
124
  open?(ctx: RemoteOpenContext): Promise<void>
114
125
  /** Existence probe: `--dry` and `--force` ask it per key when the remote cannot batch. */
115
- has(hash: string): Promise<boolean>
126
+ has(hash: string, at?: RemoteKey): Promise<boolean>
116
127
  /**
117
128
  * Optional batch existence probe: given N hashes, return the subset stored
118
129
  * remotely in ONE round-trip. Its presence says the wire can batch: a run
119
130
  * asks it once for its keys, `--dry` before its per-key answers, and
120
131
  * `--force` in the background to skip uploads of held keys. Without it a
121
132
  * run GETs each key directly (no `has`). Never throws for control flow:
122
- * `null` means "no batch info; use per-hash".
133
+ * `null` means "no batch info; use per-hash". `at[i]` is `hashes[i]`'s.
123
134
  */
124
- hasMany?(hashes: readonly string[]): Promise<Set<string> | null>
135
+ hasMany?(hashes: readonly string[], at?: readonly RemoteKey[]): Promise<Set<string> | null>
125
136
  /**
126
137
  * Fetch an artifact; `null` = miss. `body` is read once, by core,
127
138
  * straight into the local cache — an HTTP wire returns its `fetch`
@@ -131,7 +142,10 @@ export interface RemoteCacheLayer {
131
142
  * nothing and the read is a miss, so a layer can check bytes as they pass
132
143
  * (a digest, a signature) and refuse at the end.
133
144
  */
134
- get(hash: string): Promise<{ body: Blob | Response; durationMs: number | undefined } | null>
145
+ get(
146
+ hash: string,
147
+ at?: RemoteKey,
148
+ ): Promise<{ body: Blob | Response; durationMs: number | undefined } | null>
135
149
  /**
136
150
  * Store an artifact (fire-and-forget from LayeredCache's perspective).
137
151
  * `body` is file-backed (`Bun.file`) when the local store holds the
@@ -140,7 +154,7 @@ export interface RemoteCacheLayer {
140
154
  * key the remote holds keeps the stored bytes. Core never asks `has`
141
155
  * first; a wire that would overwrite checks inside.
142
156
  */
143
- put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void>
157
+ put(hash: string, body: Blob, meta: { durationMs: number } & RemoteKey): Promise<void>
144
158
  }
145
159
 
146
160
  export class LayeredCache implements CacheLayer {
@@ -161,6 +175,17 @@ export class LayeredCache implements CacheLayer {
161
175
  */
162
176
  private readonly inflight = new Map<string, Promise<boolean>>()
163
177
 
178
+ /**
179
+ * Hashes a batch probe is asking about, each resolving whether a pull may
180
+ * still find it (false: no remote holds it). A `get` that lands while the
181
+ * probe is out waits for its answer instead of spending a GET on a key
182
+ * the probe is about to call absent.
183
+ */
184
+ private readonly probing = new Map<string, Promise<boolean>>()
185
+
186
+ /** Settles once the run's batch probe has started (`probeFirst`); a pull waits on it. */
187
+ private probeStarted: Promise<void> | null = null
188
+
164
189
  /**
165
190
  * Hashes whose local artifact was materialized FROM a remote this run
166
191
  * (by `prefetch` or `get`'s read-through). A later `get` finds them as
@@ -176,8 +201,18 @@ export class LayeredCache implements CacheLayer {
176
201
  constructor(
177
202
  readonly local: Cache,
178
203
  remotes: readonly ResolvedRemote[],
204
+ /** The run's branch scope: it picks each remote key; the local store never sees it. */
205
+ private readonly branch?: BranchScope,
206
+ uploadConcurrency?: number,
179
207
  ) {
180
- this.lanes = remotes.map((r) => new RemoteLane(r, local.traffic))
208
+ this.lanes = remotes.map(
209
+ (r) =>
210
+ new RemoteLane(r, local.traffic, branch?.remote(r.remote) ?? r.remote, uploadConcurrency),
211
+ )
212
+ }
213
+
214
+ noteKey(taskId: string, hash: string): void {
215
+ this.branch?.noteKey(taskId, hash)
181
216
  }
182
217
 
183
218
  // Asked, not kept: a remote whose `open` failed leaves the run after this is built.
@@ -216,6 +251,27 @@ export class LayeredCache implements CacheLayer {
216
251
  */
217
252
  async remoteHasMany(hashes: readonly string[]): Promise<Set<string> | null> {
218
253
  if (this.readable.length === 0) return null
254
+ const answer = this.probeMany(hashes)
255
+ const waiting = hashes.filter((h) => !this.inflight.has(h) && !this.probing.has(h))
256
+ for (const h of waiting) {
257
+ this.probing.set(
258
+ h,
259
+ answer.then(
260
+ (found) => {
261
+ this.probing.delete(h)
262
+ return found === null || found.has(h)
263
+ },
264
+ () => {
265
+ this.probing.delete(h)
266
+ return true
267
+ },
268
+ ),
269
+ )
270
+ }
271
+ return answer
272
+ }
273
+
274
+ private async probeMany(hashes: readonly string[]): Promise<Set<string> | null> {
219
275
  let out: Set<string> | null = new Set()
220
276
  for (const lane of this.readable) {
221
277
  const found = await lane.hasMany(hashes)
@@ -239,6 +295,17 @@ export class LayeredCache implements CacheLayer {
239
295
  }
240
296
  }
241
297
 
298
+ probeFirst(started: Promise<void>): void {
299
+ const gate = started.then(
300
+ () => {},
301
+ () => {},
302
+ )
303
+ this.probeStarted = gate
304
+ void gate.then(() => {
305
+ if (this.probeStarted === gate) this.probeStarted = null
306
+ })
307
+ }
308
+
242
309
  probeBeforeUpload(hashes: Promise<readonly string[]>): void {
243
310
  for (const lane of this.lanes) lane.probeBeforeUpload(hashes)
244
311
  }
@@ -309,6 +376,11 @@ export class LayeredCache implements CacheLayer {
309
376
  private pullFromRemote(hash: string, ctx?: CacheGetContext): Promise<boolean> {
310
377
  const existing = this.inflight.get(hash)
311
378
  if (existing) return existing
379
+ const probed = this.probing.get(hash)
380
+ if (probed !== undefined)
381
+ return probed.then((found) => (found ? this.pullFromRemote(hash, ctx) : false))
382
+ if (this.probeStarted !== null)
383
+ return this.probeStarted.then(() => this.pullFromRemote(hash, ctx))
312
384
  const p = this.doPullFromRemote(hash, ctx)
313
385
  this.inflight.set(hash, p)
314
386
  return p
@@ -435,8 +507,10 @@ export class LayeredCache implements CacheLayer {
435
507
  return
436
508
  }
437
509
  }
438
- const body = (): { body: Blob; release?: () => Promise<void> } =>
439
- packed !== undefined ? { body: new Blob([packed]) } : this.local.pinArtifact(hash)
510
+ const body = (): Promise<{ body: Blob; release?: () => Promise<void> }> =>
511
+ packed !== undefined
512
+ ? Promise.resolve({ body: new Blob([packed]) })
513
+ : this.local.pinArtifact(hash)
440
514
  for (const lane of writable) lane.upload(hash, durationMs, body, this.local.uploads)
441
515
  }
442
516
 
@@ -8,6 +8,7 @@
8
8
 
9
9
  import type { Database } from 'bun:sqlite'
10
10
  import { xxh3hex } from '../util/index.js'
11
+ import type { RemoteKey } from './layered-cache.js'
11
12
  import type { CachePolicy } from './policy.js'
12
13
  import type { ResolvedRemote } from './remote-lane.js'
13
14
  import { scopedKey } from './scoped-cache.js'
@@ -18,17 +19,21 @@ export const PLUGIN_ENTRY_MAX_BYTES = 1024 * 1024
18
19
 
19
20
  /** What a plugin's `ctx.cache` offers. Keys are the plugin's own names. */
20
21
  export interface PluginCache {
21
- /** The bytes stored under `key`, from the first layer holding them; null when none does. */
22
- get(key: string): Promise<Uint8Array | null>
22
+ /**
23
+ * The bytes stored under `key`, from the first layer holding them; null
24
+ * when none does. On a scoped run a remote is asked for the branch's
25
+ * entry, or with `scoped` false the base branch's: one request either way.
26
+ */
27
+ get(key: string, scoped?: boolean): Promise<Uint8Array | null>
23
28
  /** Store `bytes` under `key` in every layer the policy writes, where it is not already; empty bytes store nothing. */
24
29
  put(key: string, bytes: Uint8Array): Promise<void>
25
30
  /** Whether some layer holds `key`; null when no layer could answer. */
26
- has(key: string): Promise<boolean | null>
31
+ has(key: string, scoped?: boolean): Promise<boolean | null>
27
32
  /**
28
33
  * The subset of `keys` some layer holds, in one probe per layer; null
29
34
  * when a readable remote has no batch probe: ask `get` one at a time.
30
35
  */
31
- hasMany(keys: readonly string[]): Promise<Set<string> | null>
36
+ hasMany(keys: readonly string[], scoped?: boolean): Promise<Set<string> | null>
32
37
  /**
33
38
  * What core kept of a run under `name` (a `log` or `outputs` name its
34
39
  * `RunReport` hands `teardown`), from the first readable remote holding
@@ -85,16 +90,21 @@ async function capped(body: unknown, max = PLUGIN_ENTRY_MAX_BYTES): Promise<Uint
85
90
  }
86
91
 
87
92
  /**
88
- * `plugin`'s view of the cache. A scoped run reads the default key, then
89
- * the scope's, and writes only the scope's, as tasks do (`ScopedCache`).
93
+ * `plugin`'s view of the cache. The local store never knows a scope
94
+ * (owner, 2026-10-17): it keeps every entry under its name's key. A scoped
95
+ * run writes a remote only under the scope's key, and reads one key: the
96
+ * scope's, or with `scoped` false the default (M, 2026-10-18: no request
97
+ * twice).
90
98
  */
91
99
  export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache {
92
100
  const { db, policy, scope } = host
93
101
  const readers = host.remotes.filter((r) => r.read)
94
102
  const writers = host.remotes.filter((r) => r.write)
95
- const reads = (name: string): string[] => {
103
+ const at: RemoteKey = scope === undefined ? {} : { scope }
104
+ const atOf = (key: string, name: string): RemoteKey => (key === entryKey(plugin, name) ? {} : at)
105
+ const readKey = (name: string, scoped: boolean): string => {
96
106
  const key = entryKey(plugin, name)
97
- return scope === undefined ? [key] : [key, scopedKey(scope, key)]
107
+ return scope === undefined || !scoped ? key : scopedKey(scope, key)
98
108
  }
99
109
  const writeKey = (name: string): string => {
100
110
  const key = entryKey(plugin, name)
@@ -138,24 +148,21 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
138
148
  }
139
149
 
140
150
  return {
141
- async get(name) {
142
- const keys = reads(name)
143
- for (const key of keys) {
144
- const hit = localGet(key)
145
- if (hit !== null) return hit
146
- }
151
+ async get(name, scoped = true) {
152
+ const local = entryKey(plugin, name)
153
+ const hit = localGet(local)
154
+ if (hit !== null) return hit
155
+ const key = readKey(name, scoped)
147
156
  for (const { remote } of readers) {
148
- for (const key of keys) {
149
- try {
150
- const got = await remote.get(key)
151
- if (got === null) continue
152
- const bytes = await capped(got.body)
153
- if (bytes === null) continue
154
- if (policy.localWrite) localPut(key, bytes)
155
- return bytes
156
- } catch {
157
- // a failing remote is a miss there
158
- }
157
+ try {
158
+ const got = await remote.get(key, atOf(key, name))
159
+ if (got === null) continue
160
+ const bytes = await capped(got.body)
161
+ if (bytes === null) continue
162
+ if (policy.localWrite) localPut(local, bytes)
163
+ return bytes
164
+ } catch {
165
+ // a failing remote is a miss there
159
166
  }
160
167
  }
161
168
  return null
@@ -164,45 +171,46 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
164
171
  async put(name, bytes) {
165
172
  if (bytes.byteLength === 0 || bytes.byteLength > PLUGIN_ENTRY_MAX_BYTES) return
166
173
  const key = writeKey(name)
167
- if (policy.localWrite) localPut(key, bytes)
174
+ if (policy.localWrite) localPut(entryKey(plugin, name), bytes)
168
175
  for (const { remote } of writers) {
169
176
  try {
170
- await remote.put(key, new Blob([bytes]), { durationMs: 0 })
177
+ await remote.put(key, new Blob([bytes]), { durationMs: 0, ...at })
171
178
  } catch {
172
179
  // the other layers still keep it
173
180
  }
174
181
  }
175
182
  },
176
183
 
177
- async has(name) {
178
- const keys = reads(name)
179
- if (localHas(keys).size > 0) return true
184
+ async has(name, scoped = true) {
185
+ if (localHas([entryKey(plugin, name)]).size > 0) return true
186
+ const key = readKey(name, scoped)
180
187
  let answered = policy.localRead
181
188
  for (const { remote } of readers) {
182
- for (const key of keys) {
183
- try {
184
- if (await remote.has(key)) return true
185
- answered = true
186
- } catch {
187
- // no answer from this remote
188
- }
189
+ try {
190
+ if (await remote.has(key, atOf(key, name))) return true
191
+ answered = true
192
+ } catch {
193
+ // no answer from this remote
189
194
  }
190
195
  }
191
196
  return answered ? false : null
192
197
  },
193
198
 
194
- async hasMany(names) {
199
+ async hasMany(names, scoped = true) {
195
200
  const byKey = new Map<string, string>()
196
- for (const name of names) for (const key of reads(name)) byKey.set(key, name)
201
+ for (const name of names) byKey.set(readKey(name, scoped), name)
197
202
  const keys = [...byKey.keys()]
198
203
  const found = new Set<string>()
199
- for (const key of localHas(keys)) found.add(byKey.get(key)!)
204
+ for (const key of localHas(names.map((n) => entryKey(plugin, n)))) found.add(byKey.get(key)!)
200
205
  for (const { remote } of readers) {
201
206
  const ask = keys.filter((k) => !found.has(byKey.get(k)!))
202
207
  if (ask.length === 0) break
203
208
  if (remote.hasMany === undefined) return null
204
209
  try {
205
- const got: unknown = await remote.hasMany(ask)
210
+ const got: unknown = await remote.hasMany(
211
+ ask,
212
+ ask.map((k) => atOf(k, byKey.get(k)!)),
213
+ )
206
214
  if (!(got instanceof Set)) return null
207
215
  for (const key of got) if (byKey.has(key as string)) found.add(byKey.get(key as string)!)
208
216
  } catch {
@@ -214,9 +222,9 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
214
222
 
215
223
  async getRun(name) {
216
224
  for (const { remote } of readers) {
217
- for (const key of runReadKeys(name, scope)) {
225
+ for (const [i, key] of runReadKeys(name, scope).entries()) {
218
226
  try {
219
- const got = await remote.get(key)
227
+ const got = await remote.get(key, i === 0 ? {} : at)
220
228
  if (got === null) continue
221
229
  const bytes = await capped(got.body, Infinity)
222
230
  if (bytes !== null) return bytes
@@ -4,6 +4,7 @@
4
4
  // class with the endpoint named. The order across remotes, the local store
5
5
  // and provenance are `LayeredCache`'s; nothing here knows another remote.
6
6
 
7
+ import { machineParallelism } from '../util/index.js'
7
8
  import type { RemoteKeyTraffic, RemoteTraffic } from './cache.js'
8
9
  import type { RemoteCacheLayer } from './layered-cache.js'
9
10
 
@@ -20,11 +21,13 @@ export interface ResolvedRemote {
20
21
  }
21
22
 
22
23
  /**
23
- * Cap on concurrent background PUTs per remote. Keeps a burst of cache
24
- * misses from opening one socket per task; excess uploads queue and drain
25
- * FIFO.
24
+ * Requests at once to a remote, by default: pnpm's network concurrency
25
+ * (Bun's is a fixed 64). A request waits on the network, not a core, so
26
+ * the cap follows cores only past 21 of them.
26
27
  */
27
- const UPLOAD_CONCURRENCY = 4
28
+ export function networkConcurrency(): number {
29
+ return Math.min(96, Math.max(machineParallelism() * 3, 64))
30
+ }
28
31
 
29
32
  /** The seam call a failure happened in, as the warning names it. */
30
33
  export type RemoteOp = 'probe' | 'download' | 'upload'
@@ -133,9 +136,13 @@ export class RemoteLane {
133
136
  constructor(
134
137
  resolved: ResolvedRemote,
135
138
  private readonly traffic: RemoteTraffic,
139
+ /** The wire as this run asks it: a branch scope's view of `resolved.remote`. */
140
+ remote: RemoteCacheLayer = resolved.remote,
141
+ /** Background PUTs at once; excess uploads queue and drain FIFO. */
142
+ private readonly uploadConcurrency: number = networkConcurrency(),
136
143
  ) {
137
144
  this.resolved = resolved
138
- this.remote = resolved.remote
145
+ this.remote = remote
139
146
  this.onError = resolved.onError
140
147
  this.endpoint = printableEndpoint(resolved.remote.endpoint)
141
148
  }
@@ -253,11 +260,10 @@ export class RemoteLane {
253
260
  upload(
254
261
  hash: string,
255
262
  durationMs: number,
256
- body: () => { body: Blob; release?: () => Promise<void> },
263
+ body: () => Promise<{ body: Blob; release?: () => Promise<void> }>,
257
264
  tally: { count: number; bytes: number; ms: number; failed: number },
258
265
  ): void {
259
266
  const key = this.trafficOf(hash)
260
- const counted = this.traffic.aliases.get(hash) ?? hash
261
267
  key.uploading++
262
268
  this.traffic.inFlight++
263
269
  this.uploadQueue.push(async () => {
@@ -267,7 +273,7 @@ export class RemoteLane {
267
273
  if (!this.read && (await this.holds(hash))) return
268
274
  if (this.stopped) return
269
275
  const start = performance.now()
270
- const got = body()
276
+ const got = await body()
271
277
  release = got.release
272
278
  await this.remote.put(hash, got.body, { durationMs })
273
279
  tally.count++
@@ -284,7 +290,7 @@ export class RemoteLane {
284
290
  // Nothing sent and nothing failed (the remote held it, or the run
285
291
  // stopped first): no traffic, so the footer counts no upload.
286
292
  if (key.uploading === 0 && !key.landed && key.failures.length === 0)
287
- this.traffic.keys.delete(counted)
293
+ this.traffic.keys.delete(hash)
288
294
  }
289
295
  })
290
296
  this.pump()
@@ -359,17 +365,16 @@ export class RemoteLane {
359
365
  }
360
366
 
361
367
  private trafficOf(hash: string): RemoteKeyTraffic {
362
- const counted = this.traffic.aliases.get(hash) ?? hash
363
- let key = this.traffic.keys.get(counted)
368
+ let key = this.traffic.keys.get(hash)
364
369
  if (key === undefined) {
365
370
  key = { uploading: 0, landed: false, failures: [] }
366
- this.traffic.keys.set(counted, key)
371
+ this.traffic.keys.set(hash, key)
367
372
  }
368
373
  return key
369
374
  }
370
375
 
371
376
  private pump(): void {
372
- while (this.activeUploads < UPLOAD_CONCURRENCY && this.uploadQueue.length > 0) {
377
+ while (this.activeUploads < this.uploadConcurrency && this.uploadQueue.length > 0) {
373
378
  const job = this.uploadQueue.shift()!
374
379
  this.activeUploads++
375
380
  // Jobs never reject (each wraps its PUT in the never-fail guard).
@@ -44,7 +44,10 @@ export async function putRunEntry(
44
44
  for (const { remote, write } of remotes) {
45
45
  if (!write) continue
46
46
  try {
47
- await remote.put(key, new Blob([bytes]), { durationMs: 0 })
47
+ await remote.put(key, new Blob([bytes]), {
48
+ durationMs: 0,
49
+ ...(scope === undefined ? {} : { scope }),
50
+ })
48
51
  kept = true
49
52
  } catch {
50
53
  // the other remotes still keep it