@vzn/vx 0.0.642 → 0.0.643
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/src/cache/cache.ts +1 -7
- package/src/cache/index.ts +7 -2
- package/src/cache/layer.ts +2 -0
- package/src/cache/layered-cache.ts +28 -6
- package/src/cache/plugin-cache.ts +23 -16
- package/src/cache/remote-lane.ts +6 -6
- package/src/cache/run-store.ts +4 -1
- package/src/cache/scoped-cache.ts +74 -239
- package/src/cli/help.ts +2 -4
- package/src/cli/plugin-commands.ts +11 -6
- package/src/cli/run.ts +4 -12
- package/src/cli/select.ts +7 -9
- package/src/cli/workspace-config.ts +8 -2
- package/src/config.ts +7 -16
- package/src/index.ts +4 -0
- package/src/orchestrator/affected-tasks.ts +23 -2
- package/src/orchestrator/index.ts +4 -0
- package/src/orchestrator/options.ts +9 -1
- package/src/orchestrator/plugin-host.ts +7 -4
- package/src/orchestrator/plugin.ts +20 -2
- package/src/orchestrator/prepare.ts +67 -17
- package/src/orchestrator/projects.ts +19 -5
- package/src/orchestrator/run-context.ts +80 -0
- package/src/orchestrator/run.ts +1 -0
- package/src/orchestrator/task-hash.ts +6 -0
- package/src/report/summary.ts +3 -0
- package/src/settings.ts +11 -11
- package/src/workspace/affected.ts +120 -63
- package/src/workspace/config-schema.ts +44 -16
- package/src/workspace/index.ts +3 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.643",
|
|
4
4
|
"description": "Task orchestration you own: correct, sandboxed, extensible, with a content-addressed build cache.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -21,10 +21,10 @@
|
|
|
21
21
|
"node": ">=18"
|
|
22
22
|
},
|
|
23
23
|
"optionalDependencies": {
|
|
24
|
-
"@vzn/vx-linux-x64": "0.0.
|
|
25
|
-
"@vzn/vx-linux-arm64": "0.0.
|
|
26
|
-
"@vzn/vx-darwin-x64": "0.0.
|
|
27
|
-
"@vzn/vx-darwin-arm64": "0.0.
|
|
24
|
+
"@vzn/vx-linux-x64": "0.0.643",
|
|
25
|
+
"@vzn/vx-linux-arm64": "0.0.643",
|
|
26
|
+
"@vzn/vx-darwin-x64": "0.0.643",
|
|
27
|
+
"@vzn/vx-darwin-arm64": "0.0.643"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"@anthropic-ai/sandbox-runtime": "^0.0.76"
|
package/src/cache/cache.ts
CHANGED
|
@@ -812,12 +812,6 @@ export interface RemoteTraffic {
|
|
|
812
812
|
/** Uploads queued or running across every remote. */
|
|
813
813
|
inFlight: number
|
|
814
814
|
readonly keys: Map<string, RemoteKeyTraffic>
|
|
815
|
-
/**
|
|
816
|
-
* A key a remote is asked under that is not the task's own (a scope's)
|
|
817
|
-
* → the task key it answers for: its traffic counts under the task key,
|
|
818
|
-
* which the footer and the run's rows look up.
|
|
819
|
-
*/
|
|
820
|
-
readonly aliases: Map<string, string>
|
|
821
815
|
}
|
|
822
816
|
|
|
823
817
|
export class Cache implements CacheLayer {
|
|
@@ -830,7 +824,7 @@ export class Cache implements CacheLayer {
|
|
|
830
824
|
*/
|
|
831
825
|
readonly uploads: UploadTally = { count: 0, bytes: 0, ms: 0, failed: 0 }
|
|
832
826
|
/** The same remotes' traffic per key, filled by the same `LayeredCache`. */
|
|
833
|
-
readonly traffic: RemoteTraffic = { inFlight: 0, keys: new Map()
|
|
827
|
+
readonly traffic: RemoteTraffic = { inFlight: 0, keys: new Map() }
|
|
834
828
|
|
|
835
829
|
private readonly db: Database
|
|
836
830
|
private readonly insertEntry: ReturnType<Database['prepare']>
|
package/src/cache/index.ts
CHANGED
|
@@ -79,9 +79,14 @@ export {
|
|
|
79
79
|
type GitEnumeration,
|
|
80
80
|
type BlobSizeMemo,
|
|
81
81
|
} from './git-inputs.js'
|
|
82
|
-
export {
|
|
82
|
+
export {
|
|
83
|
+
LayeredCache,
|
|
84
|
+
type RemoteCacheLayer,
|
|
85
|
+
type RemoteKey,
|
|
86
|
+
type RemoteOpenContext,
|
|
87
|
+
} from './layered-cache.js'
|
|
83
88
|
export type { ResolvedRemote } from './remote-lane.js'
|
|
84
|
-
export {
|
|
89
|
+
export { BranchScope } from './scoped-cache.js'
|
|
85
90
|
export {
|
|
86
91
|
absentPluginCache,
|
|
87
92
|
pluginCache,
|
package/src/cache/layer.ts
CHANGED
|
@@ -576,6 +576,8 @@ export interface CacheLayer {
|
|
|
576
576
|
* `hasRemote`.
|
|
577
577
|
*/
|
|
578
578
|
probeBeforeUpload?(hashes: Promise<readonly string[]>): void
|
|
579
|
+
/** `taskId`'s key is `hash`: a scoped run picks each remote key by its task (`BranchScope`). */
|
|
580
|
+
noteKey?(taskId: string, hash: string): void
|
|
579
581
|
/**
|
|
580
582
|
* Resolve once every background write-through upload settles. The
|
|
581
583
|
* orchestrator awaits it before `close()` — an upload reading layer state
|
|
@@ -36,6 +36,7 @@ import type {
|
|
|
36
36
|
} from './cache.js'
|
|
37
37
|
import type { OutputDirRow } from './cache.js'
|
|
38
38
|
import { RemoteLane, type ResolvedRemote } from './remote-lane.js'
|
|
39
|
+
import type { BranchScope } from './scoped-cache.js'
|
|
39
40
|
import type { PluginSecrets } from '../util/index.js'
|
|
40
41
|
|
|
41
42
|
const STOPPED: unique symbol = Symbol('stopped')
|
|
@@ -75,6 +76,16 @@ export interface RemoteOpenContext extends PluginSecrets {
|
|
|
75
76
|
warn(message: string): void
|
|
76
77
|
}
|
|
77
78
|
|
|
79
|
+
/**
|
|
80
|
+
* Where a remote call's key lives: `scope` names the branch scope a scoped
|
|
81
|
+
* key was derived in (`scopedKey`), absent for a task's own key. The key
|
|
82
|
+
* alone is enough; a backend that keeps one store per scope may use it to
|
|
83
|
+
* go straight to that store. turboCache(), nxCache() and vx-reapi ignore it.
|
|
84
|
+
*/
|
|
85
|
+
export interface RemoteKey {
|
|
86
|
+
readonly scope?: string
|
|
87
|
+
}
|
|
88
|
+
|
|
78
89
|
/**
|
|
79
90
|
* What a remote cache layer must provide — THE plugin seam for remote
|
|
80
91
|
* caching. Core ships no wire client; a plugin's `cache` hook returns one
|
|
@@ -112,16 +123,16 @@ export interface RemoteCacheLayer {
|
|
|
112
123
|
*/
|
|
113
124
|
open?(ctx: RemoteOpenContext): Promise<void>
|
|
114
125
|
/** Existence probe: `--dry` and `--force` ask it per key when the remote cannot batch. */
|
|
115
|
-
has(hash: string): Promise<boolean>
|
|
126
|
+
has(hash: string, at?: RemoteKey): Promise<boolean>
|
|
116
127
|
/**
|
|
117
128
|
* Optional batch existence probe: given N hashes, return the subset stored
|
|
118
129
|
* remotely in ONE round-trip. Its presence says the wire can batch: a run
|
|
119
130
|
* asks it once for its keys, `--dry` before its per-key answers, and
|
|
120
131
|
* `--force` in the background to skip uploads of held keys. Without it a
|
|
121
132
|
* run GETs each key directly (no `has`). Never throws for control flow:
|
|
122
|
-
* `null` means "no batch info; use per-hash".
|
|
133
|
+
* `null` means "no batch info; use per-hash". `at[i]` is `hashes[i]`'s.
|
|
123
134
|
*/
|
|
124
|
-
hasMany?(hashes: readonly string[]): Promise<Set<string> | null>
|
|
135
|
+
hasMany?(hashes: readonly string[], at?: readonly RemoteKey[]): Promise<Set<string> | null>
|
|
125
136
|
/**
|
|
126
137
|
* Fetch an artifact; `null` = miss. `body` is read once, by core,
|
|
127
138
|
* straight into the local cache — an HTTP wire returns its `fetch`
|
|
@@ -131,7 +142,10 @@ export interface RemoteCacheLayer {
|
|
|
131
142
|
* nothing and the read is a miss, so a layer can check bytes as they pass
|
|
132
143
|
* (a digest, a signature) and refuse at the end.
|
|
133
144
|
*/
|
|
134
|
-
get(
|
|
145
|
+
get(
|
|
146
|
+
hash: string,
|
|
147
|
+
at?: RemoteKey,
|
|
148
|
+
): Promise<{ body: Blob | Response; durationMs: number | undefined } | null>
|
|
135
149
|
/**
|
|
136
150
|
* Store an artifact (fire-and-forget from LayeredCache's perspective).
|
|
137
151
|
* `body` is file-backed (`Bun.file`) when the local store holds the
|
|
@@ -140,7 +154,7 @@ export interface RemoteCacheLayer {
|
|
|
140
154
|
* key the remote holds keeps the stored bytes. Core never asks `has`
|
|
141
155
|
* first; a wire that would overwrite checks inside.
|
|
142
156
|
*/
|
|
143
|
-
put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void>
|
|
157
|
+
put(hash: string, body: Blob, meta: { durationMs: number } & RemoteKey): Promise<void>
|
|
144
158
|
}
|
|
145
159
|
|
|
146
160
|
export class LayeredCache implements CacheLayer {
|
|
@@ -176,8 +190,16 @@ export class LayeredCache implements CacheLayer {
|
|
|
176
190
|
constructor(
|
|
177
191
|
readonly local: Cache,
|
|
178
192
|
remotes: readonly ResolvedRemote[],
|
|
193
|
+
/** The run's branch scope: it picks each remote key; the local store never sees it. */
|
|
194
|
+
private readonly branch?: BranchScope,
|
|
179
195
|
) {
|
|
180
|
-
this.lanes = remotes.map(
|
|
196
|
+
this.lanes = remotes.map(
|
|
197
|
+
(r) => new RemoteLane(r, local.traffic, branch?.remote(r.remote) ?? r.remote),
|
|
198
|
+
)
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
noteKey(taskId: string, hash: string): void {
|
|
202
|
+
this.branch?.noteKey(taskId, hash)
|
|
181
203
|
}
|
|
182
204
|
|
|
183
205
|
// Asked, not kept: a remote whose `open` failed leaves the run after this is built.
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
import type { Database } from 'bun:sqlite'
|
|
10
10
|
import { xxh3hex } from '../util/index.js'
|
|
11
|
+
import type { RemoteKey } from './layered-cache.js'
|
|
11
12
|
import type { CachePolicy } from './policy.js'
|
|
12
13
|
import type { ResolvedRemote } from './remote-lane.js'
|
|
13
14
|
import { scopedKey } from './scoped-cache.js'
|
|
@@ -85,13 +86,17 @@ async function capped(body: unknown, max = PLUGIN_ENTRY_MAX_BYTES): Promise<Uint
|
|
|
85
86
|
}
|
|
86
87
|
|
|
87
88
|
/**
|
|
88
|
-
* `plugin`'s view of the cache.
|
|
89
|
-
*
|
|
89
|
+
* `plugin`'s view of the cache. The local store never knows a scope
|
|
90
|
+
* (owner, 2026-10-17): it keeps every entry under its name's key. A scoped
|
|
91
|
+
* run reads a remote under the default key, then the scope's, and writes
|
|
92
|
+
* a remote only under the scope's.
|
|
90
93
|
*/
|
|
91
94
|
export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache {
|
|
92
95
|
const { db, policy, scope } = host
|
|
93
96
|
const readers = host.remotes.filter((r) => r.read)
|
|
94
97
|
const writers = host.remotes.filter((r) => r.write)
|
|
98
|
+
const at: RemoteKey = scope === undefined ? {} : { scope }
|
|
99
|
+
const atOf = (key: string, name: string): RemoteKey => (key === entryKey(plugin, name) ? {} : at)
|
|
95
100
|
const reads = (name: string): string[] => {
|
|
96
101
|
const key = entryKey(plugin, name)
|
|
97
102
|
return scope === undefined ? [key] : [key, scopedKey(scope, key)]
|
|
@@ -139,19 +144,18 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
139
144
|
|
|
140
145
|
return {
|
|
141
146
|
async get(name) {
|
|
147
|
+
const local = entryKey(plugin, name)
|
|
148
|
+
const hit = localGet(local)
|
|
149
|
+
if (hit !== null) return hit
|
|
142
150
|
const keys = reads(name)
|
|
143
|
-
for (const key of keys) {
|
|
144
|
-
const hit = localGet(key)
|
|
145
|
-
if (hit !== null) return hit
|
|
146
|
-
}
|
|
147
151
|
for (const { remote } of readers) {
|
|
148
152
|
for (const key of keys) {
|
|
149
153
|
try {
|
|
150
|
-
const got = await remote.get(key)
|
|
154
|
+
const got = await remote.get(key, atOf(key, name))
|
|
151
155
|
if (got === null) continue
|
|
152
156
|
const bytes = await capped(got.body)
|
|
153
157
|
if (bytes === null) continue
|
|
154
|
-
if (policy.localWrite) localPut(
|
|
158
|
+
if (policy.localWrite) localPut(local, bytes)
|
|
155
159
|
return bytes
|
|
156
160
|
} catch {
|
|
157
161
|
// a failing remote is a miss there
|
|
@@ -164,10 +168,10 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
164
168
|
async put(name, bytes) {
|
|
165
169
|
if (bytes.byteLength === 0 || bytes.byteLength > PLUGIN_ENTRY_MAX_BYTES) return
|
|
166
170
|
const key = writeKey(name)
|
|
167
|
-
if (policy.localWrite) localPut(
|
|
171
|
+
if (policy.localWrite) localPut(entryKey(plugin, name), bytes)
|
|
168
172
|
for (const { remote } of writers) {
|
|
169
173
|
try {
|
|
170
|
-
await remote.put(key, new Blob([bytes]), { durationMs: 0 })
|
|
174
|
+
await remote.put(key, new Blob([bytes]), { durationMs: 0, ...at })
|
|
171
175
|
} catch {
|
|
172
176
|
// the other layers still keep it
|
|
173
177
|
}
|
|
@@ -175,13 +179,13 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
175
179
|
},
|
|
176
180
|
|
|
177
181
|
async has(name) {
|
|
182
|
+
if (localHas([entryKey(plugin, name)]).size > 0) return true
|
|
178
183
|
const keys = reads(name)
|
|
179
|
-
if (localHas(keys).size > 0) return true
|
|
180
184
|
let answered = policy.localRead
|
|
181
185
|
for (const { remote } of readers) {
|
|
182
186
|
for (const key of keys) {
|
|
183
187
|
try {
|
|
184
|
-
if (await remote.has(key)) return true
|
|
188
|
+
if (await remote.has(key, atOf(key, name))) return true
|
|
185
189
|
answered = true
|
|
186
190
|
} catch {
|
|
187
191
|
// no answer from this remote
|
|
@@ -196,13 +200,16 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
196
200
|
for (const name of names) for (const key of reads(name)) byKey.set(key, name)
|
|
197
201
|
const keys = [...byKey.keys()]
|
|
198
202
|
const found = new Set<string>()
|
|
199
|
-
for (const key of localHas(
|
|
203
|
+
for (const key of localHas(names.map((n) => entryKey(plugin, n)))) found.add(byKey.get(key)!)
|
|
200
204
|
for (const { remote } of readers) {
|
|
201
205
|
const ask = keys.filter((k) => !found.has(byKey.get(k)!))
|
|
202
206
|
if (ask.length === 0) break
|
|
203
207
|
if (remote.hasMany === undefined) return null
|
|
204
208
|
try {
|
|
205
|
-
const got: unknown = await remote.hasMany(
|
|
209
|
+
const got: unknown = await remote.hasMany(
|
|
210
|
+
ask,
|
|
211
|
+
ask.map((k) => atOf(k, byKey.get(k)!)),
|
|
212
|
+
)
|
|
206
213
|
if (!(got instanceof Set)) return null
|
|
207
214
|
for (const key of got) if (byKey.has(key as string)) found.add(byKey.get(key as string)!)
|
|
208
215
|
} catch {
|
|
@@ -214,9 +221,9 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
214
221
|
|
|
215
222
|
async getRun(name) {
|
|
216
223
|
for (const { remote } of readers) {
|
|
217
|
-
for (const key of runReadKeys(name, scope)) {
|
|
224
|
+
for (const [i, key] of runReadKeys(name, scope).entries()) {
|
|
218
225
|
try {
|
|
219
|
-
const got = await remote.get(key)
|
|
226
|
+
const got = await remote.get(key, i === 0 ? {} : at)
|
|
220
227
|
if (got === null) continue
|
|
221
228
|
const bytes = await capped(got.body, Infinity)
|
|
222
229
|
if (bytes !== null) return bytes
|
package/src/cache/remote-lane.ts
CHANGED
|
@@ -133,9 +133,11 @@ export class RemoteLane {
|
|
|
133
133
|
constructor(
|
|
134
134
|
resolved: ResolvedRemote,
|
|
135
135
|
private readonly traffic: RemoteTraffic,
|
|
136
|
+
/** The wire as this run asks it: a branch scope's view of `resolved.remote`. */
|
|
137
|
+
remote: RemoteCacheLayer = resolved.remote,
|
|
136
138
|
) {
|
|
137
139
|
this.resolved = resolved
|
|
138
|
-
this.remote =
|
|
140
|
+
this.remote = remote
|
|
139
141
|
this.onError = resolved.onError
|
|
140
142
|
this.endpoint = printableEndpoint(resolved.remote.endpoint)
|
|
141
143
|
}
|
|
@@ -257,7 +259,6 @@ export class RemoteLane {
|
|
|
257
259
|
tally: { count: number; bytes: number; ms: number; failed: number },
|
|
258
260
|
): void {
|
|
259
261
|
const key = this.trafficOf(hash)
|
|
260
|
-
const counted = this.traffic.aliases.get(hash) ?? hash
|
|
261
262
|
key.uploading++
|
|
262
263
|
this.traffic.inFlight++
|
|
263
264
|
this.uploadQueue.push(async () => {
|
|
@@ -284,7 +285,7 @@ export class RemoteLane {
|
|
|
284
285
|
// Nothing sent and nothing failed (the remote held it, or the run
|
|
285
286
|
// stopped first): no traffic, so the footer counts no upload.
|
|
286
287
|
if (key.uploading === 0 && !key.landed && key.failures.length === 0)
|
|
287
|
-
this.traffic.keys.delete(
|
|
288
|
+
this.traffic.keys.delete(hash)
|
|
288
289
|
}
|
|
289
290
|
})
|
|
290
291
|
this.pump()
|
|
@@ -359,11 +360,10 @@ export class RemoteLane {
|
|
|
359
360
|
}
|
|
360
361
|
|
|
361
362
|
private trafficOf(hash: string): RemoteKeyTraffic {
|
|
362
|
-
|
|
363
|
-
let key = this.traffic.keys.get(counted)
|
|
363
|
+
let key = this.traffic.keys.get(hash)
|
|
364
364
|
if (key === undefined) {
|
|
365
365
|
key = { uploading: 0, landed: false, failures: [] }
|
|
366
|
-
this.traffic.keys.set(
|
|
366
|
+
this.traffic.keys.set(hash, key)
|
|
367
367
|
}
|
|
368
368
|
return key
|
|
369
369
|
}
|
package/src/cache/run-store.ts
CHANGED
|
@@ -44,7 +44,10 @@ export async function putRunEntry(
|
|
|
44
44
|
for (const { remote, write } of remotes) {
|
|
45
45
|
if (!write) continue
|
|
46
46
|
try {
|
|
47
|
-
await remote.put(key, new Blob([bytes]), {
|
|
47
|
+
await remote.put(key, new Blob([bytes]), {
|
|
48
|
+
durationMs: 0,
|
|
49
|
+
...(scope === undefined ? {} : { scope }),
|
|
50
|
+
})
|
|
48
51
|
kept = true
|
|
49
52
|
} catch {
|
|
50
53
|
// the other remotes still keep it
|
|
@@ -1,20 +1,12 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
1
|
+
// A branch's cache scope (owner, 2026-10-17): the local store never knows
|
|
2
|
+
// one, so every branch shares one local pool under the task keys. Scopes
|
|
3
|
+
// decide only which remote entry a task asks for: a task the branch changed
|
|
4
|
+
// reads the scope's key, every other task the default key, and a scoped run
|
|
5
|
+
// writes every remote under the scope's key. An artifact is packed under its
|
|
6
|
+
// task key wherever it is stored, so a remote entry ingests under that key.
|
|
5
7
|
|
|
6
8
|
import { xxh3hex } from '../util/index.js'
|
|
7
|
-
import type {
|
|
8
|
-
CacheEntry,
|
|
9
|
-
CacheGetContext,
|
|
10
|
-
CacheLayer,
|
|
11
|
-
DeclaredOutputs,
|
|
12
|
-
IngestMeta,
|
|
13
|
-
InvocationRecord,
|
|
14
|
-
OutputDirRow,
|
|
15
|
-
RunRecord,
|
|
16
|
-
SaveArgs,
|
|
17
|
-
} from './layer.js'
|
|
9
|
+
import type { RemoteCacheLayer, RemoteKey } from './layered-cache.js'
|
|
18
10
|
|
|
19
11
|
/** The key a scope stores `hash` under: task-key shaped, 16 hex, so every wire stores it unchanged. */
|
|
20
12
|
export function scopedKey(scope: string, hash: string): string {
|
|
@@ -22,231 +14,74 @@ export function scopedKey(scope: string, hash: string): string {
|
|
|
22
14
|
}
|
|
23
15
|
|
|
24
16
|
/**
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
* the run's history all reach the entry that answered. An artifact records
|
|
29
|
-
* the key it was packed under and ingest refuses another, so a derived-key
|
|
30
|
-
* collision degrades to a miss. A default artifact a remote ingest refuses is
|
|
31
|
-
* a miss there, and the scope's key is asked next (Security F6).
|
|
17
|
+
* Which remote key each task asks for, in a run with a scope. The routes
|
|
18
|
+
* come from the branch's diff, which only a remote call awaits: a run the
|
|
19
|
+
* local cache answers never waits on git.
|
|
32
20
|
*/
|
|
33
|
-
export class
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
readonly
|
|
39
|
-
|
|
40
|
-
readonly
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
)
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
if (
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
this.
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
return
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
21
|
+
export class BranchScope {
|
|
22
|
+
/** Finds the tasks that read the scope, at the first remote call; null: every task does. */
|
|
23
|
+
private route: (() => Promise<ReadonlySet<string>>) | null = null
|
|
24
|
+
private routes: Promise<ReadonlySet<string>> | null = null
|
|
25
|
+
/** Task key → its task. */
|
|
26
|
+
private readonly taskOf = new Map<string, string>()
|
|
27
|
+
|
|
28
|
+
constructor(readonly scope: string) {}
|
|
29
|
+
|
|
30
|
+
/** Only the tasks `ids` finds read the scope; set before any remote call. */
|
|
31
|
+
routeTasks(ids: () => Promise<ReadonlySet<string>>): void {
|
|
32
|
+
this.route = ids
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
noteKey(taskId: string, hash: string): void {
|
|
36
|
+
this.taskOf.set(hash, taskId)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** The remote key `hash` is read under, and the scope it is read in (none: the default key). */
|
|
40
|
+
private async readKey(hash: string): Promise<{ key: string; at: RemoteKey }> {
|
|
41
|
+
const id = this.taskOf.get(hash)
|
|
42
|
+
if (this.route !== null) this.routes ??= this.route()
|
|
43
|
+
const tasks = this.routes === null ? null : await this.routes
|
|
44
|
+
return tasks !== null && id !== undefined && !tasks.has(id)
|
|
45
|
+
? { key: hash, at: {} }
|
|
46
|
+
: { key: scopedKey(this.scope, hash), at: { scope: this.scope } }
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* `inner` as this run asks it: each task key under its route's key; every
|
|
51
|
+
* write under the scope's. Each call also names its scope, for a backend
|
|
52
|
+
* that keeps one store per scope; no wire vx ships sends it.
|
|
53
|
+
*/
|
|
54
|
+
remote(inner: RemoteCacheLayer): RemoteCacheLayer {
|
|
55
|
+
const read = (h: string) => this.readKey(h)
|
|
56
|
+
const at: RemoteKey = { scope: this.scope }
|
|
57
|
+
const hasMany = inner.hasMany
|
|
58
|
+
return {
|
|
59
|
+
...(inner.endpoint !== undefined ? { endpoint: inner.endpoint } : {}),
|
|
60
|
+
...(inner.open !== undefined ? { open: (ctx) => inner.open!(ctx) } : {}),
|
|
61
|
+
has: async (h) => {
|
|
62
|
+
const r = await read(h)
|
|
63
|
+
return inner.has(r.key, r.at)
|
|
64
|
+
},
|
|
65
|
+
...(hasMany !== undefined
|
|
66
|
+
? {
|
|
67
|
+
hasMany: async (hashes) => {
|
|
68
|
+
const reads = await Promise.all(hashes.map(read))
|
|
69
|
+
const keys = reads.map((r) => r.key)
|
|
70
|
+
const found = await hasMany.call(
|
|
71
|
+
inner,
|
|
72
|
+
keys,
|
|
73
|
+
reads.map((r) => r.at),
|
|
74
|
+
)
|
|
75
|
+
if (!(found instanceof Set)) return found
|
|
76
|
+
return new Set(hashes.filter((_, i) => found.has(keys[i]!)))
|
|
77
|
+
},
|
|
78
|
+
}
|
|
79
|
+
: {}),
|
|
80
|
+
get: async (h) => {
|
|
81
|
+
const r = await read(h)
|
|
82
|
+
return inner.get(r.key, r.at)
|
|
83
|
+
},
|
|
84
|
+
put: (h, body, meta) => inner.put(scopedKey(this.scope, h), body, { ...meta, ...at }),
|
|
84
85
|
}
|
|
85
|
-
return out
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
/** A task key the store holds under the default key or the scope's. */
|
|
89
|
-
async localHasMany(hashes: readonly string[]): Promise<Set<string>> {
|
|
90
|
-
const held = await this.inner.localHasMany?.([...hashes, ...hashes.map((h) => this.own(h))])
|
|
91
|
-
if (held === undefined) return new Set()
|
|
92
|
-
return new Set(hashes.filter((h) => held.has(h) || held.has(this.own(h))))
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
private async scopedHasMany(hashes: readonly string[]): Promise<Set<string> | null> {
|
|
96
|
-
const scoped = hashes.map((h) => this.own(h))
|
|
97
|
-
const [plain, own] = await Promise.all([
|
|
98
|
-
this.inner.remoteHasMany!(hashes),
|
|
99
|
-
this.inner.remoteHasMany!(scoped),
|
|
100
|
-
])
|
|
101
|
-
if (plain == null || own == null) return null
|
|
102
|
-
const found = new Set(plain)
|
|
103
|
-
const scopeOnly: string[] = []
|
|
104
|
-
for (let i = 0; i < hashes.length; i++) {
|
|
105
|
-
if (plain.has(hashes[i]!) || !own.has(scoped[i]!)) continue
|
|
106
|
-
found.add(hashes[i]!)
|
|
107
|
-
scopeOnly.push(hashes[i]!)
|
|
108
|
-
}
|
|
109
|
-
// Their default key is known absent: its GET is skipped.
|
|
110
|
-
this.inner.markRemoteAbsent?.(scopeOnly)
|
|
111
|
-
return found
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
markRemoteAbsent(hashes: Iterable<string>): void {
|
|
115
|
-
const all: string[] = []
|
|
116
|
-
for (const h of hashes) all.push(h, this.own(h))
|
|
117
|
-
this.inner.markRemoteAbsent?.(all)
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
key(input: Parameters<CacheLayer['key']>[0]): Promise<string> {
|
|
121
|
-
return this.inner.key(input)
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
async get(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null> {
|
|
125
|
-
// The scope's key first when only it is local: the default key's
|
|
126
|
-
// remote GET would cost a round-trip for the same inputs.
|
|
127
|
-
const own = this.own(hash)
|
|
128
|
-
const keys = (await this.inner.localHasMany?.([hash, own]))?.has(own)
|
|
129
|
-
? [own, hash]
|
|
130
|
-
: [hash, own]
|
|
131
|
-
for (const key of keys) {
|
|
132
|
-
const entry = await this.inner.get(key, ctx)
|
|
133
|
-
if (entry === null) continue
|
|
134
|
-
this.served.set(hash, key)
|
|
135
|
-
return key === hash ? entry : { ...entry, hash }
|
|
136
|
-
}
|
|
137
|
-
return null
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
private async scopedGetMany(
|
|
141
|
-
hashes: readonly string[],
|
|
142
|
-
ctx?: (hash: string) => CacheGetContext,
|
|
143
|
-
): Promise<Map<string, CacheEntry>> {
|
|
144
|
-
const out = await this.inner.getMany!(hashes, ctx)
|
|
145
|
-
for (const h of out.keys()) this.served.set(h, h)
|
|
146
|
-
const rest = hashes.filter((h) => !out.has(h))
|
|
147
|
-
if (rest.length === 0) return out
|
|
148
|
-
const byOwn = new Map(rest.map((h) => [this.own(h), h]))
|
|
149
|
-
const scoped = await this.inner.getMany!(
|
|
150
|
-
[...byOwn.keys()],
|
|
151
|
-
ctx === undefined ? undefined : (key) => ctx(byOwn.get(key)!),
|
|
152
|
-
)
|
|
153
|
-
for (const [key, entry] of scoped) {
|
|
154
|
-
const hash = byOwn.get(key)!
|
|
155
|
-
this.served.set(hash, key)
|
|
156
|
-
out.set(hash, { ...entry, hash })
|
|
157
|
-
}
|
|
158
|
-
return out
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
async has(hash: string): Promise<'local' | 'remote' | null> {
|
|
162
|
-
return (await this.inner.has(hash)) ?? (await this.inner.has(this.own(hash)))
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
async prefetch(hash: string, ctx?: CacheGetContext): Promise<boolean> {
|
|
166
|
-
return (await this.inner.prefetch(hash, ctx)) || this.inner.prefetch(this.own(hash), ctx)
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
loadOutputFilesBatch(hashes: readonly string[]): ReturnType<CacheLayer['loadOutputFilesBatch']> {
|
|
170
|
-
return this.batch(hashes, (keys) => this.inner.loadOutputFilesBatch(keys))
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
isOutputsCurrent(
|
|
174
|
-
...args: Parameters<CacheLayer['isOutputsCurrent']>
|
|
175
|
-
): ReturnType<CacheLayer['isOutputsCurrent']> {
|
|
176
|
-
return this.inner.isOutputsCurrent(...args)
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
async recordOutputDirs(
|
|
180
|
-
hash: string,
|
|
181
|
-
projectDir: string,
|
|
182
|
-
prefixes: readonly string[],
|
|
183
|
-
holds?: (files: readonly string[]) => boolean,
|
|
184
|
-
): Promise<void> {
|
|
185
|
-
await this.inner.recordOutputDirs?.(this.keyOf(hash), projectDir, prefixes, holds)
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
recordOutputStamps(hash: string, projectDir: string, workspaceRoot: string): void {
|
|
189
|
-
this.inner.recordOutputStamps?.(this.keyOf(hash), projectDir, workspaceRoot)
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
loadOutputDirsBatch(hashes: readonly string[]): Map<string, OutputDirRow[]> {
|
|
193
|
-
return this.batch(hashes, (keys) => this.inner.loadOutputDirsBatch?.(keys) ?? new Map())
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
async outputDirsCurrent(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean> {
|
|
197
|
-
return (await this.inner.outputDirsCurrent?.(projectDir, rows)) ?? false
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
restoreOutputs(
|
|
201
|
-
hash: string,
|
|
202
|
-
projectDir: string,
|
|
203
|
-
workspaceRoot?: string,
|
|
204
|
-
outputs?: DeclaredOutputs,
|
|
205
|
-
): Promise<void> {
|
|
206
|
-
return this.inner.restoreOutputs(this.keyOf(hash), projectDir, workspaceRoot, outputs)
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
save(args: SaveArgs): Promise<void> {
|
|
210
|
-
const key = this.own(args.hash)
|
|
211
|
-
this.served.set(args.hash, key)
|
|
212
|
-
return this.inner.save({ ...args, hash: key })
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
ingest(hash: string, body: Blob | Response, meta: IngestMeta): Promise<void> {
|
|
216
|
-
const key = this.own(hash)
|
|
217
|
-
this.served.set(hash, key)
|
|
218
|
-
return this.inner.ingest(key, body, meta)
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
/** History names the key that answered, so `vx why` reads that entry's components. */
|
|
222
|
-
recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void {
|
|
223
|
-
this.inner.recordRunBundle({
|
|
224
|
-
...bundle,
|
|
225
|
-
runs: bundle.runs.map((r) => (r.hash === undefined ? r : { ...r, hash: this.keyOf(r.hash) })),
|
|
226
|
-
})
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
stats(...args: Parameters<CacheLayer['stats']>): ReturnType<CacheLayer['stats']> {
|
|
230
|
-
return this.inner.stats(...args)
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
hashFile(filePath: string): Promise<string> {
|
|
234
|
-
return this.inner.hashFile(filePath)
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
outputsPath(hash: string): string {
|
|
238
|
-
return this.inner.outputsPath(this.keyOf(hash))
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
artifactSize(hash: string): number | undefined {
|
|
242
|
-
return this.inner.artifactSize?.(this.keyOf(hash))
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
prune(...args: Parameters<CacheLayer['prune']>): ReturnType<CacheLayer['prune']> {
|
|
246
|
-
return this.inner.prune(...args)
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
close(): void {
|
|
250
|
-
this.inner.close()
|
|
251
86
|
}
|
|
252
87
|
}
|