@vzn/vx 0.0.643 → 0.0.645
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +15 -23
- package/package.json +5 -5
- package/schemas/info.json +0 -11
- package/schemas/init.json +5 -0
- package/skills/vx/SKILL.md +1 -1
- package/src/cache/archive.ts +89 -2
- package/src/cache/cache.ts +396 -176
- package/src/cache/file-hashes.ts +2 -14
- package/src/cache/fold-memo.ts +7 -3
- package/src/cache/index.ts +4 -2
- package/src/cache/key-fold.ts +4 -1
- package/src/cache/layer.ts +14 -10
- package/src/cache/layered-cache.ts +157 -41
- package/src/cache/plugin-cache.ts +40 -44
- package/src/cache/remote-lane.ts +59 -25
- package/src/cache/scoped-cache.ts +3 -80
- package/src/cache/tar-stream.ts +12 -7
- package/src/cli/format.ts +1 -2
- package/src/cli/help.ts +10 -4
- package/src/cli/info.ts +2 -19
- package/src/cli/init.ts +121 -20
- package/src/cli/plan-format.ts +6 -2
- package/src/cli/plugin-commands.ts +4 -5
- package/src/cli/plugin-templates.ts +1 -1
- package/src/cli/run.ts +3 -1
- package/src/cli/upgrade.ts +39 -9
- package/src/config.ts +13 -10
- package/src/exec/sandbox-paths.ts +2 -2
- package/src/graph/scheduler.ts +3 -1
- package/src/graph/task-graph.ts +15 -11
- package/src/index.ts +2 -2
- package/src/orchestrator/affected-tasks.ts +0 -18
- package/src/orchestrator/doctor.ts +0 -17
- package/src/orchestrator/execute-task.ts +1 -1
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/options.ts +6 -8
- package/src/orchestrator/plan.ts +7 -1
- package/src/orchestrator/plugin-host.ts +6 -5
- package/src/orchestrator/plugin.ts +12 -5
- package/src/orchestrator/prepare.ts +16 -46
- package/src/orchestrator/projects.ts +8 -1
- package/src/orchestrator/remote-prefetch.ts +11 -4
- package/src/orchestrator/run-context.ts +48 -5
- package/src/orchestrator/run-settings.ts +3 -0
- package/src/orchestrator/run.ts +10 -26
- package/src/orchestrator/task-hash.ts +1 -3
- package/src/report/logger.ts +0 -4
- package/src/report/summary.ts +14 -13
- package/src/settings.ts +38 -15
- package/src/util/index.ts +1 -0
- package/src/util/paths.ts +42 -33
- package/src/util/size.ts +4 -5
- package/src/workspace/affected.ts +0 -39
- package/src/workspace/config-schema.ts +0 -36
- package/src/workspace/index.ts +0 -1
package/src/cache/file-hashes.ts
CHANGED
|
@@ -14,9 +14,6 @@ import { fileIdentity, repoFacts } from './git-inputs.js'
|
|
|
14
14
|
import { FILE_HASH_RACY_MS, isIndexFull, racyWindowMs } from './layer.js'
|
|
15
15
|
import { isPermissionError, UserError } from '../util/index.js'
|
|
16
16
|
|
|
17
|
-
const FILE_HASHES_SWEPT_AT = 'file_hashes_swept_at'
|
|
18
|
-
const DAY_MS = 24 * 60 * 60 * 1000
|
|
19
|
-
|
|
20
17
|
export class FileHashStore {
|
|
21
18
|
private readonly selectFileHash: ReturnType<Database['prepare']>
|
|
22
19
|
private readonly upsertFileHash: ReturnType<Database['prepare']>
|
|
@@ -62,22 +59,13 @@ export class FileHashStore {
|
|
|
62
59
|
* path) keeps its row forever otherwise, and `cache.db` grows with every
|
|
63
60
|
* path ever hashed. A hit does not refresh `seen_at` (a write per file
|
|
64
61
|
* per run), so a file unchanged for the window loses its row too and is
|
|
65
|
-
* read once more; a dropped row is only a memo miss.
|
|
66
|
-
* 8 ms at 100,000 rows, so it runs once a day, on its own clock in
|
|
67
|
-
* `schema_meta` (item 1082).
|
|
62
|
+
* read once more; a dropped row is only a memo miss.
|
|
68
63
|
*/
|
|
69
|
-
pruneOlderThan(cutoff: number
|
|
70
|
-
const last = this.db
|
|
71
|
-
.prepare('SELECT value FROM schema_meta WHERE key = ?')
|
|
72
|
-
.get(FILE_HASHES_SWEPT_AT) as { value: string } | null
|
|
73
|
-
if (last !== null && now - Number(last.value) < DAY_MS) return
|
|
64
|
+
pruneOlderThan(cutoff: number): void {
|
|
74
65
|
this.db.transaction(() => {
|
|
75
66
|
this.db.prepare('DELETE FROM file_hashes WHERE seen_at < ?').run(cutoff)
|
|
76
67
|
this.db.prepare('DELETE FROM blob_sizes WHERE seen_at < ?').run(cutoff)
|
|
77
68
|
this.db.prepare('DELETE FROM blob_verdicts WHERE seen_at < ?').run(cutoff)
|
|
78
|
-
this.db
|
|
79
|
-
.prepare('INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)')
|
|
80
|
-
.run(FILE_HASHES_SWEPT_AT, String(now))
|
|
81
69
|
})()
|
|
82
70
|
}
|
|
83
71
|
|
package/src/cache/fold-memo.ts
CHANGED
|
@@ -43,16 +43,20 @@ export class FoldMemo {
|
|
|
43
43
|
if (this.write) this.dirty.set(taskId, { probe, key })
|
|
44
44
|
}
|
|
45
45
|
|
|
46
|
-
/** Write the folds this run made
|
|
47
|
-
flush(
|
|
46
|
+
/** Write the folds this run made. */
|
|
47
|
+
flush(now: number = Date.now()): void {
|
|
48
48
|
if (this.dirty.size === 0) return
|
|
49
49
|
const upsert = this.db.prepare(
|
|
50
50
|
'INSERT OR REPLACE INTO fold_memo(task_id, probe, key, seen_at) VALUES (?, ?, ?, ?)',
|
|
51
51
|
)
|
|
52
52
|
this.db.transaction(() => {
|
|
53
53
|
for (const [taskId, r] of this.dirty) upsert.run(taskId, r.probe, r.key, now)
|
|
54
|
-
this.db.prepare('DELETE FROM fold_memo WHERE seen_at < ?').run(cutoff)
|
|
55
54
|
})()
|
|
56
55
|
this.dirty.clear()
|
|
57
56
|
}
|
|
57
|
+
|
|
58
|
+
/** Rows no run has rewritten since `cutoff`. */
|
|
59
|
+
prune(cutoff: number): void {
|
|
60
|
+
this.db.prepare('DELETE FROM fold_memo WHERE seen_at < ?').run(cutoff)
|
|
61
|
+
}
|
|
58
62
|
}
|
package/src/cache/index.ts
CHANGED
|
@@ -8,6 +8,8 @@ export {
|
|
|
8
8
|
applyCacheFlags,
|
|
9
9
|
ArtifactVanishedError,
|
|
10
10
|
Cache,
|
|
11
|
+
DEFAULT_STORE,
|
|
12
|
+
type StoreOptions,
|
|
11
13
|
type CacheFlags,
|
|
12
14
|
type CacheEntry,
|
|
13
15
|
type CacheKeyInput,
|
|
@@ -80,13 +82,13 @@ export {
|
|
|
80
82
|
type BlobSizeMemo,
|
|
81
83
|
} from './git-inputs.js'
|
|
82
84
|
export {
|
|
85
|
+
type ArtifactInfo,
|
|
83
86
|
LayeredCache,
|
|
84
87
|
type RemoteCacheLayer,
|
|
85
|
-
type RemoteKey,
|
|
86
88
|
type RemoteOpenContext,
|
|
87
89
|
} from './layered-cache.js'
|
|
90
|
+
export { networkConcurrency } from './remote-lane.js'
|
|
88
91
|
export type { ResolvedRemote } from './remote-lane.js'
|
|
89
|
-
export { BranchScope } from './scoped-cache.js'
|
|
90
92
|
export {
|
|
91
93
|
absentPluginCache,
|
|
92
94
|
pluginCache,
|
package/src/cache/key-fold.ts
CHANGED
|
@@ -117,7 +117,10 @@ import type { CacheKeyInput } from './layer.js'
|
|
|
117
117
|
// v46: vx's names are reserved only at the project root; a v45 artifact
|
|
118
118
|
// left out a build's own `dist/.git/**`, `dist/.vx/**` and nested
|
|
119
119
|
// `vx.lock.json` (Growth's parity, 2026-10-17).
|
|
120
|
-
|
|
120
|
+
// v47: the local store keeps plain files (`cache.store`).
|
|
121
|
+
// v48: a task glob is standard again (M, 2026-10-18): `app/[id]/page.js` is
|
|
122
|
+
// a class, and a v47 entry under its unchanged key holds the literal route.
|
|
123
|
+
export const CACHE_VERSION = 'vx-cache-v48'
|
|
121
124
|
|
|
122
125
|
/**
|
|
123
126
|
* The digest a key folds for an input gone between its enumeration and its
|
package/src/cache/layer.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
// reference to `Cache` below is a type (the local handle a layer may wrap).
|
|
6
6
|
|
|
7
7
|
import type { Cache } from './cache.js'
|
|
8
|
+
import type { ArtifactInfo } from './layered-cache.js'
|
|
8
9
|
import { fsClockLag, stampGrainMs } from './fs-clock.js'
|
|
9
10
|
|
|
10
11
|
export interface CacheKeyInput {
|
|
@@ -493,8 +494,8 @@ export class CorruptArtifactError extends Error {
|
|
|
493
494
|
|
|
494
495
|
/**
|
|
495
496
|
* `restoreOutputs` found no artifact where the probe found one: it was
|
|
496
|
-
* removed in between — a `vx cache clean` in another shell, or
|
|
497
|
-
*
|
|
497
|
+
* removed in between — a `vx cache clean` in another shell, or one for
|
|
498
|
+
* another workspace sharing the cache directory. The
|
|
498
499
|
* bytes are not wrong, they are gone, so the entry is a MISS and the task
|
|
499
500
|
* runs (execute-task.ts); nothing about the cache needs throwing away.
|
|
500
501
|
* Also thrown, with its `reason`, for an artifact whose bytes ARE wrong
|
|
@@ -510,8 +511,7 @@ export class ArtifactVanishedError extends Error {
|
|
|
510
511
|
cause?: unknown,
|
|
511
512
|
) {
|
|
512
513
|
super(
|
|
513
|
-
reason ??
|
|
514
|
-
`its cache artifact ${hash} vanished before the restore (a prune or another run's retention)`,
|
|
514
|
+
reason ?? `its cache artifact ${hash} vanished before the restore (a vx cache clean)`,
|
|
515
515
|
cause === undefined ? undefined : { cause },
|
|
516
516
|
)
|
|
517
517
|
this.name = 'ArtifactVanishedError'
|
|
@@ -551,12 +551,12 @@ export interface CacheLayer {
|
|
|
551
551
|
*/
|
|
552
552
|
readonly hasRemote?: boolean
|
|
553
553
|
/**
|
|
554
|
-
* Batch existence probe over the remote — the
|
|
555
|
-
*
|
|
556
|
-
* per-hash". Only meaningful with `hasRemote`; a remote layer that can't
|
|
554
|
+
* Batch existence probe over the remote — the `hashes` stored remotely
|
|
555
|
+
* and what the first remote holding each knows of it (`ArtifactInfo`), in
|
|
556
|
+
* ONE round-trip, or `null` for "no batch info; use per-hash". Only meaningful with `hasRemote`; a remote layer that can't
|
|
557
557
|
* batch omits it and the prefetch pass falls back to per-hash GETs.
|
|
558
558
|
*/
|
|
559
|
-
remoteHasMany?(hashes: readonly string[]): Promise<
|
|
559
|
+
remoteHasMany?(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null>
|
|
560
560
|
/**
|
|
561
561
|
* The subset of `hashes` the local store already holds. The prefetch
|
|
562
562
|
* pass probes the remote only for the rest, so a run the local cache
|
|
@@ -576,8 +576,12 @@ export interface CacheLayer {
|
|
|
576
576
|
* `hasRemote`.
|
|
577
577
|
*/
|
|
578
578
|
probeBeforeUpload?(hashes: Promise<readonly string[]>): void
|
|
579
|
-
/**
|
|
580
|
-
|
|
579
|
+
/**
|
|
580
|
+
* The run's batch probe starts when `started` settles: until then a pull
|
|
581
|
+
* waits, so a lookup that lands first does not GET a key the probe is
|
|
582
|
+
* about to call absent. Only meaningful with `hasRemote`.
|
|
583
|
+
*/
|
|
584
|
+
probeFirst?(started: Promise<void>): void
|
|
581
585
|
/**
|
|
582
586
|
* Resolve once every background write-through upload settles. The
|
|
583
587
|
* orchestrator awaits it before `close()` — an upload reading layer state
|
|
@@ -36,7 +36,6 @@ import type {
|
|
|
36
36
|
} from './cache.js'
|
|
37
37
|
import type { OutputDirRow } from './cache.js'
|
|
38
38
|
import { RemoteLane, type ResolvedRemote } from './remote-lane.js'
|
|
39
|
-
import type { BranchScope } from './scoped-cache.js'
|
|
40
39
|
import type { PluginSecrets } from '../util/index.js'
|
|
41
40
|
|
|
42
41
|
const STOPPED: unique symbol = Symbol('stopped')
|
|
@@ -46,7 +45,11 @@ const STOPPED: unique symbol = Symbol('stopped')
|
|
|
46
45
|
* fails at once, removes its temp, and releases the connection; the cache
|
|
47
46
|
* then closes under no write.
|
|
48
47
|
*/
|
|
49
|
-
function endsOnStop(
|
|
48
|
+
function endsOnStop(
|
|
49
|
+
body: Blob | Response,
|
|
50
|
+
stopped: Promise<typeof STOPPED>,
|
|
51
|
+
onChunk: (bytes: number) => void = () => {},
|
|
52
|
+
): Blob | Response {
|
|
50
53
|
if (body instanceof Blob || body.body === null) return body
|
|
51
54
|
const reader = body.body.getReader()
|
|
52
55
|
let stop = false
|
|
@@ -61,13 +64,28 @@ function endsOnStop(body: Blob | Response, stopped: Promise<typeof STOPPED>): Bl
|
|
|
61
64
|
const next = await reader.read()
|
|
62
65
|
if (stop) controller.error(new Error('the run was stopped'))
|
|
63
66
|
else if (next.done) controller.close()
|
|
64
|
-
else
|
|
67
|
+
else {
|
|
68
|
+
onChunk(next.value.byteLength)
|
|
69
|
+
controller.enqueue(next.value)
|
|
70
|
+
}
|
|
65
71
|
},
|
|
66
72
|
cancel: (reason) => reader.cancel(reason),
|
|
67
73
|
})
|
|
68
74
|
return new Response(stream, { headers: body.headers })
|
|
69
75
|
}
|
|
70
76
|
|
|
77
|
+
const judged = (i: ArtifactInfo): i is { size: number; durationMs: number } =>
|
|
78
|
+
i.size !== undefined && i.durationMs !== undefined
|
|
79
|
+
|
|
80
|
+
/** How often a task waiting on its download weighs running instead. */
|
|
81
|
+
const JUDGE_EVERY_MS = 25
|
|
82
|
+
|
|
83
|
+
interface Download {
|
|
84
|
+
readonly start: number
|
|
85
|
+
received: number
|
|
86
|
+
abandon(): void
|
|
87
|
+
}
|
|
88
|
+
|
|
71
89
|
/** What a remote's `open` is handed: the run's CI verdict and its plugin's keychain. */
|
|
72
90
|
export interface RemoteOpenContext extends PluginSecrets {
|
|
73
91
|
/** The run is on CI: nobody may be asked. */
|
|
@@ -77,13 +95,13 @@ export interface RemoteOpenContext extends PluginSecrets {
|
|
|
77
95
|
}
|
|
78
96
|
|
|
79
97
|
/**
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
* go straight to that store. turboCache(), nxCache() and vx-reapi ignore it.
|
|
98
|
+
* What a remote knows of an artifact it holds, before it is downloaded:
|
|
99
|
+
* the body's size in bytes and the producing task's duration, each when
|
|
100
|
+
* the wire carries it (Turbo's batch query and `HEAD` do).
|
|
84
101
|
*/
|
|
85
|
-
export interface
|
|
86
|
-
readonly
|
|
102
|
+
export interface ArtifactInfo {
|
|
103
|
+
readonly size?: number
|
|
104
|
+
readonly durationMs?: number
|
|
87
105
|
}
|
|
88
106
|
|
|
89
107
|
/**
|
|
@@ -122,17 +140,21 @@ export interface RemoteCacheLayer {
|
|
|
122
140
|
* plugin's keychain and may ask on the terminal.
|
|
123
141
|
*/
|
|
124
142
|
open?(ctx: RemoteOpenContext): Promise<void>
|
|
125
|
-
/** Existence probe: `--dry` and `--force` ask it per key when the remote cannot batch. */
|
|
126
|
-
has(hash: string, at?: RemoteKey): Promise<boolean>
|
|
127
143
|
/**
|
|
128
|
-
*
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
144
|
+
* Existence probe: what the remote knows of the stored artifact, or null
|
|
145
|
+
* when it holds none. `--dry` and `--force` ask it per key when the
|
|
146
|
+
* remote cannot batch.
|
|
147
|
+
*/
|
|
148
|
+
has(hash: string): Promise<ArtifactInfo | null>
|
|
149
|
+
/**
|
|
150
|
+
* Optional batch existence probe: given N hashes, the stored ones and what
|
|
151
|
+
* the remote knows of each, in ONE round-trip. Its presence says the wire
|
|
152
|
+
* can batch: a run asks it once for its keys, `--dry` before its per-key
|
|
153
|
+
* answers, and `--force` in the background to skip uploads of held keys.
|
|
154
|
+
* Without it a run GETs each key directly (no `has`). Never throws for
|
|
155
|
+
* control flow: `null` means "no batch info; use per-hash".
|
|
134
156
|
*/
|
|
135
|
-
hasMany?(hashes: readonly string[]
|
|
157
|
+
hasMany?(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null>
|
|
136
158
|
/**
|
|
137
159
|
* Fetch an artifact; `null` = miss. `body` is read once, by core,
|
|
138
160
|
* straight into the local cache — an HTTP wire returns its `fetch`
|
|
@@ -142,10 +164,7 @@ export interface RemoteCacheLayer {
|
|
|
142
164
|
* nothing and the read is a miss, so a layer can check bytes as they pass
|
|
143
165
|
* (a digest, a signature) and refuse at the end.
|
|
144
166
|
*/
|
|
145
|
-
get(
|
|
146
|
-
hash: string,
|
|
147
|
-
at?: RemoteKey,
|
|
148
|
-
): Promise<{ body: Blob | Response; durationMs: number | undefined } | null>
|
|
167
|
+
get(hash: string): Promise<{ body: Blob | Response; durationMs: number | undefined } | null>
|
|
149
168
|
/**
|
|
150
169
|
* Store an artifact (fire-and-forget from LayeredCache's perspective).
|
|
151
170
|
* `body` is file-backed (`Bun.file`) when the local store holds the
|
|
@@ -154,7 +173,7 @@ export interface RemoteCacheLayer {
|
|
|
154
173
|
* key the remote holds keeps the stored bytes. Core never asks `has`
|
|
155
174
|
* first; a wire that would overwrite checks inside.
|
|
156
175
|
*/
|
|
157
|
-
put(hash: string, body: Blob, meta: { durationMs: number }
|
|
176
|
+
put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void>
|
|
158
177
|
}
|
|
159
178
|
|
|
160
179
|
export class LayeredCache implements CacheLayer {
|
|
@@ -175,6 +194,17 @@ export class LayeredCache implements CacheLayer {
|
|
|
175
194
|
*/
|
|
176
195
|
private readonly inflight = new Map<string, Promise<boolean>>()
|
|
177
196
|
|
|
197
|
+
/**
|
|
198
|
+
* Hashes a batch probe is asking about, each resolving whether a pull may
|
|
199
|
+
* still find it (false: no remote holds it). A `get` that lands while the
|
|
200
|
+
* probe is out waits for its answer instead of spending a GET on a key
|
|
201
|
+
* the probe is about to call absent.
|
|
202
|
+
*/
|
|
203
|
+
private readonly probing = new Map<string, Promise<boolean>>()
|
|
204
|
+
|
|
205
|
+
/** Settles once the run's batch probe has started (`probeFirst`); a pull waits on it. */
|
|
206
|
+
private probeStarted: Promise<void> | null = null
|
|
207
|
+
|
|
178
208
|
/**
|
|
179
209
|
* Hashes whose local artifact was materialized FROM a remote this run
|
|
180
210
|
* (by `prefetch` or `get`'s read-through). A later `get` finds them as
|
|
@@ -185,21 +215,18 @@ export class LayeredCache implements CacheLayer {
|
|
|
185
215
|
* its download and ingest took.
|
|
186
216
|
*/
|
|
187
217
|
private readonly remoteSourced = new Map<string, number>()
|
|
218
|
+
/** What the batch probe said of each held key: its size and stored duration. */
|
|
219
|
+
private readonly infos = new Map<string, ArtifactInfo>()
|
|
220
|
+
/** Bodies being read now, by key: how far along, and how to give one up. */
|
|
221
|
+
private readonly downloads = new Map<string, Download>()
|
|
188
222
|
|
|
189
223
|
/** Remotes in declared order: the first readable one holding a hash answers it. */
|
|
190
224
|
constructor(
|
|
191
225
|
readonly local: Cache,
|
|
192
226
|
remotes: readonly ResolvedRemote[],
|
|
193
|
-
|
|
194
|
-
private readonly branch?: BranchScope,
|
|
227
|
+
uploadConcurrency?: number,
|
|
195
228
|
) {
|
|
196
|
-
this.lanes = remotes.map(
|
|
197
|
-
(r) => new RemoteLane(r, local.traffic, branch?.remote(r.remote) ?? r.remote),
|
|
198
|
-
)
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
noteKey(taskId: string, hash: string): void {
|
|
202
|
-
this.branch?.noteKey(taskId, hash)
|
|
229
|
+
this.lanes = remotes.map((r) => new RemoteLane(r, local.traffic, uploadConcurrency))
|
|
203
230
|
}
|
|
204
231
|
|
|
205
232
|
// Asked, not kept: a remote whose `open` failed leaves the run after this is built.
|
|
@@ -236,13 +263,37 @@ export class LayeredCache implements CacheLayer {
|
|
|
236
263
|
* absences either way, so a hash only a sibling holds costs it no GET.
|
|
237
264
|
* Never throws.
|
|
238
265
|
*/
|
|
239
|
-
async remoteHasMany(hashes: readonly string[]): Promise<
|
|
266
|
+
async remoteHasMany(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
|
|
240
267
|
if (this.readable.length === 0) return null
|
|
241
|
-
|
|
268
|
+
const answer = this.probeMany(hashes).then((found) => {
|
|
269
|
+
if (found !== null) for (const [h, info] of found) this.infos.set(h, info)
|
|
270
|
+
return found
|
|
271
|
+
})
|
|
272
|
+
const waiting = hashes.filter((h) => !this.inflight.has(h) && !this.probing.has(h))
|
|
273
|
+
for (const h of waiting) {
|
|
274
|
+
this.probing.set(
|
|
275
|
+
h,
|
|
276
|
+
answer.then(
|
|
277
|
+
(found) => {
|
|
278
|
+
this.probing.delete(h)
|
|
279
|
+
return found === null || found.has(h)
|
|
280
|
+
},
|
|
281
|
+
() => {
|
|
282
|
+
this.probing.delete(h)
|
|
283
|
+
return true
|
|
284
|
+
},
|
|
285
|
+
),
|
|
286
|
+
)
|
|
287
|
+
}
|
|
288
|
+
return answer
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
private async probeMany(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
|
|
292
|
+
let out: Map<string, ArtifactInfo> | null = new Map()
|
|
242
293
|
for (const lane of this.readable) {
|
|
243
294
|
const found = await lane.hasMany(hashes)
|
|
244
295
|
if (found === null) out = null
|
|
245
|
-
else if (out !== null) for (const h of found) out.
|
|
296
|
+
else if (out !== null) for (const [h, info] of found) if (!out.has(h)) out.set(h, info)
|
|
246
297
|
}
|
|
247
298
|
return out
|
|
248
299
|
}
|
|
@@ -261,6 +312,17 @@ export class LayeredCache implements CacheLayer {
|
|
|
261
312
|
}
|
|
262
313
|
}
|
|
263
314
|
|
|
315
|
+
probeFirst(started: Promise<void>): void {
|
|
316
|
+
const gate = started.then(
|
|
317
|
+
() => {},
|
|
318
|
+
() => {},
|
|
319
|
+
)
|
|
320
|
+
this.probeStarted = gate
|
|
321
|
+
void gate.then(() => {
|
|
322
|
+
if (this.probeStarted === gate) this.probeStarted = null
|
|
323
|
+
})
|
|
324
|
+
}
|
|
325
|
+
|
|
264
326
|
probeBeforeUpload(hashes: Promise<readonly string[]>): void {
|
|
265
327
|
for (const lane of this.lanes) lane.probeBeforeUpload(hashes)
|
|
266
328
|
}
|
|
@@ -283,7 +345,7 @@ export class LayeredCache implements CacheLayer {
|
|
|
283
345
|
// remote walk per key: a settled `false` means no remote had it
|
|
284
346
|
// (no second probe), and a settled/in-flight `true` means the
|
|
285
347
|
// artifact was/will-be ingested locally — re-read below.
|
|
286
|
-
const ingested = await this.pullFromRemote(hash, ctx)
|
|
348
|
+
const ingested = await this.awaitOrRun(hash, this.pullFromRemote(hash, ctx))
|
|
287
349
|
if (!ingested) return null
|
|
288
350
|
|
|
289
351
|
// Read past the local READ gate: `ingest` is deliberately ungated, so
|
|
@@ -305,6 +367,39 @@ export class LayeredCache implements CacheLayer {
|
|
|
305
367
|
return this.asRemote(hash, materialized)
|
|
306
368
|
}
|
|
307
369
|
|
|
370
|
+
/**
|
|
371
|
+
* A pull the task now waits on, given up when the rest of its body would
|
|
372
|
+
* take longer than running the task (its stored duration, doubled for
|
|
373
|
+
* the save a run adds): the task then runs here, a miss. Only a key the
|
|
374
|
+
* batch probe sized and timed is judged, from its own body's pace so far.
|
|
375
|
+
* The facts are read each tick: the probe can answer after the task asks.
|
|
376
|
+
*/
|
|
377
|
+
private async awaitOrRun(hash: string, pull: Promise<boolean>): Promise<boolean> {
|
|
378
|
+
if (this.infos.has(hash) && !judged(this.infos.get(hash)!)) return pull
|
|
379
|
+
for (;;) {
|
|
380
|
+
const done = await Promise.race([pull, Bun.sleep(JUDGE_EVERY_MS).then(() => undefined)])
|
|
381
|
+
if (done !== undefined) return done
|
|
382
|
+
const info = this.infos.get(hash)
|
|
383
|
+
if (info === undefined) continue
|
|
384
|
+
if (!judged(info)) return pull
|
|
385
|
+
const budget = info.durationMs * 2
|
|
386
|
+
const dl = this.downloads.get(hash)
|
|
387
|
+
if (dl === undefined) continue
|
|
388
|
+
const elapsed = performance.now() - dl.start
|
|
389
|
+
const left =
|
|
390
|
+
dl.received === 0
|
|
391
|
+
? elapsed > budget
|
|
392
|
+
? Infinity
|
|
393
|
+
: 0
|
|
394
|
+
: ((info.size - dl.received) * elapsed) / dl.received
|
|
395
|
+
if (left > budget) {
|
|
396
|
+
dl.abandon()
|
|
397
|
+
await pull
|
|
398
|
+
return false
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
308
403
|
private asRemote(hash: string, entry: CacheEntry): CacheEntry {
|
|
309
404
|
const fetchMs = this.remoteSourced.get(hash)
|
|
310
405
|
return fetchMs === undefined ? entry : { ...entry, source: 'remote', fetchMs }
|
|
@@ -315,7 +410,7 @@ export class LayeredCache implements CacheLayer {
|
|
|
315
410
|
// a remote cache. Remote errors degrade to a miss (never throw).
|
|
316
411
|
async has(hash: string): Promise<'local' | 'remote' | null> {
|
|
317
412
|
if ((await this.local.has(hash)) === 'local') return 'local'
|
|
318
|
-
for (const lane of this.readable) if (await lane.has(hash)) return 'remote'
|
|
413
|
+
for (const lane of this.readable) if ((await lane.has(hash)) !== null) return 'remote'
|
|
319
414
|
return null
|
|
320
415
|
}
|
|
321
416
|
|
|
@@ -331,6 +426,11 @@ export class LayeredCache implements CacheLayer {
|
|
|
331
426
|
private pullFromRemote(hash: string, ctx?: CacheGetContext): Promise<boolean> {
|
|
332
427
|
const existing = this.inflight.get(hash)
|
|
333
428
|
if (existing) return existing
|
|
429
|
+
const probed = this.probing.get(hash)
|
|
430
|
+
if (probed !== undefined)
|
|
431
|
+
return probed.then((found) => (found ? this.pullFromRemote(hash, ctx) : false))
|
|
432
|
+
if (this.probeStarted !== null)
|
|
433
|
+
return this.probeStarted.then(() => this.pullFromRemote(hash, ctx))
|
|
334
434
|
const p = this.doPullFromRemote(hash, ctx)
|
|
335
435
|
this.inflight.set(hash, p)
|
|
336
436
|
return p
|
|
@@ -384,10 +484,19 @@ export class LayeredCache implements CacheLayer {
|
|
|
384
484
|
durationMs: durationOf(remoteBody.durationMs),
|
|
385
485
|
...(ctx?.outputs !== undefined ? { outputs: ctx.outputs } : {}),
|
|
386
486
|
}
|
|
487
|
+
let abandon = (): void => {}
|
|
488
|
+
const abandoned = new Promise<typeof STOPPED>((resolve) => (abandon = () => resolve(STOPPED)))
|
|
489
|
+
const dl: Download = { start: performance.now(), received: 0, abandon }
|
|
490
|
+
this.downloads.set(hash, dl)
|
|
491
|
+
const ends = Promise.race([stopped, abandoned])
|
|
387
492
|
try {
|
|
388
|
-
const ingest = this.local.ingest(
|
|
493
|
+
const ingest = this.local.ingest(
|
|
494
|
+
hash,
|
|
495
|
+
endsOnStop(remoteBody.body, ends, (n) => (dl.received += n)),
|
|
496
|
+
meta,
|
|
497
|
+
)
|
|
389
498
|
ingest.catch(() => undefined)
|
|
390
|
-
if ((await Promise.race([ingest,
|
|
499
|
+
if ((await Promise.race([ingest, ends])) === STOPPED) return false
|
|
391
500
|
} catch (err) {
|
|
392
501
|
// The bytes came off the network — a corrupt/truncated remote
|
|
393
502
|
// artifact must degrade to a cache miss (task re-executes), not
|
|
@@ -395,8 +504,13 @@ export class LayeredCache implements CacheLayer {
|
|
|
395
504
|
// down: `Cache.restoreOutputs` drops it (A-52).
|
|
396
505
|
lane.report('download', hash, err)
|
|
397
506
|
return false
|
|
507
|
+
} finally {
|
|
508
|
+
this.downloads.delete(hash)
|
|
398
509
|
}
|
|
399
510
|
this.remoteSourced.set(hash, Math.round(performance.now() - fetchStart))
|
|
511
|
+
this.local.downloads.count++
|
|
512
|
+
this.local.downloads.bytes +=
|
|
513
|
+
remoteBody.body instanceof Blob ? remoteBody.body.size : dl.received
|
|
400
514
|
return true
|
|
401
515
|
}
|
|
402
516
|
|
|
@@ -457,8 +571,10 @@ export class LayeredCache implements CacheLayer {
|
|
|
457
571
|
return
|
|
458
572
|
}
|
|
459
573
|
}
|
|
460
|
-
const body = (): { body: Blob; release?: () => Promise<void> } =>
|
|
461
|
-
packed !== undefined
|
|
574
|
+
const body = (): Promise<{ body: Blob; release?: () => Promise<void> }> =>
|
|
575
|
+
packed !== undefined
|
|
576
|
+
? Promise.resolve({ body: new Blob([packed]) })
|
|
577
|
+
: this.local.pinArtifact(hash)
|
|
462
578
|
for (const lane of writable) lane.upload(hash, durationMs, body, this.local.uploads)
|
|
463
579
|
}
|
|
464
580
|
|
|
@@ -8,7 +8,6 @@
|
|
|
8
8
|
|
|
9
9
|
import type { Database } from 'bun:sqlite'
|
|
10
10
|
import { xxh3hex } from '../util/index.js'
|
|
11
|
-
import type { RemoteKey } from './layered-cache.js'
|
|
12
11
|
import type { CachePolicy } from './policy.js'
|
|
13
12
|
import type { ResolvedRemote } from './remote-lane.js'
|
|
14
13
|
import { scopedKey } from './scoped-cache.js'
|
|
@@ -19,17 +18,21 @@ export const PLUGIN_ENTRY_MAX_BYTES = 1024 * 1024
|
|
|
19
18
|
|
|
20
19
|
/** What a plugin's `ctx.cache` offers. Keys are the plugin's own names. */
|
|
21
20
|
export interface PluginCache {
|
|
22
|
-
/**
|
|
23
|
-
|
|
21
|
+
/**
|
|
22
|
+
* The bytes stored under `key`, from the first layer holding them; null
|
|
23
|
+
* when none does. On a scoped run a remote is asked for the branch's
|
|
24
|
+
* entry, or with `scoped` false the base branch's: one request either way.
|
|
25
|
+
*/
|
|
26
|
+
get(key: string, scoped?: boolean): Promise<Uint8Array | null>
|
|
24
27
|
/** Store `bytes` under `key` in every layer the policy writes, where it is not already; empty bytes store nothing. */
|
|
25
28
|
put(key: string, bytes: Uint8Array): Promise<void>
|
|
26
29
|
/** Whether some layer holds `key`; null when no layer could answer. */
|
|
27
|
-
has(key: string): Promise<boolean | null>
|
|
30
|
+
has(key: string, scoped?: boolean): Promise<boolean | null>
|
|
28
31
|
/**
|
|
29
32
|
* The subset of `keys` some layer holds, in one probe per layer; null
|
|
30
33
|
* when a readable remote has no batch probe: ask `get` one at a time.
|
|
31
34
|
*/
|
|
32
|
-
hasMany(keys: readonly string[]): Promise<Set<string> | null>
|
|
35
|
+
hasMany(keys: readonly string[], scoped?: boolean): Promise<Set<string> | null>
|
|
33
36
|
/**
|
|
34
37
|
* What core kept of a run under `name` (a `log` or `outputs` name its
|
|
35
38
|
* `RunReport` hands `teardown`), from the first readable remote holding
|
|
@@ -88,18 +91,17 @@ async function capped(body: unknown, max = PLUGIN_ENTRY_MAX_BYTES): Promise<Uint
|
|
|
88
91
|
/**
|
|
89
92
|
* `plugin`'s view of the cache. The local store never knows a scope
|
|
90
93
|
* (owner, 2026-10-17): it keeps every entry under its name's key. A scoped
|
|
91
|
-
* run
|
|
92
|
-
*
|
|
94
|
+
* run writes a remote only under the scope's key, and reads one key: the
|
|
95
|
+
* scope's, or with `scoped` false the default (M, 2026-10-18: no request
|
|
96
|
+
* twice).
|
|
93
97
|
*/
|
|
94
98
|
export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache {
|
|
95
99
|
const { db, policy, scope } = host
|
|
96
100
|
const readers = host.remotes.filter((r) => r.read)
|
|
97
101
|
const writers = host.remotes.filter((r) => r.write)
|
|
98
|
-
const
|
|
99
|
-
const atOf = (key: string, name: string): RemoteKey => (key === entryKey(plugin, name) ? {} : at)
|
|
100
|
-
const reads = (name: string): string[] => {
|
|
102
|
+
const readKey = (name: string, scoped: boolean): string => {
|
|
101
103
|
const key = entryKey(plugin, name)
|
|
102
|
-
return scope === undefined ?
|
|
104
|
+
return scope === undefined || !scoped ? key : scopedKey(scope, key)
|
|
103
105
|
}
|
|
104
106
|
const writeKey = (name: string): string => {
|
|
105
107
|
const key = entryKey(plugin, name)
|
|
@@ -143,23 +145,21 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
143
145
|
}
|
|
144
146
|
|
|
145
147
|
return {
|
|
146
|
-
async get(name) {
|
|
148
|
+
async get(name, scoped = true) {
|
|
147
149
|
const local = entryKey(plugin, name)
|
|
148
150
|
const hit = localGet(local)
|
|
149
151
|
if (hit !== null) return hit
|
|
150
|
-
const
|
|
152
|
+
const key = readKey(name, scoped)
|
|
151
153
|
for (const { remote } of readers) {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
// a failing remote is a miss there
|
|
162
|
-
}
|
|
154
|
+
try {
|
|
155
|
+
const got = await remote.get(key)
|
|
156
|
+
if (got === null) continue
|
|
157
|
+
const bytes = await capped(got.body)
|
|
158
|
+
if (bytes === null) continue
|
|
159
|
+
if (policy.localWrite) localPut(local, bytes)
|
|
160
|
+
return bytes
|
|
161
|
+
} catch {
|
|
162
|
+
// a failing remote is a miss there
|
|
163
163
|
}
|
|
164
164
|
}
|
|
165
165
|
return null
|
|
@@ -171,33 +171,31 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
171
171
|
if (policy.localWrite) localPut(entryKey(plugin, name), bytes)
|
|
172
172
|
for (const { remote } of writers) {
|
|
173
173
|
try {
|
|
174
|
-
await remote.put(key, new Blob([bytes]), { durationMs: 0
|
|
174
|
+
await remote.put(key, new Blob([bytes]), { durationMs: 0 })
|
|
175
175
|
} catch {
|
|
176
176
|
// the other layers still keep it
|
|
177
177
|
}
|
|
178
178
|
}
|
|
179
179
|
},
|
|
180
180
|
|
|
181
|
-
async has(name) {
|
|
181
|
+
async has(name, scoped = true) {
|
|
182
182
|
if (localHas([entryKey(plugin, name)]).size > 0) return true
|
|
183
|
-
const
|
|
183
|
+
const key = readKey(name, scoped)
|
|
184
184
|
let answered = policy.localRead
|
|
185
185
|
for (const { remote } of readers) {
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
// no answer from this remote
|
|
192
|
-
}
|
|
186
|
+
try {
|
|
187
|
+
if (await remote.has(key)) return true
|
|
188
|
+
answered = true
|
|
189
|
+
} catch {
|
|
190
|
+
// no answer from this remote
|
|
193
191
|
}
|
|
194
192
|
}
|
|
195
193
|
return answered ? false : null
|
|
196
194
|
},
|
|
197
195
|
|
|
198
|
-
async hasMany(names) {
|
|
196
|
+
async hasMany(names, scoped = true) {
|
|
199
197
|
const byKey = new Map<string, string>()
|
|
200
|
-
for (const name of names)
|
|
198
|
+
for (const name of names) byKey.set(readKey(name, scoped), name)
|
|
201
199
|
const keys = [...byKey.keys()]
|
|
202
200
|
const found = new Set<string>()
|
|
203
201
|
for (const key of localHas(names.map((n) => entryKey(plugin, n)))) found.add(byKey.get(key)!)
|
|
@@ -206,12 +204,10 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
206
204
|
if (ask.length === 0) break
|
|
207
205
|
if (remote.hasMany === undefined) return null
|
|
208
206
|
try {
|
|
209
|
-
const got: unknown = await remote.hasMany(
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
if (!(got instanceof Set)) return null
|
|
214
|
-
for (const key of got) if (byKey.has(key as string)) found.add(byKey.get(key as string)!)
|
|
207
|
+
const got: unknown = await remote.hasMany(ask)
|
|
208
|
+
if (!(got instanceof Map)) return null
|
|
209
|
+
for (const key of got.keys())
|
|
210
|
+
if (byKey.has(key as string)) found.add(byKey.get(key as string)!)
|
|
215
211
|
} catch {
|
|
216
212
|
return null
|
|
217
213
|
}
|
|
@@ -221,9 +217,9 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
221
217
|
|
|
222
218
|
async getRun(name) {
|
|
223
219
|
for (const { remote } of readers) {
|
|
224
|
-
for (const
|
|
220
|
+
for (const key of runReadKeys(name, scope)) {
|
|
225
221
|
try {
|
|
226
|
-
const got = await remote.get(key
|
|
222
|
+
const got = await remote.get(key)
|
|
227
223
|
if (got === null) continue
|
|
228
224
|
const bytes = await capped(got.body, Infinity)
|
|
229
225
|
if (bytes !== null) return bytes
|