@vzn/vx 0.0.646 → 0.0.649
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -22
- package/launcher.cjs +9 -2
- package/launcher.sh +2 -2
- package/package.json +5 -5
- package/schemas/cache.json +5 -5
- package/schemas/error.json +1 -1
- package/schemas/info.json +16 -2
- package/schemas/last.json +112 -5
- package/schemas/plan.json +28 -2
- package/schemas/show.json +10 -1
- package/skills/vx/SKILL.md +18 -2
- package/src/bin.ts +53 -12
- package/src/cache/archive.ts +324 -23
- package/src/cache/cache.ts +268 -53
- package/src/cache/git-inputs.ts +219 -87
- package/src/cache/index.ts +3 -2
- package/src/cache/inputs.ts +460 -78
- package/src/cache/key-fold.ts +15 -6
- package/src/cache/layer.ts +49 -9
- package/src/cache/layered-cache.ts +123 -58
- package/src/cache/linked-inputs.ts +1 -1
- package/src/cache/output-index.ts +19 -3
- package/src/cache/plugin-cache.ts +49 -23
- package/src/cache/remote-hits.ts +72 -0
- package/src/cache/remote-lane.ts +90 -14
- package/src/cache/run-history.ts +70 -23
- package/src/cache/run-store.ts +11 -10
- package/src/cache/schema.ts +25 -6
- package/src/cache/tar-stream.ts +5 -0
- package/src/cache/umask.ts +1 -2
- package/src/cli/cache.ts +58 -27
- package/src/cli/completions.ts +12 -5
- package/src/cli/docs.ts +42 -16
- package/src/cli/help.ts +58 -19
- package/src/cli/index.ts +79 -33
- package/src/cli/info.ts +12 -6
- package/src/cli/init.ts +186 -31
- package/src/cli/last.ts +439 -45
- package/src/cli/lock.ts +197 -52
- package/src/cli/plan-format.ts +91 -49
- package/src/cli/plugin-commands.ts +70 -14
- package/src/cli/plugin-templates.ts +13 -13
- package/src/cli/registry.ts +209 -0
- package/src/cli/run.ts +109 -34
- package/src/cli/select.ts +143 -46
- package/src/cli/show.ts +59 -16
- package/src/cli/task-verb.ts +16 -6
- package/src/cli/upgrade.ts +151 -49
- package/src/cli/watch-cycle.ts +10 -5
- package/src/cli/watch-filter.ts +71 -3
- package/src/cli/watch-fs.ts +22 -6
- package/src/cli/watch-judge.ts +48 -6
- package/src/cli/watch.ts +163 -42
- package/src/cli/why.ts +211 -73
- package/src/cli/workspace-config.ts +54 -15
- package/src/config.ts +13 -1
- package/src/exec/env.ts +7 -2
- package/src/exec/executor.ts +154 -10
- package/src/exec/index.ts +6 -2
- package/src/exec/kill-tree.ts +15 -0
- package/src/exec/local-executor.ts +1 -1
- package/src/exec/proc-sample.ts +3 -1
- package/src/exec/runner.ts +253 -54
- package/src/exec/sandbox-binds.ts +7 -3
- package/src/exec/sandbox-paths.ts +1 -1
- package/src/exec/sandbox-runtime.ts +211 -24
- package/src/exec/sandbox-violations.ts +52 -20
- package/src/graph/critical-path.ts +79 -0
- package/src/graph/default-build.ts +19 -0
- package/src/graph/definition.ts +38 -0
- package/src/graph/folds.ts +38 -5
- package/src/graph/index.ts +4 -1
- package/src/graph/priorities.ts +74 -8
- package/src/graph/scheduler.ts +270 -60
- package/src/graph/task-graph.ts +228 -35
- package/src/handoff.ts +97 -9
- package/src/history/failure-mode.ts +6 -3
- package/src/history/history.ts +167 -61
- package/src/history/index.ts +2 -0
- package/src/history/metrics.ts +90 -30
- package/src/index.ts +30 -13
- package/src/orchestrator/admission.ts +10 -6
- package/src/orchestrator/affected-tasks.ts +11 -6
- package/src/orchestrator/deferred-outputs.ts +2 -0
- package/src/orchestrator/doctor.ts +134 -42
- package/src/orchestrator/execute-args.ts +4 -1
- package/src/orchestrator/execute-task.ts +130 -38
- package/src/orchestrator/fingerprint-watch.ts +1 -1
- package/src/orchestrator/graph-memo.ts +140 -0
- package/src/orchestrator/graph-stage.ts +18 -1
- package/src/orchestrator/held-servers.ts +1 -1
- package/src/orchestrator/hit-restore.ts +217 -119
- package/src/orchestrator/index.ts +4 -3
- package/src/orchestrator/local-floor.ts +119 -25
- package/src/orchestrator/local-shortcircuit.ts +28 -4
- package/src/orchestrator/lockfile-claim.ts +96 -16
- package/src/orchestrator/miss-save.ts +42 -8
- package/src/orchestrator/options.ts +8 -6
- package/src/orchestrator/output-host.ts +43 -14
- package/src/orchestrator/output-log.ts +90 -10
- package/src/orchestrator/persistent-task.ts +84 -29
- package/src/orchestrator/persistent.ts +16 -4
- package/src/orchestrator/placement.ts +185 -17
- package/src/orchestrator/plan.ts +64 -7
- package/src/orchestrator/plugin-host.ts +214 -42
- package/src/orchestrator/plugin.ts +113 -37
- package/src/orchestrator/prepare.ts +168 -30
- package/src/orchestrator/projects.ts +65 -40
- package/src/orchestrator/remote-prefetch.ts +119 -25
- package/src/orchestrator/run-artifacts.ts +95 -32
- package/src/orchestrator/run-context.ts +39 -41
- package/src/orchestrator/run-failures.ts +65 -23
- package/src/orchestrator/run-id.ts +6 -3
- package/src/orchestrator/run-keys.ts +149 -13
- package/src/orchestrator/run-live.ts +294 -0
- package/src/orchestrator/run-lock.ts +13 -7
- package/src/orchestrator/run-records.ts +35 -29
- package/src/orchestrator/run-report.ts +3 -3
- package/src/orchestrator/run-steps.ts +58 -22
- package/src/orchestrator/run-stop.ts +22 -5
- package/src/orchestrator/run.ts +331 -88
- package/src/orchestrator/sandbox-fix.ts +7 -5
- package/src/orchestrator/sandbox-request.ts +70 -20
- package/src/orchestrator/shell-verdict.ts +5 -5
- package/src/orchestrator/signals.ts +26 -1
- package/src/orchestrator/stable-keys.ts +92 -8
- package/src/orchestrator/steps.ts +2 -2
- package/src/orchestrator/task-artifacts.ts +90 -26
- package/src/orchestrator/task-hash.ts +78 -13
- package/src/orchestrator/task-log-buffer.ts +72 -36
- package/src/orchestrator/task-process.ts +7 -1
- package/src/orchestrator/telemetry-host.ts +22 -2
- package/src/orchestrator/telemetry.ts +53 -21
- package/src/orchestrator/upstream.ts +15 -0
- package/src/report/events.ts +32 -13
- package/src/report/forecast.ts +31 -9
- package/src/report/framed-output.ts +141 -50
- package/src/report/index.ts +1 -0
- package/src/report/logger.ts +170 -179
- package/src/report/path-links.ts +12 -4
- package/src/report/plain-output.ts +13 -5
- package/src/report/status-line.ts +81 -53
- package/src/report/summary.ts +114 -32
- package/src/report/terminal-signals.ts +13 -2
- package/src/settings.ts +7 -6
- package/src/util/bun-version.ts +4 -1
- package/src/util/colors.ts +4 -2
- package/src/util/config-frame.ts +41 -4
- package/src/{graph → util}/dependency-spec.ts +22 -13
- package/src/util/edit-distance.ts +13 -1
- package/src/util/env-flag.ts +9 -0
- package/src/util/errors.ts +59 -3
- package/src/util/essential-env.ts +57 -0
- package/src/util/git-ignored.ts +1 -1
- package/src/util/hash.ts +35 -0
- package/src/util/hide-env.ts +45 -5
- package/src/util/index.ts +21 -5
- package/src/util/num.ts +8 -0
- package/src/util/paths.ts +91 -21
- package/src/util/report.ts +6 -1
- package/src/util/secret-mask.ts +62 -1
- package/src/util/secrets.ts +7 -2
- package/src/util/size.ts +11 -3
- package/src/util/tail.ts +6 -1
- package/src/util/timing.ts +13 -1
- package/src/util/ui.ts +61 -3
- package/src/util/which.ts +1 -1
- package/src/util/write-file.ts +67 -0
- package/src/workspace/affected.ts +710 -107
- package/src/workspace/bracket-literal.ts +84 -0
- package/src/workspace/canonical-dir.ts +37 -0
- package/src/workspace/config-bytecode.ts +17 -11
- package/src/workspace/config-eval.ts +29 -8
- package/src/workspace/config-imports.ts +199 -58
- package/src/workspace/config-schema.ts +242 -39
- package/src/workspace/filter.ts +37 -8
- package/src/workspace/index.ts +11 -1
- package/src/workspace/json-data.ts +3 -4
- package/src/workspace/lockfile.ts +57 -8
- package/src/workspace/migration.ts +69 -18
- package/src/workspace/package-graph.ts +5 -4
- package/src/workspace/project-loader.ts +226 -37
- package/src/workspace/repo-id.ts +1 -1
- package/src/workspace/workspace.ts +176 -33
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// A plugin's own entries in the cache the workspace declares (`ctx.cache`,
|
|
2
|
-
//
|
|
2
|
+
// 2026-10-14): the local store first, then each remote in declared
|
|
3
3
|
// order, under the run's policy and scope. The plugin speaks names; core
|
|
4
4
|
// derives the key, so no plugin reaches another's entries or a task's. An
|
|
5
5
|
// entry is written once and never replaced (the cache is immutable). Every
|
|
@@ -7,9 +7,9 @@
|
|
|
7
7
|
// warnings): what a plugin keeps here is a hint.
|
|
8
8
|
|
|
9
9
|
import type { Database } from 'bun:sqlite'
|
|
10
|
-
import { xxh3hex } from '../util/index.js'
|
|
10
|
+
import { thrownMessage, xxh3hex } from '../util/index.js'
|
|
11
11
|
import type { CachePolicy } from './policy.js'
|
|
12
|
-
import type
|
|
12
|
+
import { sayRemoteFault, type ResolvedRemote } from './remote-lane.js'
|
|
13
13
|
import { scopedKey } from './scoped-cache.js'
|
|
14
14
|
import { runReadKeys } from './run-store.js'
|
|
15
15
|
|
|
@@ -73,6 +73,7 @@ async function capped(body: unknown, max = PLUGIN_ENTRY_MAX_BYTES): Promise<Uint
|
|
|
73
73
|
if (done) break
|
|
74
74
|
size += value.byteLength
|
|
75
75
|
if (size > max) {
|
|
76
|
+
// quiet: the body is refused for its size, said by the miss; its cancel adds nothing
|
|
76
77
|
await reader.cancel().catch(() => {})
|
|
77
78
|
return null
|
|
78
79
|
}
|
|
@@ -90,15 +91,30 @@ async function capped(body: unknown, max = PLUGIN_ENTRY_MAX_BYTES): Promise<Uint
|
|
|
90
91
|
|
|
91
92
|
/**
|
|
92
93
|
* `plugin`'s view of the cache. The local store never knows a scope
|
|
93
|
-
* (
|
|
94
|
+
* (2026-10-17): it keeps every entry under its name's key. A scoped
|
|
94
95
|
* run writes a remote only under the scope's key, and reads one key: the
|
|
95
|
-
* scope's, or with `scoped` false the default (
|
|
96
|
+
* scope's, or with `scoped` false the default (2026-10-18: no request
|
|
96
97
|
* twice).
|
|
97
98
|
*/
|
|
98
99
|
export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache {
|
|
99
100
|
const { db, policy, scope } = host
|
|
100
|
-
|
|
101
|
+
// A plugin's entries are not task artifacts: a policy that reads no
|
|
102
|
+
// remote artifact (a cold run, `--force`) still reads them, so the run is
|
|
103
|
+
// planned on the default branch's timings. A cold CI run asked workers
|
|
104
|
+
// for 128 tasks with 0 measured, at 25% of their cores (2026-10-23).
|
|
105
|
+
const readers = host.remotes.filter((r) => r.read || r.write)
|
|
101
106
|
const writers = host.remotes.filter((r) => r.write)
|
|
107
|
+
// Each remote's fault said once per kind: unsaid, a remote down left DTE
|
|
108
|
+
// planning on no timings and no line said why.
|
|
109
|
+
const said = new Map<ResolvedRemote, Set<string>>()
|
|
110
|
+
const fault = (r: ResolvedRemote, op: string, what: string, err: unknown): void => {
|
|
111
|
+
const message = thrownMessage(err)
|
|
112
|
+
const seen = said.get(r) ?? new Set<string>()
|
|
113
|
+
said.set(r, seen)
|
|
114
|
+
if (seen.has(`${op}\0${message}`)) return
|
|
115
|
+
seen.add(`${op}\0${message}`)
|
|
116
|
+
sayRemoteFault(r, new Error(`${op} of ${plugin}'s ${what} failed: ${message}`, { cause: err }))
|
|
117
|
+
}
|
|
102
118
|
const readKey = (name: string, scoped: boolean): string => {
|
|
103
119
|
const key = entryKey(plugin, name)
|
|
104
120
|
return scope === undefined || !scoped ? key : scopedKey(scope, key)
|
|
@@ -115,6 +131,7 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
115
131
|
bytes: Uint8Array
|
|
116
132
|
} | null
|
|
117
133
|
return row === null ? null : new Uint8Array(row.bytes)
|
|
134
|
+
// quiet: the local index, not a remote: an unreadable one holds nothing
|
|
118
135
|
} catch {
|
|
119
136
|
return null
|
|
120
137
|
}
|
|
@@ -127,6 +144,7 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
127
144
|
.query(`SELECT key FROM plugin_blobs WHERE key IN (${keys.map(() => '?').join(', ')})`)
|
|
128
145
|
.all(...keys) as Array<{ key: string }>
|
|
129
146
|
for (const r of rows) found.add(r.key)
|
|
147
|
+
// quiet: the local index, not a remote
|
|
130
148
|
} catch {
|
|
131
149
|
// an unreadable index holds nothing
|
|
132
150
|
}
|
|
@@ -139,6 +157,7 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
139
157
|
bytes,
|
|
140
158
|
Date.now(),
|
|
141
159
|
)
|
|
160
|
+
// quiet: the local index, not a remote
|
|
142
161
|
} catch {
|
|
143
162
|
// a read-only or busy index keeps nothing
|
|
144
163
|
}
|
|
@@ -150,16 +169,17 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
150
169
|
const hit = localGet(local)
|
|
151
170
|
if (hit !== null) return hit
|
|
152
171
|
const key = readKey(name, scoped)
|
|
153
|
-
for (const
|
|
172
|
+
for (const r of readers) {
|
|
154
173
|
try {
|
|
155
|
-
const got = await remote.get(key)
|
|
174
|
+
const got = await r.remote.get(key)
|
|
156
175
|
if (got === null) continue
|
|
157
176
|
const bytes = await capped(got.body)
|
|
158
177
|
if (bytes === null) continue
|
|
159
178
|
if (policy.localWrite) localPut(local, bytes)
|
|
160
179
|
return bytes
|
|
161
|
-
} catch {
|
|
180
|
+
} catch (err) {
|
|
162
181
|
// a failing remote is a miss there
|
|
182
|
+
fault(r, 'get', name, err)
|
|
163
183
|
}
|
|
164
184
|
}
|
|
165
185
|
return null
|
|
@@ -169,11 +189,12 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
169
189
|
if (bytes.byteLength === 0 || bytes.byteLength > PLUGIN_ENTRY_MAX_BYTES) return
|
|
170
190
|
const key = writeKey(name)
|
|
171
191
|
if (policy.localWrite) localPut(entryKey(plugin, name), bytes)
|
|
172
|
-
for (const
|
|
192
|
+
for (const r of writers) {
|
|
173
193
|
try {
|
|
174
|
-
await remote.put(key, new Blob([bytes]), { durationMs: 0 })
|
|
175
|
-
} catch {
|
|
194
|
+
await r.remote.put(key, new Blob([bytes]), { durationMs: 0 })
|
|
195
|
+
} catch (err) {
|
|
176
196
|
// the other layers still keep it
|
|
197
|
+
fault(r, 'put', name, err)
|
|
177
198
|
}
|
|
178
199
|
}
|
|
179
200
|
},
|
|
@@ -182,12 +203,13 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
182
203
|
if (localHas([entryKey(plugin, name)]).size > 0) return true
|
|
183
204
|
const key = readKey(name, scoped)
|
|
184
205
|
let answered = policy.localRead
|
|
185
|
-
for (const
|
|
206
|
+
for (const r of readers) {
|
|
186
207
|
try {
|
|
187
|
-
if (await remote.has(key)) return true
|
|
208
|
+
if (await r.remote.has(key)) return true
|
|
188
209
|
answered = true
|
|
189
|
-
} catch {
|
|
210
|
+
} catch (err) {
|
|
190
211
|
// no answer from this remote
|
|
212
|
+
fault(r, 'has', name, err)
|
|
191
213
|
}
|
|
192
214
|
}
|
|
193
215
|
return answered ? false : null
|
|
@@ -198,17 +220,20 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
198
220
|
for (const name of names) byKey.set(readKey(name, scoped), name)
|
|
199
221
|
const keys = [...byKey.keys()]
|
|
200
222
|
const found = new Set<string>()
|
|
201
|
-
|
|
202
|
-
|
|
223
|
+
// The local store keys entries unscoped (`entryKey`), not by `readKey`.
|
|
224
|
+
const byLocal = new Map(names.map((n) => [entryKey(plugin, n), n]))
|
|
225
|
+
for (const key of localHas([...byLocal.keys()])) found.add(byLocal.get(key)!)
|
|
226
|
+
for (const r of readers) {
|
|
203
227
|
const ask = keys.filter((k) => !found.has(byKey.get(k)!))
|
|
204
228
|
if (ask.length === 0) break
|
|
205
|
-
if (remote.hasMany === undefined) return null
|
|
229
|
+
if (r.remote.hasMany === undefined) return null
|
|
206
230
|
try {
|
|
207
|
-
const got: unknown = await remote.hasMany(ask)
|
|
231
|
+
const got: unknown = await r.remote.hasMany(ask)
|
|
208
232
|
if (!(got instanceof Map)) return null
|
|
209
233
|
for (const key of got.keys())
|
|
210
234
|
if (byKey.has(key as string)) found.add(byKey.get(key as string)!)
|
|
211
|
-
} catch {
|
|
235
|
+
} catch (err) {
|
|
236
|
+
fault(r, 'hasMany', `${ask.length} entries`, err)
|
|
212
237
|
return null
|
|
213
238
|
}
|
|
214
239
|
}
|
|
@@ -216,15 +241,16 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
|
|
|
216
241
|
},
|
|
217
242
|
|
|
218
243
|
async getRun(name) {
|
|
219
|
-
for (const
|
|
244
|
+
for (const r of readers) {
|
|
220
245
|
for (const key of runReadKeys(name, scope)) {
|
|
221
246
|
try {
|
|
222
|
-
const got = await remote.get(key)
|
|
247
|
+
const got = await r.remote.get(key)
|
|
223
248
|
if (got === null) continue
|
|
224
249
|
const bytes = await capped(got.body, Infinity)
|
|
225
250
|
if (bytes !== null) return bytes
|
|
226
|
-
} catch {
|
|
251
|
+
} catch (err) {
|
|
227
252
|
// a failing remote is a miss there
|
|
253
|
+
fault(r, 'get', name, err)
|
|
228
254
|
}
|
|
229
255
|
}
|
|
230
256
|
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// What the remotes said they hold, kept across runs and processes: `vx dte
|
|
2
|
+
// plan` and `vx run` are separate commands on GitLab, and a run trusts this
|
|
3
|
+
// at once instead of waiting on a has-many (2026-10-23). Only hits are
|
|
4
|
+
// kept: a key not here is a miss. Every has-many that answers rewrites the
|
|
5
|
+
// keys it asked, and a GET that finds nothing forgets its key.
|
|
6
|
+
|
|
7
|
+
import type { Database } from 'bun:sqlite'
|
|
8
|
+
import { inHashes } from './schema.js'
|
|
9
|
+
|
|
10
|
+
export interface RemoteHit {
|
|
11
|
+
readonly size?: number
|
|
12
|
+
readonly durationMs?: number
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export class RemoteHits {
|
|
16
|
+
constructor(
|
|
17
|
+
private readonly db: Database,
|
|
18
|
+
/** Off for a reading verb or an index this user cannot write. */
|
|
19
|
+
private readonly write: boolean,
|
|
20
|
+
) {}
|
|
21
|
+
|
|
22
|
+
/** The keys of `hashes` a remote last said it holds. */
|
|
23
|
+
get(hashes: readonly string[]): Map<string, RemoteHit> {
|
|
24
|
+
const out = new Map<string, RemoteHit>()
|
|
25
|
+
if (hashes.length === 0) return out
|
|
26
|
+
const { test, params } = inHashes(hashes)
|
|
27
|
+
try {
|
|
28
|
+
const rows = this.db
|
|
29
|
+
.query(`SELECT hash, size, duration_ms FROM remote_hits WHERE hash ${test}`)
|
|
30
|
+
.all(...params) as Array<{ hash: string; size: number | null; duration_ms: number | null }>
|
|
31
|
+
for (const r of rows)
|
|
32
|
+
out.set(r.hash, {
|
|
33
|
+
...(r.size === null ? {} : { size: r.size }),
|
|
34
|
+
...(r.duration_ms === null ? {} : { durationMs: r.duration_ms }),
|
|
35
|
+
})
|
|
36
|
+
} catch {
|
|
37
|
+
// An unreadable or busy index knows no hit.
|
|
38
|
+
}
|
|
39
|
+
return out
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** A has-many's whole answer: `found` are hits, the rest of `asked` no longer are. */
|
|
43
|
+
record(asked: readonly string[], found: ReadonlyMap<string, RemoteHit>, now = Date.now()): void {
|
|
44
|
+
if (!this.write || asked.length === 0) return
|
|
45
|
+
try {
|
|
46
|
+
const upsert = this.db.query(
|
|
47
|
+
'INSERT OR REPLACE INTO remote_hits(hash, size, duration_ms, seen_at) VALUES (?, ?, ?, ?)',
|
|
48
|
+
)
|
|
49
|
+
const gone = asked.filter((h) => !found.has(h))
|
|
50
|
+
this.db.transaction(() => {
|
|
51
|
+
for (const [hash, info] of found)
|
|
52
|
+
upsert.run(hash, info.size ?? null, info.durationMs ?? null, now)
|
|
53
|
+
if (gone.length > 0) {
|
|
54
|
+
const { test, params } = inHashes(gone)
|
|
55
|
+
this.db.query(`DELETE FROM remote_hits WHERE hash ${test}`).run(...params)
|
|
56
|
+
}
|
|
57
|
+
})()
|
|
58
|
+
} catch {
|
|
59
|
+
// A busy index keeps nothing: the next run asks again.
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** No remote served `hash`: a miss from now on. */
|
|
64
|
+
forget(hash: string): void {
|
|
65
|
+
if (!this.write) return
|
|
66
|
+
try {
|
|
67
|
+
this.db.query('DELETE FROM remote_hits WHERE hash = ?').run(hash)
|
|
68
|
+
} catch {
|
|
69
|
+
// A busy index keeps the row: its next GET forgets it.
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
package/src/cache/remote-lane.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// class with the endpoint named. The order across remotes, the local store
|
|
5
5
|
// and provenance are `LayeredCache`'s; nothing here knows another remote.
|
|
6
6
|
|
|
7
|
-
import { machineParallelism } from '../util/index.js'
|
|
7
|
+
import { machineParallelism, thrownMessage, ui } from '../util/index.js'
|
|
8
8
|
import type { RemoteKeyTraffic, RemoteTraffic } from './cache.js'
|
|
9
9
|
import type { ArtifactInfo, RemoteCacheLayer } from './layered-cache.js'
|
|
10
10
|
|
|
@@ -20,6 +20,19 @@ export interface ResolvedRemote {
|
|
|
20
20
|
readonly onError?: (err: Error) => void
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
/**
|
|
24
|
+
* A fault of `r` that a caller outside its lane degrades to a miss, said as
|
|
25
|
+
* the lane says its own: to its `onError`, else stderr. Reporting never
|
|
26
|
+
* escalates.
|
|
27
|
+
*/
|
|
28
|
+
export function sayRemoteFault(r: ResolvedRemote, e: Error): void {
|
|
29
|
+
try {
|
|
30
|
+
if (r.onError) r.onError(e)
|
|
31
|
+
else ui.say('warn', `[vx] remote cache: ${e.message}\n`)
|
|
32
|
+
// quiet: a throwing reporter must never fail the run
|
|
33
|
+
} catch {}
|
|
34
|
+
}
|
|
35
|
+
|
|
23
36
|
/**
|
|
24
37
|
* Requests at once to a remote, by default: pnpm's network concurrency
|
|
25
38
|
* (Bun's is a fixed 64). A request waits on the network, not a core, so
|
|
@@ -84,7 +97,7 @@ function printableEndpoint(endpoint: unknown): string | undefined {
|
|
|
84
97
|
|
|
85
98
|
/**
|
|
86
99
|
* What makes two failures "the same way": the error's `code` when it
|
|
87
|
-
* carries one (a gRPC status, an errno, Bun's `
|
|
100
|
+
* carries one (a gRPC status, an errno, Bun's `ECONNREFUSED`), because
|
|
88
101
|
* gRPC writes the elapsed time into its message and no two deadline messages
|
|
89
102
|
* match; else the message with the artifact's hash taken out, so a 500 on
|
|
90
103
|
* one hash and a 500 on the next are one class. The operation is not part of
|
|
@@ -115,6 +128,8 @@ export class RemoteLane {
|
|
|
115
128
|
private readonly absent = new Set<string>()
|
|
116
129
|
/** What a `hasMany` said this remote holds, as `absent` records the rest. */
|
|
117
130
|
private readonly held = new Map<string, ArtifactInfo>()
|
|
131
|
+
/** Keys whose bytes here did not read back (`distrust`). */
|
|
132
|
+
private readonly distrusted = new Set<string>()
|
|
118
133
|
|
|
119
134
|
/**
|
|
120
135
|
* A remote the run writes but does not read (`--force`): which of the
|
|
@@ -229,22 +244,62 @@ export class RemoteLane {
|
|
|
229
244
|
}
|
|
230
245
|
|
|
231
246
|
/**
|
|
232
|
-
*
|
|
233
|
-
*
|
|
247
|
+
* `hashes` asked one by one (`has`), `concurrency` at a time: what a
|
|
248
|
+
* remote that answered no batch query holds, recording the rest as
|
|
249
|
+
* absent. Null when any probe failed: then the answer is not complete.
|
|
250
|
+
*/
|
|
251
|
+
async hasEach(
|
|
252
|
+
hashes: readonly string[],
|
|
253
|
+
concurrency: number,
|
|
254
|
+
): Promise<Map<string, ArtifactInfo> | null> {
|
|
255
|
+
const out = new Map<string, ArtifactInfo>()
|
|
256
|
+
const absent: string[] = []
|
|
257
|
+
let failed = false
|
|
258
|
+
let next = 0
|
|
259
|
+
const one = async (): Promise<void> => {
|
|
260
|
+
while (!failed && next < hashes.length) {
|
|
261
|
+
const h = hashes[next++]!
|
|
262
|
+
let found: unknown
|
|
263
|
+
try {
|
|
264
|
+
found = this.held.get(h) ?? (this.absent.has(h) ? null : await this.remote.has(h))
|
|
265
|
+
} catch (err) {
|
|
266
|
+
this.report('probe', h, err)
|
|
267
|
+
failed = true
|
|
268
|
+
return
|
|
269
|
+
}
|
|
270
|
+
if (found === null || found === undefined) absent.push(h)
|
|
271
|
+
else out.set(h, artifactInfo(found) ?? {})
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
await Promise.all(Array.from({ length: Math.max(1, concurrency) }, one))
|
|
275
|
+
if (failed) return null
|
|
276
|
+
for (const h of absent) this.absent.add(h)
|
|
277
|
+
for (const [h, info] of out) this.held.set(h, info)
|
|
278
|
+
return out
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* The artifact's body, null on a miss or a known absence (unless
|
|
283
|
+
* `fresh`: saved since the probe), undefined on a failure or a
|
|
284
|
+
* malformed answer: the key may still be there.
|
|
234
285
|
*/
|
|
235
286
|
async get(
|
|
236
287
|
hash: string,
|
|
237
288
|
fresh = false,
|
|
238
|
-
): Promise<{ body: Blob | Response; durationMs?: number } | null> {
|
|
289
|
+
): Promise<{ body: Blob | Response; durationMs?: number } | null | undefined> {
|
|
239
290
|
if (!fresh && this.absent.has(hash)) return null
|
|
240
291
|
let result: unknown
|
|
241
292
|
try {
|
|
242
293
|
result = await this.remote.get(hash)
|
|
243
294
|
} catch (err) {
|
|
244
295
|
this.report('download', hash, err)
|
|
296
|
+
return undefined
|
|
297
|
+
}
|
|
298
|
+
if (!result) {
|
|
299
|
+
// Gone, whatever a probe said: the re-run's save uploads it.
|
|
300
|
+
this.held.delete(hash)
|
|
245
301
|
return null
|
|
246
302
|
}
|
|
247
|
-
if (!result) return null
|
|
248
303
|
if (typeof result !== 'object' || !isBody((result as { body?: unknown }).body)) {
|
|
249
304
|
const shape =
|
|
250
305
|
typeof result !== 'object'
|
|
@@ -257,7 +312,7 @@ export class RemoteLane {
|
|
|
257
312
|
`get() resolved ${shape} (expected { body: Blob | Response, durationMs } or null)`,
|
|
258
313
|
),
|
|
259
314
|
)
|
|
260
|
-
return
|
|
315
|
+
return undefined
|
|
261
316
|
}
|
|
262
317
|
// It holds the key: a save of it after this restore uploads nothing.
|
|
263
318
|
const got = result as { body: Blob | Response; durationMs?: number }
|
|
@@ -265,6 +320,11 @@ export class RemoteLane {
|
|
|
265
320
|
return got
|
|
266
321
|
}
|
|
267
322
|
|
|
323
|
+
/** `hash`'s bytes here are unreadable: a save of it uploads, held or not. */
|
|
324
|
+
distrust(hash: string): void {
|
|
325
|
+
this.distrusted.add(hash)
|
|
326
|
+
}
|
|
327
|
+
|
|
268
328
|
/**
|
|
269
329
|
* Start the batch probe `presence` reads. A lane that reads, never
|
|
270
330
|
* writes, or cannot batch (no `hasMany`) ignores it: each upload then
|
|
@@ -293,27 +353,41 @@ export class RemoteLane {
|
|
|
293
353
|
return (await this.has(hash)) !== null
|
|
294
354
|
}
|
|
295
355
|
|
|
296
|
-
/**
|
|
356
|
+
/**
|
|
357
|
+
* Queue a PUT in this remote's bounded pool; `body` is asked for when the
|
|
358
|
+
* job starts. Resolves true once this remote holds `hash`: put now, or held already.
|
|
359
|
+
* `absent`: no remote held it when the run asked, so none is asked again.
|
|
360
|
+
*/
|
|
297
361
|
upload(
|
|
298
362
|
hash: string,
|
|
299
363
|
durationMs: number,
|
|
300
364
|
body: () => Promise<{ body: Blob; release?: () => Promise<void> }>,
|
|
301
365
|
tally: { count: number; bytes: number; ms: number; failed: number },
|
|
302
|
-
|
|
366
|
+
absent = false,
|
|
367
|
+
): Promise<boolean> {
|
|
303
368
|
const key = this.trafficOf(hash)
|
|
304
369
|
key.uploading++
|
|
305
370
|
this.traffic.inFlight++
|
|
371
|
+
const { promise, resolve } = Promise.withResolvers<boolean>()
|
|
306
372
|
this.uploadQueue.push(async () => {
|
|
307
373
|
let release: (() => Promise<void>) | undefined
|
|
374
|
+
let holds = false
|
|
308
375
|
try {
|
|
309
376
|
if (this.stopped) return
|
|
310
|
-
|
|
311
|
-
|
|
377
|
+
// A key whose bytes here did not read back is sent over them.
|
|
378
|
+
const distrusted = this.distrusted.has(hash)
|
|
379
|
+
const known = !distrusted && (absent || this.absent.has(hash))
|
|
380
|
+
if (!distrusted) {
|
|
381
|
+
if (this.held.has(hash)) return void (holds = true)
|
|
382
|
+
if (!known && !this.read && (await this.holds(hash))) return void (holds = true)
|
|
383
|
+
}
|
|
312
384
|
if (this.stopped) return
|
|
313
385
|
const start = performance.now()
|
|
314
386
|
const got = await body()
|
|
315
387
|
release = got.release
|
|
316
|
-
|
|
388
|
+
if (known) this.remote.missing?.(hash)
|
|
389
|
+
if ((await this.remote.put(hash, got.body, { durationMs })) === false) return
|
|
390
|
+
holds = true
|
|
317
391
|
tally.count++
|
|
318
392
|
tally.bytes += got.body.size
|
|
319
393
|
tally.ms += Math.round(performance.now() - start)
|
|
@@ -329,9 +403,11 @@ export class RemoteLane {
|
|
|
329
403
|
// stopped first): no traffic, so the footer counts no upload.
|
|
330
404
|
if (key.uploading === 0 && !key.landed && key.failures.length === 0)
|
|
331
405
|
this.traffic.keys.delete(hash)
|
|
406
|
+
resolve(holds)
|
|
332
407
|
}
|
|
333
408
|
})
|
|
334
409
|
this.pump()
|
|
410
|
+
return promise
|
|
335
411
|
}
|
|
336
412
|
|
|
337
413
|
/** Settles when the run is stopped. */
|
|
@@ -381,7 +457,7 @@ export class RemoteLane {
|
|
|
381
457
|
* server in it (item 749).
|
|
382
458
|
*/
|
|
383
459
|
report(op: RemoteOp, subject: string | { batch: string }, err: unknown): void {
|
|
384
|
-
const e = err instanceof Error ? err : new Error(
|
|
460
|
+
const e = err instanceof Error ? err : new Error(thrownMessage(err))
|
|
385
461
|
const hash = typeof subject === 'string' ? subject : undefined
|
|
386
462
|
// A batch probe's failure is no one task's: each of its hashes is asked
|
|
387
463
|
// again on its own, and that ask is the one counted.
|
|
@@ -435,7 +511,7 @@ export class RemoteLane {
|
|
|
435
511
|
// hiccup into a run failure.
|
|
436
512
|
try {
|
|
437
513
|
if (this.onError) this.onError(e)
|
|
438
|
-
else
|
|
514
|
+
else ui.say('warn', `[vx] remote cache: ${e.message}\n`)
|
|
439
515
|
} catch {
|
|
440
516
|
// swallow — reporting must never escalate
|
|
441
517
|
}
|
package/src/cache/run-history.ts
CHANGED
|
@@ -6,19 +6,45 @@
|
|
|
6
6
|
import type { Database, SQLQueryBindings } from 'bun:sqlite'
|
|
7
7
|
import { lazyStatement } from './schema.js'
|
|
8
8
|
import type { InvocationRecord, RunRecord } from './layer.js'
|
|
9
|
+
import { isMeasure } from '../util/index.js'
|
|
9
10
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
11
|
+
/** The `runs` columns `bindRun` fills, in its order. */
|
|
12
|
+
const RUN_COLUMNS = [
|
|
13
|
+
'hash',
|
|
14
|
+
'project',
|
|
15
|
+
'task',
|
|
16
|
+
'status',
|
|
17
|
+
'exit_code',
|
|
18
|
+
'duration_ms',
|
|
19
|
+
'forward_args',
|
|
20
|
+
'started_at',
|
|
21
|
+
'ended_at',
|
|
22
|
+
'run_id',
|
|
23
|
+
'cpu_ms',
|
|
24
|
+
'peak_rss_bytes',
|
|
25
|
+
'wallclock_start_ns',
|
|
26
|
+
'wallclock_end_ns',
|
|
27
|
+
'cache_hit',
|
|
28
|
+
'attempts',
|
|
29
|
+
'cached',
|
|
30
|
+
'blocked_by',
|
|
31
|
+
'timed_out',
|
|
32
|
+
'sandbox_violations',
|
|
33
|
+
'not_ready',
|
|
34
|
+
'restored',
|
|
35
|
+
'remote_error',
|
|
36
|
+
'peak_cores',
|
|
37
|
+
'avg_rss_bytes',
|
|
38
|
+
'placement',
|
|
39
|
+
'placement_cores',
|
|
40
|
+
'placement_memory',
|
|
41
|
+
'keyless_by',
|
|
42
|
+
'def',
|
|
43
|
+
'cpu_share',
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
const INSERT_RUNS = `INSERT INTO runs(${RUN_COLUMNS.join(', ')}) VALUES `
|
|
47
|
+
const RUN_TUPLE = `(${RUN_COLUMNS.map(() => '?').join(', ')})`
|
|
22
48
|
|
|
23
49
|
/** An INSERT of `n` `runs` rows. */
|
|
24
50
|
function insertRunsSql(n: number): string {
|
|
@@ -26,14 +52,15 @@ function insertRunsSql(n: number): string {
|
|
|
26
52
|
}
|
|
27
53
|
|
|
28
54
|
/**
|
|
29
|
-
* Rows per INSERT:
|
|
30
|
-
* ceiling
|
|
31
|
-
*
|
|
55
|
+
* Rows per INSERT: as many as stay within 999 bound variables, the oldest
|
|
56
|
+
* default ceiling, read off the column count (33 rows were sized for 30
|
|
57
|
+
* columns and bound 1,023 at 31). A statement per row was 8.7 ms of 1,000
|
|
58
|
+
* rows, 40 at a time 6.4 (2026-10-03).
|
|
32
59
|
*/
|
|
33
|
-
const RUNS_PER_INSERT =
|
|
60
|
+
const RUNS_PER_INSERT = Math.floor(999 / RUN_COLUMNS.length)
|
|
34
61
|
|
|
35
|
-
/** Rows per `task_durations` INSERT:
|
|
36
|
-
const DURATIONS_PER_INSERT =
|
|
62
|
+
/** Rows per `task_durations` INSERT: 7 variables each, under 999. */
|
|
63
|
+
const DURATIONS_PER_INSERT = 140
|
|
37
64
|
|
|
38
65
|
/** Rows per `task_outcomes` INSERT: 6 variables each, under 999. */
|
|
39
66
|
const OUTCOMES_PER_INSERT = 160
|
|
@@ -63,6 +90,20 @@ const PASSED_HASHES = `
|
|
|
63
90
|
/** A local run's `task_durations.source`: its branch, `''` when it had none. */
|
|
64
91
|
export const branchSource = (branch: string | null): string => `branch:${branch ?? ''}`
|
|
65
92
|
|
|
93
|
+
/**
|
|
94
|
+
* How long a run would have taken alone: on a crowded worker
|
|
95
|
+
* (`cpuShare` < 1) each runnable thread waited for a core. A task that
|
|
96
|
+
* kept `n` cores busy lost its CPU's wait spread over them, under one
|
|
97
|
+
* core all of it. vx-dte#test (20 s CPU in 75 s) took 81-102 s beside
|
|
98
|
+
* test shards and the plan, reading those, said 118 s for a 92 s run
|
|
99
|
+
* (main fd5c72e, 2026-10-23).
|
|
100
|
+
*/
|
|
101
|
+
function aloneMs(r: Pick<RunRecord, 'durationMs' | 'cpuMs' | 'cpuShare'>): number {
|
|
102
|
+
const { durationMs: d, cpuMs: c, cpuShare: s } = r
|
|
103
|
+
if (c === undefined || s === undefined || !(s > 0 && s < 1) || !(d > 0)) return d
|
|
104
|
+
return Math.round(d - (c / s - c) / Math.max(1, c / (d * s)))
|
|
105
|
+
}
|
|
106
|
+
|
|
66
107
|
export class RunHistory {
|
|
67
108
|
private readonly insertInvocation: ReturnType<Database['prepare']>
|
|
68
109
|
|
|
@@ -106,16 +147,19 @@ export class RunHistory {
|
|
|
106
147
|
.immediate()
|
|
107
148
|
}
|
|
108
149
|
|
|
109
|
-
/** One `task_durations` row per executed success: what a forecast reads. */
|
|
150
|
+
/** One `task_durations` row per executed success, as if run alone: what a forecast reads. */
|
|
110
151
|
private insertDurations(runs: readonly RunRecord[], source: string, sha: string | null): void {
|
|
111
|
-
const done = runs.filter(
|
|
152
|
+
const done = runs.filter(
|
|
153
|
+
(r) => r.status === 'success' && r.cacheHit !== true && isMeasure(aloneMs(r)),
|
|
154
|
+
)
|
|
112
155
|
for (let i = 0; i < done.length; i += DURATIONS_PER_INSERT) {
|
|
113
156
|
const block = done.slice(i, i + DURATIONS_PER_INSERT)
|
|
114
157
|
const params: SQLQueryBindings[] = []
|
|
115
|
-
for (const r of block)
|
|
158
|
+
for (const r of block)
|
|
159
|
+
params.push(r.project, r.task, source, sha, aloneMs(r), r.endedAt, r.def ?? null)
|
|
116
160
|
this.db
|
|
117
161
|
.query(
|
|
118
|
-
`INSERT INTO task_durations(project, task, source, sha, duration_ms, recorded_at) VALUES ${Array(block.length).fill('(?, ?, ?, ?, ?, ?)').join(', ')}`,
|
|
162
|
+
`INSERT INTO task_durations(project, task, source, sha, duration_ms, recorded_at, def) VALUES ${Array(block.length).fill('(?, ?, ?, ?, ?, ?, ?)').join(', ')}`,
|
|
119
163
|
)
|
|
120
164
|
.run(...params)
|
|
121
165
|
}
|
|
@@ -171,7 +215,7 @@ export class RunHistory {
|
|
|
171
215
|
for (let i = 0; i < runs.length; i += RUNS_PER_INSERT) {
|
|
172
216
|
const block = runs.slice(i, i + RUNS_PER_INSERT)
|
|
173
217
|
const params: SQLQueryBindings[] = []
|
|
174
|
-
for (const r of block)
|
|
218
|
+
for (const r of block) for (const v of bindRun(r, digest)) params.push(v)
|
|
175
219
|
// `query` caches by text: one statement for every full block.
|
|
176
220
|
this.db.query(insertRunsSql(block.length)).run(...params)
|
|
177
221
|
}
|
|
@@ -234,6 +278,7 @@ export class RunHistory {
|
|
|
234
278
|
this.db.prepare('DELETE FROM task_durations WHERE recorded_at < ?').run(cutoff)
|
|
235
279
|
this.db.prepare('DELETE FROM task_outcomes WHERE recorded_at < ?').run(cutoff)
|
|
236
280
|
this.db.prepare('DELETE FROM plugin_blobs WHERE at < ?').run(cutoff)
|
|
281
|
+
this.db.prepare('DELETE FROM remote_hits WHERE seen_at < ?').run(cutoff)
|
|
237
282
|
}
|
|
238
283
|
}
|
|
239
284
|
|
|
@@ -274,6 +319,8 @@ function bindRun(run: RunRecord, digestValue: (v: string) => string): SQLQueryBi
|
|
|
274
319
|
run.placementCores ?? null,
|
|
275
320
|
run.placementMemoryBytes ?? null,
|
|
276
321
|
run.keylessBy ?? null,
|
|
322
|
+
run.def ?? null,
|
|
323
|
+
run.cpuShare ?? null,
|
|
277
324
|
]
|
|
278
325
|
}
|
|
279
326
|
|
package/src/cache/run-store.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// What core keeps of a run that writes a remote (
|
|
1
|
+
// What core keeps of a run that writes a remote (2026-10-15): a failed
|
|
2
2
|
// task's log and the files its outputs left, an uncached task's log. Names
|
|
3
3
|
// come from the run id and task (`run/<id>/<task>/log`); each key is derived
|
|
4
4
|
// in a namespace no task key or plugin entry shares. Remote only: locally
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// the cache is free). Plugins read them by name (`ctx.cache.getRun`).
|
|
7
7
|
|
|
8
8
|
import { xxh3hex } from '../util/index.js'
|
|
9
|
-
import type
|
|
9
|
+
import { sayRemoteFault, type ResolvedRemote } from './remote-lane.js'
|
|
10
10
|
import { scopedKey } from './scoped-cache.js'
|
|
11
11
|
import { tarPackInto, tarSize } from './tar-stream.js'
|
|
12
12
|
|
|
@@ -30,8 +30,8 @@ export function writesRunStore(remotes: readonly ResolvedRemote[]): boolean {
|
|
|
30
30
|
|
|
31
31
|
/**
|
|
32
32
|
* Store `bytes` under `name` in every remote the policy writes. An entry is
|
|
33
|
-
* written once; a remote's failure leaves the others theirs
|
|
34
|
-
* a run keeps here is a record, never its verdict.
|
|
33
|
+
* written once; a remote's failure is said and leaves the others theirs:
|
|
34
|
+
* what a run keeps here is a record, never its verdict.
|
|
35
35
|
*/
|
|
36
36
|
export async function putRunEntry(
|
|
37
37
|
remotes: readonly ResolvedRemote[],
|
|
@@ -41,16 +41,17 @@ export async function putRunEntry(
|
|
|
41
41
|
): Promise<boolean> {
|
|
42
42
|
const key = scope === undefined ? runKey(name) : scopedKey(scope, runKey(name))
|
|
43
43
|
let kept = false
|
|
44
|
-
for (const
|
|
45
|
-
if (!write) continue
|
|
44
|
+
for (const r of remotes) {
|
|
45
|
+
if (!r.write) continue
|
|
46
46
|
try {
|
|
47
|
-
await remote.put(key, new Blob([bytes]), {
|
|
47
|
+
const put = await r.remote.put(key, new Blob([bytes]), {
|
|
48
48
|
durationMs: 0,
|
|
49
49
|
...(scope === undefined ? {} : { scope }),
|
|
50
50
|
})
|
|
51
|
-
kept
|
|
52
|
-
|
|
53
|
-
|
|
51
|
+
// `false`: the remote sent nothing, so nothing is kept there.
|
|
52
|
+
if (put !== false) kept = true
|
|
53
|
+
} catch (err) {
|
|
54
|
+
sayRemoteFault(r, err instanceof Error ? err : new Error(String(err)))
|
|
54
55
|
}
|
|
55
56
|
}
|
|
56
57
|
return kept
|