@vzn/vx 0.0.646 → 0.0.649

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/README.md +29 -22
  2. package/launcher.cjs +9 -2
  3. package/launcher.sh +2 -2
  4. package/package.json +5 -5
  5. package/schemas/cache.json +5 -5
  6. package/schemas/error.json +1 -1
  7. package/schemas/info.json +16 -2
  8. package/schemas/last.json +112 -5
  9. package/schemas/plan.json +28 -2
  10. package/schemas/show.json +10 -1
  11. package/skills/vx/SKILL.md +18 -2
  12. package/src/bin.ts +53 -12
  13. package/src/cache/archive.ts +324 -23
  14. package/src/cache/cache.ts +268 -53
  15. package/src/cache/git-inputs.ts +219 -87
  16. package/src/cache/index.ts +3 -2
  17. package/src/cache/inputs.ts +460 -78
  18. package/src/cache/key-fold.ts +15 -6
  19. package/src/cache/layer.ts +49 -9
  20. package/src/cache/layered-cache.ts +123 -58
  21. package/src/cache/linked-inputs.ts +1 -1
  22. package/src/cache/output-index.ts +19 -3
  23. package/src/cache/plugin-cache.ts +49 -23
  24. package/src/cache/remote-hits.ts +72 -0
  25. package/src/cache/remote-lane.ts +90 -14
  26. package/src/cache/run-history.ts +70 -23
  27. package/src/cache/run-store.ts +11 -10
  28. package/src/cache/schema.ts +25 -6
  29. package/src/cache/tar-stream.ts +5 -0
  30. package/src/cache/umask.ts +1 -2
  31. package/src/cli/cache.ts +58 -27
  32. package/src/cli/completions.ts +12 -5
  33. package/src/cli/docs.ts +42 -16
  34. package/src/cli/help.ts +58 -19
  35. package/src/cli/index.ts +79 -33
  36. package/src/cli/info.ts +12 -6
  37. package/src/cli/init.ts +186 -31
  38. package/src/cli/last.ts +439 -45
  39. package/src/cli/lock.ts +197 -52
  40. package/src/cli/plan-format.ts +91 -49
  41. package/src/cli/plugin-commands.ts +70 -14
  42. package/src/cli/plugin-templates.ts +13 -13
  43. package/src/cli/registry.ts +209 -0
  44. package/src/cli/run.ts +109 -34
  45. package/src/cli/select.ts +143 -46
  46. package/src/cli/show.ts +59 -16
  47. package/src/cli/task-verb.ts +16 -6
  48. package/src/cli/upgrade.ts +151 -49
  49. package/src/cli/watch-cycle.ts +10 -5
  50. package/src/cli/watch-filter.ts +71 -3
  51. package/src/cli/watch-fs.ts +22 -6
  52. package/src/cli/watch-judge.ts +48 -6
  53. package/src/cli/watch.ts +163 -42
  54. package/src/cli/why.ts +211 -73
  55. package/src/cli/workspace-config.ts +54 -15
  56. package/src/config.ts +13 -1
  57. package/src/exec/env.ts +7 -2
  58. package/src/exec/executor.ts +154 -10
  59. package/src/exec/index.ts +6 -2
  60. package/src/exec/kill-tree.ts +15 -0
  61. package/src/exec/local-executor.ts +1 -1
  62. package/src/exec/proc-sample.ts +3 -1
  63. package/src/exec/runner.ts +253 -54
  64. package/src/exec/sandbox-binds.ts +7 -3
  65. package/src/exec/sandbox-paths.ts +1 -1
  66. package/src/exec/sandbox-runtime.ts +211 -24
  67. package/src/exec/sandbox-violations.ts +52 -20
  68. package/src/graph/critical-path.ts +79 -0
  69. package/src/graph/default-build.ts +19 -0
  70. package/src/graph/definition.ts +38 -0
  71. package/src/graph/folds.ts +38 -5
  72. package/src/graph/index.ts +4 -1
  73. package/src/graph/priorities.ts +74 -8
  74. package/src/graph/scheduler.ts +270 -60
  75. package/src/graph/task-graph.ts +228 -35
  76. package/src/handoff.ts +97 -9
  77. package/src/history/failure-mode.ts +6 -3
  78. package/src/history/history.ts +167 -61
  79. package/src/history/index.ts +2 -0
  80. package/src/history/metrics.ts +90 -30
  81. package/src/index.ts +30 -13
  82. package/src/orchestrator/admission.ts +10 -6
  83. package/src/orchestrator/affected-tasks.ts +11 -6
  84. package/src/orchestrator/deferred-outputs.ts +2 -0
  85. package/src/orchestrator/doctor.ts +134 -42
  86. package/src/orchestrator/execute-args.ts +4 -1
  87. package/src/orchestrator/execute-task.ts +130 -38
  88. package/src/orchestrator/fingerprint-watch.ts +1 -1
  89. package/src/orchestrator/graph-memo.ts +140 -0
  90. package/src/orchestrator/graph-stage.ts +18 -1
  91. package/src/orchestrator/held-servers.ts +1 -1
  92. package/src/orchestrator/hit-restore.ts +217 -119
  93. package/src/orchestrator/index.ts +4 -3
  94. package/src/orchestrator/local-floor.ts +119 -25
  95. package/src/orchestrator/local-shortcircuit.ts +28 -4
  96. package/src/orchestrator/lockfile-claim.ts +96 -16
  97. package/src/orchestrator/miss-save.ts +42 -8
  98. package/src/orchestrator/options.ts +8 -6
  99. package/src/orchestrator/output-host.ts +43 -14
  100. package/src/orchestrator/output-log.ts +90 -10
  101. package/src/orchestrator/persistent-task.ts +84 -29
  102. package/src/orchestrator/persistent.ts +16 -4
  103. package/src/orchestrator/placement.ts +185 -17
  104. package/src/orchestrator/plan.ts +64 -7
  105. package/src/orchestrator/plugin-host.ts +214 -42
  106. package/src/orchestrator/plugin.ts +113 -37
  107. package/src/orchestrator/prepare.ts +168 -30
  108. package/src/orchestrator/projects.ts +65 -40
  109. package/src/orchestrator/remote-prefetch.ts +119 -25
  110. package/src/orchestrator/run-artifacts.ts +95 -32
  111. package/src/orchestrator/run-context.ts +39 -41
  112. package/src/orchestrator/run-failures.ts +65 -23
  113. package/src/orchestrator/run-id.ts +6 -3
  114. package/src/orchestrator/run-keys.ts +149 -13
  115. package/src/orchestrator/run-live.ts +294 -0
  116. package/src/orchestrator/run-lock.ts +13 -7
  117. package/src/orchestrator/run-records.ts +35 -29
  118. package/src/orchestrator/run-report.ts +3 -3
  119. package/src/orchestrator/run-steps.ts +58 -22
  120. package/src/orchestrator/run-stop.ts +22 -5
  121. package/src/orchestrator/run.ts +331 -88
  122. package/src/orchestrator/sandbox-fix.ts +7 -5
  123. package/src/orchestrator/sandbox-request.ts +70 -20
  124. package/src/orchestrator/shell-verdict.ts +5 -5
  125. package/src/orchestrator/signals.ts +26 -1
  126. package/src/orchestrator/stable-keys.ts +92 -8
  127. package/src/orchestrator/steps.ts +2 -2
  128. package/src/orchestrator/task-artifacts.ts +90 -26
  129. package/src/orchestrator/task-hash.ts +78 -13
  130. package/src/orchestrator/task-log-buffer.ts +72 -36
  131. package/src/orchestrator/task-process.ts +7 -1
  132. package/src/orchestrator/telemetry-host.ts +22 -2
  133. package/src/orchestrator/telemetry.ts +53 -21
  134. package/src/orchestrator/upstream.ts +15 -0
  135. package/src/report/events.ts +32 -13
  136. package/src/report/forecast.ts +31 -9
  137. package/src/report/framed-output.ts +141 -50
  138. package/src/report/index.ts +1 -0
  139. package/src/report/logger.ts +170 -179
  140. package/src/report/path-links.ts +12 -4
  141. package/src/report/plain-output.ts +13 -5
  142. package/src/report/status-line.ts +81 -53
  143. package/src/report/summary.ts +114 -32
  144. package/src/report/terminal-signals.ts +13 -2
  145. package/src/settings.ts +7 -6
  146. package/src/util/bun-version.ts +4 -1
  147. package/src/util/colors.ts +4 -2
  148. package/src/util/config-frame.ts +41 -4
  149. package/src/{graph → util}/dependency-spec.ts +22 -13
  150. package/src/util/edit-distance.ts +13 -1
  151. package/src/util/env-flag.ts +9 -0
  152. package/src/util/errors.ts +59 -3
  153. package/src/util/essential-env.ts +57 -0
  154. package/src/util/git-ignored.ts +1 -1
  155. package/src/util/hash.ts +35 -0
  156. package/src/util/hide-env.ts +45 -5
  157. package/src/util/index.ts +21 -5
  158. package/src/util/num.ts +8 -0
  159. package/src/util/paths.ts +91 -21
  160. package/src/util/report.ts +6 -1
  161. package/src/util/secret-mask.ts +62 -1
  162. package/src/util/secrets.ts +7 -2
  163. package/src/util/size.ts +11 -3
  164. package/src/util/tail.ts +6 -1
  165. package/src/util/timing.ts +13 -1
  166. package/src/util/ui.ts +61 -3
  167. package/src/util/which.ts +1 -1
  168. package/src/util/write-file.ts +67 -0
  169. package/src/workspace/affected.ts +710 -107
  170. package/src/workspace/bracket-literal.ts +84 -0
  171. package/src/workspace/canonical-dir.ts +37 -0
  172. package/src/workspace/config-bytecode.ts +17 -11
  173. package/src/workspace/config-eval.ts +29 -8
  174. package/src/workspace/config-imports.ts +199 -58
  175. package/src/workspace/config-schema.ts +242 -39
  176. package/src/workspace/filter.ts +37 -8
  177. package/src/workspace/index.ts +11 -1
  178. package/src/workspace/json-data.ts +3 -4
  179. package/src/workspace/lockfile.ts +57 -8
  180. package/src/workspace/migration.ts +69 -18
  181. package/src/workspace/package-graph.ts +5 -4
  182. package/src/workspace/project-loader.ts +226 -37
  183. package/src/workspace/repo-id.ts +1 -1
  184. package/src/workspace/workspace.ts +176 -33
@@ -1,5 +1,5 @@
1
1
  // A plugin's own entries in the cache the workspace declares (`ctx.cache`,
2
- // owner 2026-10-14): the local store first, then each remote in declared
2
+ // 2026-10-14): the local store first, then each remote in declared
3
3
  // order, under the run's policy and scope. The plugin speaks names; core
4
4
  // derives the key, so no plugin reaches another's entries or a task's. An
5
5
  // entry is written once and never replaced (the cache is immutable). Every
@@ -7,9 +7,9 @@
7
7
  // warnings): what a plugin keeps here is a hint.
8
8
 
9
9
  import type { Database } from 'bun:sqlite'
10
- import { xxh3hex } from '../util/index.js'
10
+ import { thrownMessage, xxh3hex } from '../util/index.js'
11
11
  import type { CachePolicy } from './policy.js'
12
- import type { ResolvedRemote } from './remote-lane.js'
12
+ import { sayRemoteFault, type ResolvedRemote } from './remote-lane.js'
13
13
  import { scopedKey } from './scoped-cache.js'
14
14
  import { runReadKeys } from './run-store.js'
15
15
 
@@ -73,6 +73,7 @@ async function capped(body: unknown, max = PLUGIN_ENTRY_MAX_BYTES): Promise<Uint
73
73
  if (done) break
74
74
  size += value.byteLength
75
75
  if (size > max) {
76
+ // quiet: the body is refused for its size, said by the miss; its cancel adds nothing
76
77
  await reader.cancel().catch(() => {})
77
78
  return null
78
79
  }
@@ -90,15 +91,30 @@ async function capped(body: unknown, max = PLUGIN_ENTRY_MAX_BYTES): Promise<Uint
90
91
 
91
92
  /**
92
93
  * `plugin`'s view of the cache. The local store never knows a scope
93
- * (owner, 2026-10-17): it keeps every entry under its name's key. A scoped
94
+ * (2026-10-17): it keeps every entry under its name's key. A scoped
94
95
  * run writes a remote only under the scope's key, and reads one key: the
95
- * scope's, or with `scoped` false the default (M, 2026-10-18: no request
96
+ * scope's, or with `scoped` false the default (2026-10-18: no request
96
97
  * twice).
97
98
  */
98
99
  export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache {
99
100
  const { db, policy, scope } = host
100
- const readers = host.remotes.filter((r) => r.read)
101
+ // A plugin's entries are not task artifacts: a policy that reads no
102
+ // remote artifact (a cold run, `--force`) still reads them, so the run is
103
+ // planned on the default branch's timings. A cold CI run asked workers
104
+ // for 128 tasks with 0 measured, at 25% of their cores (2026-10-23).
105
+ const readers = host.remotes.filter((r) => r.read || r.write)
101
106
  const writers = host.remotes.filter((r) => r.write)
107
+ // Each remote's fault said once per kind: unsaid, a remote down left DTE
108
+ // planning on no timings and no line said why.
109
+ const said = new Map<ResolvedRemote, Set<string>>()
110
+ const fault = (r: ResolvedRemote, op: string, what: string, err: unknown): void => {
111
+ const message = thrownMessage(err)
112
+ const seen = said.get(r) ?? new Set<string>()
113
+ said.set(r, seen)
114
+ if (seen.has(`${op}\0${message}`)) return
115
+ seen.add(`${op}\0${message}`)
116
+ sayRemoteFault(r, new Error(`${op} of ${plugin}'s ${what} failed: ${message}`, { cause: err }))
117
+ }
102
118
  const readKey = (name: string, scoped: boolean): string => {
103
119
  const key = entryKey(plugin, name)
104
120
  return scope === undefined || !scoped ? key : scopedKey(scope, key)
@@ -115,6 +131,7 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
115
131
  bytes: Uint8Array
116
132
  } | null
117
133
  return row === null ? null : new Uint8Array(row.bytes)
134
+ // quiet: the local index, not a remote: an unreadable one holds nothing
118
135
  } catch {
119
136
  return null
120
137
  }
@@ -127,6 +144,7 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
127
144
  .query(`SELECT key FROM plugin_blobs WHERE key IN (${keys.map(() => '?').join(', ')})`)
128
145
  .all(...keys) as Array<{ key: string }>
129
146
  for (const r of rows) found.add(r.key)
147
+ // quiet: the local index, not a remote
130
148
  } catch {
131
149
  // an unreadable index holds nothing
132
150
  }
@@ -139,6 +157,7 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
139
157
  bytes,
140
158
  Date.now(),
141
159
  )
160
+ // quiet: the local index, not a remote
142
161
  } catch {
143
162
  // a read-only or busy index keeps nothing
144
163
  }
@@ -150,16 +169,17 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
150
169
  const hit = localGet(local)
151
170
  if (hit !== null) return hit
152
171
  const key = readKey(name, scoped)
153
- for (const { remote } of readers) {
172
+ for (const r of readers) {
154
173
  try {
155
- const got = await remote.get(key)
174
+ const got = await r.remote.get(key)
156
175
  if (got === null) continue
157
176
  const bytes = await capped(got.body)
158
177
  if (bytes === null) continue
159
178
  if (policy.localWrite) localPut(local, bytes)
160
179
  return bytes
161
- } catch {
180
+ } catch (err) {
162
181
  // a failing remote is a miss there
182
+ fault(r, 'get', name, err)
163
183
  }
164
184
  }
165
185
  return null
@@ -169,11 +189,12 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
169
189
  if (bytes.byteLength === 0 || bytes.byteLength > PLUGIN_ENTRY_MAX_BYTES) return
170
190
  const key = writeKey(name)
171
191
  if (policy.localWrite) localPut(entryKey(plugin, name), bytes)
172
- for (const { remote } of writers) {
192
+ for (const r of writers) {
173
193
  try {
174
- await remote.put(key, new Blob([bytes]), { durationMs: 0 })
175
- } catch {
194
+ await r.remote.put(key, new Blob([bytes]), { durationMs: 0 })
195
+ } catch (err) {
176
196
  // the other layers still keep it
197
+ fault(r, 'put', name, err)
177
198
  }
178
199
  }
179
200
  },
@@ -182,12 +203,13 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
182
203
  if (localHas([entryKey(plugin, name)]).size > 0) return true
183
204
  const key = readKey(name, scoped)
184
205
  let answered = policy.localRead
185
- for (const { remote } of readers) {
206
+ for (const r of readers) {
186
207
  try {
187
- if (await remote.has(key)) return true
208
+ if (await r.remote.has(key)) return true
188
209
  answered = true
189
- } catch {
210
+ } catch (err) {
190
211
  // no answer from this remote
212
+ fault(r, 'has', name, err)
191
213
  }
192
214
  }
193
215
  return answered ? false : null
@@ -198,17 +220,20 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
198
220
  for (const name of names) byKey.set(readKey(name, scoped), name)
199
221
  const keys = [...byKey.keys()]
200
222
  const found = new Set<string>()
201
- for (const key of localHas(names.map((n) => entryKey(plugin, n)))) found.add(byKey.get(key)!)
202
- for (const { remote } of readers) {
223
+ // The local store keys entries unscoped (`entryKey`), not by `readKey`.
224
+ const byLocal = new Map(names.map((n) => [entryKey(plugin, n), n]))
225
+ for (const key of localHas([...byLocal.keys()])) found.add(byLocal.get(key)!)
226
+ for (const r of readers) {
203
227
  const ask = keys.filter((k) => !found.has(byKey.get(k)!))
204
228
  if (ask.length === 0) break
205
- if (remote.hasMany === undefined) return null
229
+ if (r.remote.hasMany === undefined) return null
206
230
  try {
207
- const got: unknown = await remote.hasMany(ask)
231
+ const got: unknown = await r.remote.hasMany(ask)
208
232
  if (!(got instanceof Map)) return null
209
233
  for (const key of got.keys())
210
234
  if (byKey.has(key as string)) found.add(byKey.get(key as string)!)
211
- } catch {
235
+ } catch (err) {
236
+ fault(r, 'hasMany', `${ask.length} entries`, err)
212
237
  return null
213
238
  }
214
239
  }
@@ -216,15 +241,16 @@ export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache
216
241
  },
217
242
 
218
243
  async getRun(name) {
219
- for (const { remote } of readers) {
244
+ for (const r of readers) {
220
245
  for (const key of runReadKeys(name, scope)) {
221
246
  try {
222
- const got = await remote.get(key)
247
+ const got = await r.remote.get(key)
223
248
  if (got === null) continue
224
249
  const bytes = await capped(got.body, Infinity)
225
250
  if (bytes !== null) return bytes
226
- } catch {
251
+ } catch (err) {
227
252
  // a failing remote is a miss there
253
+ fault(r, 'get', name, err)
228
254
  }
229
255
  }
230
256
  }
@@ -0,0 +1,72 @@
1
+ // What the remotes said they hold, kept across runs and processes: `vx dte
2
+ // plan` and `vx run` are separate commands on GitLab, and a run trusts this
3
+ // at once instead of waiting on a has-many (2026-10-23). Only hits are
4
+ // kept: a key not here is a miss. Every has-many that answers rewrites the
5
+ // keys it asked, and a GET that finds nothing forgets its key.
6
+
7
+ import type { Database } from 'bun:sqlite'
8
+ import { inHashes } from './schema.js'
9
+
10
+ export interface RemoteHit {
11
+ readonly size?: number
12
+ readonly durationMs?: number
13
+ }
14
+
15
+ export class RemoteHits {
16
+ constructor(
17
+ private readonly db: Database,
18
+ /** Off for a reading verb or an index this user cannot write. */
19
+ private readonly write: boolean,
20
+ ) {}
21
+
22
+ /** The keys of `hashes` a remote last said it holds. */
23
+ get(hashes: readonly string[]): Map<string, RemoteHit> {
24
+ const out = new Map<string, RemoteHit>()
25
+ if (hashes.length === 0) return out
26
+ const { test, params } = inHashes(hashes)
27
+ try {
28
+ const rows = this.db
29
+ .query(`SELECT hash, size, duration_ms FROM remote_hits WHERE hash ${test}`)
30
+ .all(...params) as Array<{ hash: string; size: number | null; duration_ms: number | null }>
31
+ for (const r of rows)
32
+ out.set(r.hash, {
33
+ ...(r.size === null ? {} : { size: r.size }),
34
+ ...(r.duration_ms === null ? {} : { durationMs: r.duration_ms }),
35
+ })
36
+ } catch {
37
+ // An unreadable or busy index knows no hit.
38
+ }
39
+ return out
40
+ }
41
+
42
+ /** A has-many's whole answer: `found` are hits, the rest of `asked` no longer are. */
43
+ record(asked: readonly string[], found: ReadonlyMap<string, RemoteHit>, now = Date.now()): void {
44
+ if (!this.write || asked.length === 0) return
45
+ try {
46
+ const upsert = this.db.query(
47
+ 'INSERT OR REPLACE INTO remote_hits(hash, size, duration_ms, seen_at) VALUES (?, ?, ?, ?)',
48
+ )
49
+ const gone = asked.filter((h) => !found.has(h))
50
+ this.db.transaction(() => {
51
+ for (const [hash, info] of found)
52
+ upsert.run(hash, info.size ?? null, info.durationMs ?? null, now)
53
+ if (gone.length > 0) {
54
+ const { test, params } = inHashes(gone)
55
+ this.db.query(`DELETE FROM remote_hits WHERE hash ${test}`).run(...params)
56
+ }
57
+ })()
58
+ } catch {
59
+ // A busy index keeps nothing: the next run asks again.
60
+ }
61
+ }
62
+
63
+ /** No remote served `hash`: a miss from now on. */
64
+ forget(hash: string): void {
65
+ if (!this.write) return
66
+ try {
67
+ this.db.query('DELETE FROM remote_hits WHERE hash = ?').run(hash)
68
+ } catch {
69
+ // A busy index keeps the row: its next GET forgets it.
70
+ }
71
+ }
72
+ }
@@ -4,7 +4,7 @@
4
4
  // class with the endpoint named. The order across remotes, the local store
5
5
  // and provenance are `LayeredCache`'s; nothing here knows another remote.
6
6
 
7
- import { machineParallelism } from '../util/index.js'
7
+ import { machineParallelism, thrownMessage, ui } from '../util/index.js'
8
8
  import type { RemoteKeyTraffic, RemoteTraffic } from './cache.js'
9
9
  import type { ArtifactInfo, RemoteCacheLayer } from './layered-cache.js'
10
10
 
@@ -20,6 +20,19 @@ export interface ResolvedRemote {
20
20
  readonly onError?: (err: Error) => void
21
21
  }
22
22
 
23
+ /**
24
+ * A fault of `r` that a caller outside its lane degrades to a miss, said as
25
+ * the lane says its own: to its `onError`, else stderr. Reporting never
26
+ * escalates.
27
+ */
28
+ export function sayRemoteFault(r: ResolvedRemote, e: Error): void {
29
+ try {
30
+ if (r.onError) r.onError(e)
31
+ else ui.say('warn', `[vx] remote cache: ${e.message}\n`)
32
+ // quiet: a throwing reporter must never fail the run
33
+ } catch {}
34
+ }
35
+
23
36
  /**
24
37
  * Requests at once to a remote, by default: pnpm's network concurrency
25
38
  * (Bun's is a fixed 64). A request waits on the network, not a core, so
@@ -84,7 +97,7 @@ function printableEndpoint(endpoint: unknown): string | undefined {
84
97
 
85
98
  /**
86
99
  * What makes two failures "the same way": the error's `code` when it
87
- * carries one (a gRPC status, an errno, Bun's `ConnectionRefused`), because
100
+ * carries one (a gRPC status, an errno, Bun's `ECONNREFUSED`), because
88
101
  * gRPC writes the elapsed time into its message and no two deadline messages
89
102
  * match; else the message with the artifact's hash taken out, so a 500 on
90
103
  * one hash and a 500 on the next are one class. The operation is not part of
@@ -115,6 +128,8 @@ export class RemoteLane {
115
128
  private readonly absent = new Set<string>()
116
129
  /** What a `hasMany` said this remote holds, as `absent` records the rest. */
117
130
  private readonly held = new Map<string, ArtifactInfo>()
131
+ /** Keys whose bytes here did not read back (`distrust`). */
132
+ private readonly distrusted = new Set<string>()
118
133
 
119
134
  /**
120
135
  * A remote the run writes but does not read (`--force`): which of the
@@ -229,22 +244,62 @@ export class RemoteLane {
229
244
  }
230
245
 
231
246
  /**
232
- * The artifact's body, or null on a miss, a known absence (unless
233
- * `fresh`: saved since the probe), a failure or a malformed answer.
247
+ * `hashes` asked one by one (`has`), `concurrency` at a time: what a
248
+ * remote that answered no batch query holds, recording the rest as
249
+ * absent. Null when any probe failed: then the answer is not complete.
250
+ */
251
+ async hasEach(
252
+ hashes: readonly string[],
253
+ concurrency: number,
254
+ ): Promise<Map<string, ArtifactInfo> | null> {
255
+ const out = new Map<string, ArtifactInfo>()
256
+ const absent: string[] = []
257
+ let failed = false
258
+ let next = 0
259
+ const one = async (): Promise<void> => {
260
+ while (!failed && next < hashes.length) {
261
+ const h = hashes[next++]!
262
+ let found: unknown
263
+ try {
264
+ found = this.held.get(h) ?? (this.absent.has(h) ? null : await this.remote.has(h))
265
+ } catch (err) {
266
+ this.report('probe', h, err)
267
+ failed = true
268
+ return
269
+ }
270
+ if (found === null || found === undefined) absent.push(h)
271
+ else out.set(h, artifactInfo(found) ?? {})
272
+ }
273
+ }
274
+ await Promise.all(Array.from({ length: Math.max(1, concurrency) }, one))
275
+ if (failed) return null
276
+ for (const h of absent) this.absent.add(h)
277
+ for (const [h, info] of out) this.held.set(h, info)
278
+ return out
279
+ }
280
+
281
+ /**
282
+ * The artifact's body, null on a miss or a known absence (unless
283
+ * `fresh`: saved since the probe), undefined on a failure or a
284
+ * malformed answer: the key may still be there.
234
285
  */
235
286
  async get(
236
287
  hash: string,
237
288
  fresh = false,
238
- ): Promise<{ body: Blob | Response; durationMs?: number } | null> {
289
+ ): Promise<{ body: Blob | Response; durationMs?: number } | null | undefined> {
239
290
  if (!fresh && this.absent.has(hash)) return null
240
291
  let result: unknown
241
292
  try {
242
293
  result = await this.remote.get(hash)
243
294
  } catch (err) {
244
295
  this.report('download', hash, err)
296
+ return undefined
297
+ }
298
+ if (!result) {
299
+ // Gone, whatever a probe said: the re-run's save uploads it.
300
+ this.held.delete(hash)
245
301
  return null
246
302
  }
247
- if (!result) return null
248
303
  if (typeof result !== 'object' || !isBody((result as { body?: unknown }).body)) {
249
304
  const shape =
250
305
  typeof result !== 'object'
@@ -257,7 +312,7 @@ export class RemoteLane {
257
312
  `get() resolved ${shape} (expected { body: Blob | Response, durationMs } or null)`,
258
313
  ),
259
314
  )
260
- return null
315
+ return undefined
261
316
  }
262
317
  // It holds the key: a save of it after this restore uploads nothing.
263
318
  const got = result as { body: Blob | Response; durationMs?: number }
@@ -265,6 +320,11 @@ export class RemoteLane {
265
320
  return got
266
321
  }
267
322
 
323
+ /** `hash`'s bytes here are unreadable: a save of it uploads, held or not. */
324
+ distrust(hash: string): void {
325
+ this.distrusted.add(hash)
326
+ }
327
+
268
328
  /**
269
329
  * Start the batch probe `presence` reads. A lane that reads, never
270
330
  * writes, or cannot batch (no `hasMany`) ignores it: each upload then
@@ -293,27 +353,41 @@ export class RemoteLane {
293
353
  return (await this.has(hash)) !== null
294
354
  }
295
355
 
296
- /** Queue a PUT in this remote's bounded pool; `body` is asked for when the job starts. */
356
+ /**
357
+ * Queue a PUT in this remote's bounded pool; `body` is asked for when the
358
+ * job starts. Resolves true once this remote holds `hash`: put now, or held already.
359
+ * `absent`: no remote held it when the run asked, so none is asked again.
360
+ */
297
361
  upload(
298
362
  hash: string,
299
363
  durationMs: number,
300
364
  body: () => Promise<{ body: Blob; release?: () => Promise<void> }>,
301
365
  tally: { count: number; bytes: number; ms: number; failed: number },
302
- ): void {
366
+ absent = false,
367
+ ): Promise<boolean> {
303
368
  const key = this.trafficOf(hash)
304
369
  key.uploading++
305
370
  this.traffic.inFlight++
371
+ const { promise, resolve } = Promise.withResolvers<boolean>()
306
372
  this.uploadQueue.push(async () => {
307
373
  let release: (() => Promise<void>) | undefined
374
+ let holds = false
308
375
  try {
309
376
  if (this.stopped) return
310
- if (this.held.has(hash)) return
311
- if (!this.read && (await this.holds(hash))) return
377
+ // A key whose bytes here did not read back is sent over them.
378
+ const distrusted = this.distrusted.has(hash)
379
+ const known = !distrusted && (absent || this.absent.has(hash))
380
+ if (!distrusted) {
381
+ if (this.held.has(hash)) return void (holds = true)
382
+ if (!known && !this.read && (await this.holds(hash))) return void (holds = true)
383
+ }
312
384
  if (this.stopped) return
313
385
  const start = performance.now()
314
386
  const got = await body()
315
387
  release = got.release
316
- await this.remote.put(hash, got.body, { durationMs })
388
+ if (known) this.remote.missing?.(hash)
389
+ if ((await this.remote.put(hash, got.body, { durationMs })) === false) return
390
+ holds = true
317
391
  tally.count++
318
392
  tally.bytes += got.body.size
319
393
  tally.ms += Math.round(performance.now() - start)
@@ -329,9 +403,11 @@ export class RemoteLane {
329
403
  // stopped first): no traffic, so the footer counts no upload.
330
404
  if (key.uploading === 0 && !key.landed && key.failures.length === 0)
331
405
  this.traffic.keys.delete(hash)
406
+ resolve(holds)
332
407
  }
333
408
  })
334
409
  this.pump()
410
+ return promise
335
411
  }
336
412
 
337
413
  /** Settles when the run is stopped. */
@@ -381,7 +457,7 @@ export class RemoteLane {
381
457
  * server in it (item 749).
382
458
  */
383
459
  report(op: RemoteOp, subject: string | { batch: string }, err: unknown): void {
384
- const e = err instanceof Error ? err : new Error(String(err))
460
+ const e = err instanceof Error ? err : new Error(thrownMessage(err))
385
461
  const hash = typeof subject === 'string' ? subject : undefined
386
462
  // A batch probe's failure is no one task's: each of its hashes is asked
387
463
  // again on its own, and that ask is the one counted.
@@ -435,7 +511,7 @@ export class RemoteLane {
435
511
  // hiccup into a run failure.
436
512
  try {
437
513
  if (this.onError) this.onError(e)
438
- else process.stderr.write(`[vx] remote cache: ${e.message}\n`)
514
+ else ui.say('warn', `[vx] remote cache: ${e.message}\n`)
439
515
  } catch {
440
516
  // swallow — reporting must never escalate
441
517
  }
@@ -6,19 +6,45 @@
6
6
  import type { Database, SQLQueryBindings } from 'bun:sqlite'
7
7
  import { lazyStatement } from './schema.js'
8
8
  import type { InvocationRecord, RunRecord } from './layer.js'
9
+ import { isMeasure } from '../util/index.js'
9
10
 
10
- const INSERT_RUNS = `
11
- INSERT INTO runs(
12
- hash, project, task, status, exit_code, duration_ms, forward_args,
13
- started_at, ended_at,
14
- run_id, cpu_ms, peak_rss_bytes, wallclock_start_ns, wallclock_end_ns,
15
- cache_hit, attempts, cached,
16
- blocked_by, timed_out, sandbox_violations, not_ready, restored, remote_error,
17
- peak_cores, avg_rss_bytes, placement, placement_cores, placement_memory, keyless_by
18
- )
19
- VALUES `
20
- /** One `runs` row's placeholders: the 29 columns `bindRun` fills. */
21
- const RUN_TUPLE = `(${Array(29).fill('?').join(', ')})`
11
+ /** The `runs` columns `bindRun` fills, in its order. */
12
+ const RUN_COLUMNS = [
13
+ 'hash',
14
+ 'project',
15
+ 'task',
16
+ 'status',
17
+ 'exit_code',
18
+ 'duration_ms',
19
+ 'forward_args',
20
+ 'started_at',
21
+ 'ended_at',
22
+ 'run_id',
23
+ 'cpu_ms',
24
+ 'peak_rss_bytes',
25
+ 'wallclock_start_ns',
26
+ 'wallclock_end_ns',
27
+ 'cache_hit',
28
+ 'attempts',
29
+ 'cached',
30
+ 'blocked_by',
31
+ 'timed_out',
32
+ 'sandbox_violations',
33
+ 'not_ready',
34
+ 'restored',
35
+ 'remote_error',
36
+ 'peak_cores',
37
+ 'avg_rss_bytes',
38
+ 'placement',
39
+ 'placement_cores',
40
+ 'placement_memory',
41
+ 'keyless_by',
42
+ 'def',
43
+ 'cpu_share',
44
+ ]
45
+
46
+ const INSERT_RUNS = `INSERT INTO runs(${RUN_COLUMNS.join(', ')}) VALUES `
47
+ const RUN_TUPLE = `(${RUN_COLUMNS.map(() => '?').join(', ')})`
22
48
 
23
49
  /** An INSERT of `n` `runs` rows. */
24
50
  function insertRunsSql(n: number): string {
@@ -26,14 +52,15 @@ function insertRunsSql(n: number): string {
26
52
  }
27
53
 
28
54
  /**
29
- * Rows per INSERT: 34 × 29 variables stays under 999, the oldest default
30
- * ceiling. A statement per row was 8.7 ms of 1,000 rows, 40 at a time 6.4
31
- * (2026-10-03).
55
+ * Rows per INSERT: as many as stay within 999 bound variables, the oldest
56
+ * default ceiling, read off the column count (33 rows were sized for 30
57
+ * columns and bound 1,023 at 31). A statement per row was 8.7 ms of 1,000
58
+ * rows, 40 at a time 6.4 (2026-10-03).
32
59
  */
33
- const RUNS_PER_INSERT = 34
60
+ const RUNS_PER_INSERT = Math.floor(999 / RUN_COLUMNS.length)
34
61
 
35
- /** Rows per `task_durations` INSERT: 6 variables each, under 999. */
36
- const DURATIONS_PER_INSERT = 160
62
+ /** Rows per `task_durations` INSERT: 7 variables each, under 999. */
63
+ const DURATIONS_PER_INSERT = 140
37
64
 
38
65
  /** Rows per `task_outcomes` INSERT: 6 variables each, under 999. */
39
66
  const OUTCOMES_PER_INSERT = 160
@@ -63,6 +90,20 @@ const PASSED_HASHES = `
63
90
  /** A local run's `task_durations.source`: its branch, `''` when it had none. */
64
91
  export const branchSource = (branch: string | null): string => `branch:${branch ?? ''}`
65
92
 
93
+ /**
94
+ * How long a run would have taken alone: on a crowded worker
95
+ * (`cpuShare` < 1) each runnable thread waited for a core. A task that
96
+ * kept `n` cores busy lost its CPU's wait spread over them, under one
97
+ * core all of it. vx-dte#test (20 s CPU in 75 s) took 81-102 s beside
98
+ * test shards and the plan, reading those, said 118 s for a 92 s run
99
+ * (main fd5c72e, 2026-10-23).
100
+ */
101
+ function aloneMs(r: Pick<RunRecord, 'durationMs' | 'cpuMs' | 'cpuShare'>): number {
102
+ const { durationMs: d, cpuMs: c, cpuShare: s } = r
103
+ if (c === undefined || s === undefined || !(s > 0 && s < 1) || !(d > 0)) return d
104
+ return Math.round(d - (c / s - c) / Math.max(1, c / (d * s)))
105
+ }
106
+
66
107
  export class RunHistory {
67
108
  private readonly insertInvocation: ReturnType<Database['prepare']>
68
109
 
@@ -106,16 +147,19 @@ export class RunHistory {
106
147
  .immediate()
107
148
  }
108
149
 
109
- /** One `task_durations` row per executed success: what a forecast reads. */
150
+ /** One `task_durations` row per executed success, as if run alone: what a forecast reads. */
110
151
  private insertDurations(runs: readonly RunRecord[], source: string, sha: string | null): void {
111
- const done = runs.filter((r) => r.status === 'success' && r.cacheHit !== true)
152
+ const done = runs.filter(
153
+ (r) => r.status === 'success' && r.cacheHit !== true && isMeasure(aloneMs(r)),
154
+ )
112
155
  for (let i = 0; i < done.length; i += DURATIONS_PER_INSERT) {
113
156
  const block = done.slice(i, i + DURATIONS_PER_INSERT)
114
157
  const params: SQLQueryBindings[] = []
115
- for (const r of block) params.push(r.project, r.task, source, sha, r.durationMs, r.endedAt)
158
+ for (const r of block)
159
+ params.push(r.project, r.task, source, sha, aloneMs(r), r.endedAt, r.def ?? null)
116
160
  this.db
117
161
  .query(
118
- `INSERT INTO task_durations(project, task, source, sha, duration_ms, recorded_at) VALUES ${Array(block.length).fill('(?, ?, ?, ?, ?, ?)').join(', ')}`,
162
+ `INSERT INTO task_durations(project, task, source, sha, duration_ms, recorded_at, def) VALUES ${Array(block.length).fill('(?, ?, ?, ?, ?, ?, ?)').join(', ')}`,
119
163
  )
120
164
  .run(...params)
121
165
  }
@@ -171,7 +215,7 @@ export class RunHistory {
171
215
  for (let i = 0; i < runs.length; i += RUNS_PER_INSERT) {
172
216
  const block = runs.slice(i, i + RUNS_PER_INSERT)
173
217
  const params: SQLQueryBindings[] = []
174
- for (const r of block) params.push(...bindRun(r, digest))
218
+ for (const r of block) for (const v of bindRun(r, digest)) params.push(v)
175
219
  // `query` caches by text: one statement for every full block.
176
220
  this.db.query(insertRunsSql(block.length)).run(...params)
177
221
  }
@@ -234,6 +278,7 @@ export class RunHistory {
234
278
  this.db.prepare('DELETE FROM task_durations WHERE recorded_at < ?').run(cutoff)
235
279
  this.db.prepare('DELETE FROM task_outcomes WHERE recorded_at < ?').run(cutoff)
236
280
  this.db.prepare('DELETE FROM plugin_blobs WHERE at < ?').run(cutoff)
281
+ this.db.prepare('DELETE FROM remote_hits WHERE seen_at < ?').run(cutoff)
237
282
  }
238
283
  }
239
284
 
@@ -274,6 +319,8 @@ function bindRun(run: RunRecord, digestValue: (v: string) => string): SQLQueryBi
274
319
  run.placementCores ?? null,
275
320
  run.placementMemoryBytes ?? null,
276
321
  run.keylessBy ?? null,
322
+ run.def ?? null,
323
+ run.cpuShare ?? null,
277
324
  ]
278
325
  }
279
326
 
@@ -1,4 +1,4 @@
1
- // What core keeps of a run that writes a remote (owner 2026-10-15): a failed
1
+ // What core keeps of a run that writes a remote (2026-10-15): a failed
2
2
  // task's log and the files its outputs left, an uncached task's log. Names
3
3
  // come from the run id and task (`run/<id>/<task>/log`); each key is derived
4
4
  // in a namespace no task key or plugin entry shares. Remote only: locally
@@ -6,7 +6,7 @@
6
6
  // the cache is free). Plugins read them by name (`ctx.cache.getRun`).
7
7
 
8
8
  import { xxh3hex } from '../util/index.js'
9
- import type { ResolvedRemote } from './remote-lane.js'
9
+ import { sayRemoteFault, type ResolvedRemote } from './remote-lane.js'
10
10
  import { scopedKey } from './scoped-cache.js'
11
11
  import { tarPackInto, tarSize } from './tar-stream.js'
12
12
 
@@ -30,8 +30,8 @@ export function writesRunStore(remotes: readonly ResolvedRemote[]): boolean {
30
30
 
31
31
  /**
32
32
  * Store `bytes` under `name` in every remote the policy writes. An entry is
33
- * written once; a remote's failure leaves the others theirs, silently: what
34
- * a run keeps here is a record, never its verdict.
33
+ * written once; a remote's failure is said and leaves the others theirs:
34
+ * what a run keeps here is a record, never its verdict.
35
35
  */
36
36
  export async function putRunEntry(
37
37
  remotes: readonly ResolvedRemote[],
@@ -41,16 +41,17 @@ export async function putRunEntry(
41
41
  ): Promise<boolean> {
42
42
  const key = scope === undefined ? runKey(name) : scopedKey(scope, runKey(name))
43
43
  let kept = false
44
- for (const { remote, write } of remotes) {
45
- if (!write) continue
44
+ for (const r of remotes) {
45
+ if (!r.write) continue
46
46
  try {
47
- await remote.put(key, new Blob([bytes]), {
47
+ const put = await r.remote.put(key, new Blob([bytes]), {
48
48
  durationMs: 0,
49
49
  ...(scope === undefined ? {} : { scope }),
50
50
  })
51
- kept = true
52
- } catch {
53
- // the other remotes still keep it
51
+ // `false`: the remote sent nothing, so nothing is kept there.
52
+ if (put !== false) kept = true
53
+ } catch (err) {
54
+ sayRemoteFault(r, err instanceof Error ? err : new Error(String(err)))
54
55
  }
55
56
  }
56
57
  return kept