@vzn/vx 0.0.636 → 0.0.638
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -11
- package/docs/caching.md +246 -213
- package/docs/cli.md +791 -705
- package/docs/execution.md +68 -76
- package/docs/features.md +47 -42
- package/docs/patterns.md +26 -25
- package/docs/schema.md +392 -205
- package/docs/security.md +76 -39
- package/launcher.sh +43 -0
- package/package.json +5 -5
- package/schemas/cache.json +41 -19
- package/schemas/info.json +23 -11
- package/schemas/last.json +48 -2
- package/schemas/lock.json +1 -1
- package/schemas/plan.json +4 -4
- package/schemas/show.json +5 -3
- package/schemas/why.json +15 -1
- package/skills/vx/SKILL.md +28 -13
- package/src/bin.ts +30 -9
- package/src/cache/archive.ts +11 -0
- package/src/cache/cache.ts +124 -53
- package/src/cache/chained-cache.ts +4 -0
- package/src/cache/git-inputs.ts +277 -63
- package/src/cache/index.ts +15 -1
- package/src/cache/inputs.ts +102 -36
- package/src/cache/key-fold.ts +8 -7
- package/src/cache/layer.ts +15 -18
- package/src/cache/layered-cache.ts +20 -102
- package/src/cache/plugin-cache.ts +248 -0
- package/src/cache/policy.ts +32 -27
- package/src/cache/run-history.ts +104 -14
- package/src/cache/run-store.ts +77 -0
- package/src/cache/schema.ts +58 -31
- package/src/cache/scoped-cache.ts +237 -0
- package/src/cli/cache.ts +173 -34
- package/src/cli/completions.ts +11 -7
- package/src/cli/core-alias.ts +1 -1
- package/src/cli/foreign-flags.ts +13 -12
- package/src/cli/help.ts +110 -70
- package/src/cli/index.ts +67 -24
- package/src/cli/info.ts +43 -27
- package/src/cli/init.ts +115 -407
- package/src/cli/last.ts +157 -9
- package/src/cli/lock.ts +356 -122
- package/src/cli/plan-format.ts +7 -5
- package/src/cli/plugin-commands.ts +81 -14
- package/src/cli/plugin-templates.ts +11 -11
- package/src/cli/run.ts +116 -160
- package/src/cli/select.ts +50 -47
- package/src/cli/show.ts +83 -59
- package/src/cli/task-verb.ts +6 -13
- package/src/cli/upgrade.ts +415 -149
- package/src/cli/watch-cycle.ts +9 -2
- package/src/cli/watch-filter.ts +39 -61
- package/src/cli/watch-fs.ts +29 -16
- package/src/cli/watch-judge.ts +18 -4
- package/src/cli/watch-set.ts +16 -42
- package/src/cli/watch.ts +227 -230
- package/src/cli/why.ts +38 -18
- package/src/cli/workspace-config.ts +33 -70
- package/src/config.ts +169 -70
- package/src/exec/env.ts +1 -31
- package/src/exec/executor.ts +1 -5
- package/src/exec/kill-tree.ts +2 -1
- package/src/exec/sandbox-runtime.ts +12 -1
- package/src/exec/sandbox-violations.ts +9 -1
- package/src/graph/dependency-spec.ts +1 -1
- package/src/graph/index.ts +1 -0
- package/src/graph/priorities.ts +1 -1
- package/src/graph/scheduler.ts +27 -7
- package/src/graph/task-graph.ts +76 -24
- package/src/handoff.ts +169 -0
- package/src/index.ts +35 -58
- package/src/orchestrator/affected-tasks.ts +15 -8
- package/src/orchestrator/command-windows.ts +44 -0
- package/src/orchestrator/deferred-outputs.ts +12 -10
- package/src/orchestrator/doctor.ts +98 -110
- package/src/orchestrator/download-policy.ts +3 -3
- package/src/orchestrator/events.ts +19 -3
- package/src/orchestrator/excluded-keys.ts +0 -2
- package/src/orchestrator/execute-task.ts +147 -69
- package/src/orchestrator/failure-mode.ts +19 -10
- package/src/orchestrator/fingerprint-watch.ts +18 -15
- package/src/orchestrator/forecast.ts +10 -2
- package/src/orchestrator/framed-output.ts +9 -5
- package/src/orchestrator/graph-stage.ts +5 -5
- package/src/orchestrator/history.ts +126 -30
- package/src/orchestrator/hit-restore.ts +15 -3
- package/src/orchestrator/index.ts +20 -5
- package/src/orchestrator/local-shortcircuit.ts +2 -1
- package/src/orchestrator/lockfile-claim.ts +79 -20
- package/src/orchestrator/logger.ts +80 -41
- package/src/orchestrator/metrics.ts +78 -9
- package/src/orchestrator/miss-save.ts +3 -0
- package/src/orchestrator/options.ts +42 -29
- package/src/orchestrator/output-host.ts +72 -0
- package/src/orchestrator/plan.ts +6 -8
- package/src/orchestrator/plugin-host.ts +100 -66
- package/src/orchestrator/plugin.ts +173 -67
- package/src/orchestrator/prepare.ts +163 -78
- package/src/orchestrator/projects.ts +105 -99
- package/src/orchestrator/remote-prefetch.ts +11 -6
- package/src/orchestrator/run-context.ts +25 -6
- package/src/orchestrator/run-failures.ts +17 -8
- package/src/orchestrator/run-lock.ts +1 -1
- package/src/orchestrator/run-records.ts +1 -1
- package/src/orchestrator/run-report.ts +9 -9
- package/src/orchestrator/run-settings.ts +48 -0
- package/src/orchestrator/run.ts +244 -119
- package/src/orchestrator/sandbox-fix.ts +39 -20
- package/src/orchestrator/sandbox-request.ts +84 -13
- package/src/orchestrator/signals.ts +1 -1
- package/src/orchestrator/stable-keys.ts +13 -11
- package/src/orchestrator/status-line.ts +49 -7
- package/src/orchestrator/summary.ts +115 -106
- package/src/orchestrator/task-hash.ts +12 -83
- package/src/orchestrator/telemetry.ts +4 -4
- package/src/settings.ts +428 -0
- package/src/{orchestrator → util}/colors.ts +5 -0
- package/src/util/config-frame.ts +31 -2
- package/src/util/errors.ts +33 -1
- package/src/util/essential-env.ts +31 -0
- package/src/util/hide-env.ts +107 -0
- package/src/util/index.ts +16 -3
- package/src/util/paths.ts +10 -0
- package/src/util/refusal-notice.ts +111 -0
- package/src/util/report.ts +31 -0
- package/src/util/settle.ts +49 -30
- package/src/util/size.ts +23 -4
- package/src/util/timing.ts +3 -2
- package/src/util/ui.ts +186 -0
- package/src/util/ulid.ts +1 -1
- package/src/util/verbs.ts +0 -15
- package/src/workspace/affected.ts +84 -232
- package/src/workspace/config-eval.ts +15 -20
- package/src/workspace/config-imports.ts +78 -3
- package/src/workspace/config-schema.ts +209 -268
- package/src/workspace/dependency-rules.ts +65 -0
- package/src/workspace/filter.ts +2 -2
- package/src/workspace/fingerprint.ts +11 -121
- package/src/workspace/index.ts +20 -27
- package/src/workspace/load-reads.ts +4 -4
- package/src/workspace/lockfile.ts +96 -140
- package/src/workspace/migration.ts +307 -215
- package/src/workspace/package-graph.ts +30 -216
- package/src/workspace/project-loader.ts +80 -285
- package/src/workspace/repo-id.ts +2 -1
- package/src/workspace/workspace.ts +363 -831
- package/src/cache/config-evals.ts +0 -130
- package/src/workspace/config-cache.ts +0 -661
- package/src/workspace/migrate-scripts.ts +0 -1121
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
// A plugin's own entries in the cache the workspace declares (`ctx.cache`,
|
|
2
|
+
// owner 2026-10-14): the local store first, then each remote in declared
|
|
3
|
+
// order, under the run's policy and scope. The plugin speaks names; core
|
|
4
|
+
// derives the key, so no plugin reaches another's entries or a task's. An
|
|
5
|
+
// entry is written once and never replaced (the cache is immutable). Every
|
|
6
|
+
// failure is an absent answer, never a warning (M: no extra runtime
|
|
7
|
+
// warnings): what a plugin keeps here is a hint.
|
|
8
|
+
|
|
9
|
+
import type { Database } from 'bun:sqlite'
|
|
10
|
+
import { xxh3hex } from '../util/index.js'
|
|
11
|
+
import type { CachePolicy } from './policy.js'
|
|
12
|
+
import type { RemoteCacheLayer } from './layered-cache.js'
|
|
13
|
+
import { scopedKey } from './scoped-cache.js'
|
|
14
|
+
import { runReadKeys } from './run-store.js'
|
|
15
|
+
|
|
16
|
+
/** The largest entry kept or read, in bytes: a remote's larger body is a miss, read no further. */
|
|
17
|
+
export const PLUGIN_ENTRY_MAX_BYTES = 1024 * 1024
|
|
18
|
+
|
|
19
|
+
/** What a plugin's `ctx.cache` offers. Keys are the plugin's own names. */
|
|
20
|
+
export interface PluginCache {
|
|
21
|
+
/** The bytes stored under `key`, from the first layer holding them; null when none does. */
|
|
22
|
+
get(key: string): Promise<Uint8Array | null>
|
|
23
|
+
/** Store `bytes` under `key` in every layer the policy writes, where it is not already; empty bytes store nothing. */
|
|
24
|
+
put(key: string, bytes: Uint8Array): Promise<void>
|
|
25
|
+
/** Whether some layer holds `key`; null when no layer could answer. */
|
|
26
|
+
has(key: string): Promise<boolean | null>
|
|
27
|
+
/**
|
|
28
|
+
* The subset of `keys` some layer holds, in one probe per layer; null
|
|
29
|
+
* when a readable remote has no batch probe: ask `get` one at a time.
|
|
30
|
+
*/
|
|
31
|
+
hasMany(keys: readonly string[]): Promise<Set<string> | null>
|
|
32
|
+
/**
|
|
33
|
+
* What core kept of a run under `name` (a `log` or `outputs` name its
|
|
34
|
+
* `RunReport` hands `teardown`), from the first readable remote holding
|
|
35
|
+
* it; null when none does. Uncapped: core wrote it.
|
|
36
|
+
*/
|
|
37
|
+
getRun(name: string): Promise<Uint8Array | null>
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** One remote as the plugin cache reaches it: its wire and what the policy lets it do. */
|
|
41
|
+
export interface PluginCacheRemote {
|
|
42
|
+
readonly remote: RemoteCacheLayer
|
|
43
|
+
readonly read: boolean
|
|
44
|
+
readonly write: boolean
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface PluginCacheHost {
|
|
48
|
+
/** The run's index; its store holds `plugin_blobs`. */
|
|
49
|
+
readonly db: Database
|
|
50
|
+
readonly policy: CachePolicy
|
|
51
|
+
readonly remotes: readonly PluginCacheRemote[]
|
|
52
|
+
readonly scope: string | undefined
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** The key `name` stores under for `plugin`: task-key shaped, so every wire stores it unchanged. */
|
|
56
|
+
function entryKey(plugin: string, name: string): string {
|
|
57
|
+
return xxh3hex(`vx-plugin\0${plugin}\0${name}`)
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* A remote body's bytes; null past the cap (read no further than it) and for
|
|
62
|
+
* an empty or odd body, which kept would hide every later good one: an entry
|
|
63
|
+
* is written once (Security, #3580).
|
|
64
|
+
*/
|
|
65
|
+
async function capped(body: unknown, max = PLUGIN_ENTRY_MAX_BYTES): Promise<Uint8Array | null> {
|
|
66
|
+
if (body instanceof Blob) {
|
|
67
|
+
if (body.size === 0 || body.size > max) return null
|
|
68
|
+
return new Uint8Array(await body.arrayBuffer())
|
|
69
|
+
}
|
|
70
|
+
if (!(body instanceof Response) || body.body === null) return null
|
|
71
|
+
const reader = body.body.getReader()
|
|
72
|
+
const chunks: Uint8Array[] = []
|
|
73
|
+
let size = 0
|
|
74
|
+
for (;;) {
|
|
75
|
+
const { done, value } = await reader.read()
|
|
76
|
+
if (done) break
|
|
77
|
+
size += value.byteLength
|
|
78
|
+
if (size > max) {
|
|
79
|
+
await reader.cancel().catch(() => {})
|
|
80
|
+
return null
|
|
81
|
+
}
|
|
82
|
+
chunks.push(value)
|
|
83
|
+
}
|
|
84
|
+
if (size === 0) return null
|
|
85
|
+
const out = new Uint8Array(size)
|
|
86
|
+
let at = 0
|
|
87
|
+
for (const c of chunks) {
|
|
88
|
+
out.set(c, at)
|
|
89
|
+
at += c.byteLength
|
|
90
|
+
}
|
|
91
|
+
return out
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* `plugin`'s view of the cache. A scoped run reads the default key, then
|
|
96
|
+
* the scope's, and writes only the scope's, as tasks do (`ScopedCache`).
|
|
97
|
+
*/
|
|
98
|
+
export function pluginCache(plugin: string, host: PluginCacheHost): PluginCache {
|
|
99
|
+
const { db, policy, scope } = host
|
|
100
|
+
const readers = host.remotes.filter((r) => r.read)
|
|
101
|
+
const writers = host.remotes.filter((r) => r.write)
|
|
102
|
+
const reads = (name: string): string[] => {
|
|
103
|
+
const key = entryKey(plugin, name)
|
|
104
|
+
return scope === undefined ? [key] : [key, scopedKey(scope, key)]
|
|
105
|
+
}
|
|
106
|
+
const writeKey = (name: string): string => {
|
|
107
|
+
const key = entryKey(plugin, name)
|
|
108
|
+
return scope === undefined ? key : scopedKey(scope, key)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const localGet = (key: string): Uint8Array | null => {
|
|
112
|
+
if (!policy.localRead) return null
|
|
113
|
+
try {
|
|
114
|
+
const row = db.query('SELECT bytes FROM plugin_blobs WHERE key = ?').get(key) as {
|
|
115
|
+
bytes: Uint8Array
|
|
116
|
+
} | null
|
|
117
|
+
return row === null ? null : new Uint8Array(row.bytes)
|
|
118
|
+
} catch {
|
|
119
|
+
return null
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
const localHas = (keys: readonly string[]): Set<string> => {
|
|
123
|
+
const found = new Set<string>()
|
|
124
|
+
if (!policy.localRead || keys.length === 0) return found
|
|
125
|
+
try {
|
|
126
|
+
const rows = db
|
|
127
|
+
.query(`SELECT key FROM plugin_blobs WHERE key IN (${keys.map(() => '?').join(', ')})`)
|
|
128
|
+
.all(...keys) as Array<{ key: string }>
|
|
129
|
+
for (const r of rows) found.add(r.key)
|
|
130
|
+
} catch {
|
|
131
|
+
// an unreadable index holds nothing
|
|
132
|
+
}
|
|
133
|
+
return found
|
|
134
|
+
}
|
|
135
|
+
const localPut = (key: string, bytes: Uint8Array): void => {
|
|
136
|
+
try {
|
|
137
|
+
db.query('INSERT OR IGNORE INTO plugin_blobs(key, bytes, at) VALUES (?, ?, ?)').run(
|
|
138
|
+
key,
|
|
139
|
+
bytes,
|
|
140
|
+
Date.now(),
|
|
141
|
+
)
|
|
142
|
+
} catch {
|
|
143
|
+
// a read-only or busy index keeps nothing
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return {
|
|
148
|
+
async get(name) {
|
|
149
|
+
const keys = reads(name)
|
|
150
|
+
for (const key of keys) {
|
|
151
|
+
const hit = localGet(key)
|
|
152
|
+
if (hit !== null) return hit
|
|
153
|
+
}
|
|
154
|
+
for (const { remote } of readers) {
|
|
155
|
+
for (const key of keys) {
|
|
156
|
+
try {
|
|
157
|
+
const got = await remote.get(key)
|
|
158
|
+
if (got === null) continue
|
|
159
|
+
const bytes = await capped(got.body)
|
|
160
|
+
if (bytes === null) continue
|
|
161
|
+
if (policy.localWrite) localPut(key, bytes)
|
|
162
|
+
return bytes
|
|
163
|
+
} catch {
|
|
164
|
+
// a failing remote is a miss there
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return null
|
|
169
|
+
},
|
|
170
|
+
|
|
171
|
+
async put(name, bytes) {
|
|
172
|
+
if (bytes.byteLength === 0 || bytes.byteLength > PLUGIN_ENTRY_MAX_BYTES) return
|
|
173
|
+
const key = writeKey(name)
|
|
174
|
+
if (policy.localWrite) localPut(key, bytes)
|
|
175
|
+
for (const { remote } of writers) {
|
|
176
|
+
try {
|
|
177
|
+
if (await remote.has(key)) continue
|
|
178
|
+
await remote.put(key, new Blob([bytes]), { durationMs: 0 })
|
|
179
|
+
} catch {
|
|
180
|
+
// the other layers still keep it
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
},
|
|
184
|
+
|
|
185
|
+
async has(name) {
|
|
186
|
+
const keys = reads(name)
|
|
187
|
+
if (localHas(keys).size > 0) return true
|
|
188
|
+
let answered = policy.localRead
|
|
189
|
+
for (const { remote } of readers) {
|
|
190
|
+
for (const key of keys) {
|
|
191
|
+
try {
|
|
192
|
+
if (await remote.has(key)) return true
|
|
193
|
+
answered = true
|
|
194
|
+
} catch {
|
|
195
|
+
// no answer from this remote
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return answered ? false : null
|
|
200
|
+
},
|
|
201
|
+
|
|
202
|
+
async hasMany(names) {
|
|
203
|
+
const byKey = new Map<string, string>()
|
|
204
|
+
for (const name of names) for (const key of reads(name)) byKey.set(key, name)
|
|
205
|
+
const keys = [...byKey.keys()]
|
|
206
|
+
const found = new Set<string>()
|
|
207
|
+
for (const key of localHas(keys)) found.add(byKey.get(key)!)
|
|
208
|
+
for (const { remote } of readers) {
|
|
209
|
+
const ask = keys.filter((k) => !found.has(byKey.get(k)!))
|
|
210
|
+
if (ask.length === 0) break
|
|
211
|
+
if (remote.hasMany === undefined) return null
|
|
212
|
+
try {
|
|
213
|
+
const got: unknown = await remote.hasMany(ask)
|
|
214
|
+
if (!(got instanceof Set)) return null
|
|
215
|
+
for (const key of got) if (byKey.has(key as string)) found.add(byKey.get(key as string)!)
|
|
216
|
+
} catch {
|
|
217
|
+
return null
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return found
|
|
221
|
+
},
|
|
222
|
+
|
|
223
|
+
async getRun(name) {
|
|
224
|
+
for (const { remote } of readers) {
|
|
225
|
+
for (const key of runReadKeys(name, scope)) {
|
|
226
|
+
try {
|
|
227
|
+
const got = await remote.get(key)
|
|
228
|
+
if (got === null) continue
|
|
229
|
+
const bytes = await capped(got.body, Infinity)
|
|
230
|
+
if (bytes !== null) return bytes
|
|
231
|
+
} catch {
|
|
232
|
+
// a failing remote is a miss there
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
return null
|
|
237
|
+
},
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/** A cache with no layer: every answer absent. What a context built outside a run holds. */
|
|
242
|
+
export const absentPluginCache: PluginCache = {
|
|
243
|
+
get: async () => null,
|
|
244
|
+
put: async () => {},
|
|
245
|
+
has: async () => null,
|
|
246
|
+
hasMany: async () => null,
|
|
247
|
+
getRun: async () => null,
|
|
248
|
+
}
|
package/src/cache/policy.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// The run-level cache policy: four axes (local/remote × read/write) and
|
|
2
|
-
// the `--cache=<spec>` grammar that sets them. Pure data + parsing; no I/O.
|
|
2
|
+
// the `--cache-policy=<spec>` grammar that sets them. Pure data + parsing; no I/O.
|
|
3
3
|
|
|
4
4
|
import { UserError } from '../util/index.js'
|
|
5
5
|
|
|
@@ -10,8 +10,7 @@ import { UserError } from '../util/index.js'
|
|
|
10
10
|
* cache) is distinct from `--no-cache` (disable everything).
|
|
11
11
|
*
|
|
12
12
|
* The task-artifact get/save path is gated, and the local axes also gate
|
|
13
|
-
* the local store's
|
|
14
|
-
* writes (`cache.ts`). `recordRun`, `stats`, `prune`, key derivation, and
|
|
13
|
+
* the local store's file-hash writes (`cache.ts`). `recordRun`, `stats`, `prune`, key derivation, and
|
|
15
14
|
* prefetch-ingest are never affected — they are bookkeeping/analytics
|
|
16
15
|
* that a run policy has no business disabling.
|
|
17
16
|
*/
|
|
@@ -20,12 +19,6 @@ export interface CachePolicy {
|
|
|
20
19
|
localWrite: boolean
|
|
21
20
|
remoteRead: boolean
|
|
22
21
|
remoteWrite: boolean
|
|
23
|
-
/**
|
|
24
|
-
* An untrusted scope (`cacheScope: 'pr-123'`): remote reads try this
|
|
25
|
-
* scope's keys, then the trusted ones; remote writes go to this scope's
|
|
26
|
-
* keys only. Absent = trusted. See {@link scopeCachePolicy}.
|
|
27
|
-
*/
|
|
28
|
-
remoteScope?: string
|
|
29
22
|
}
|
|
30
23
|
|
|
31
24
|
/** All four axes on — the default when no cache flag is passed. */
|
|
@@ -36,13 +29,13 @@ export const FULL_CACHE_POLICY: CachePolicy = {
|
|
|
36
29
|
remoteWrite: true,
|
|
37
30
|
}
|
|
38
31
|
|
|
39
|
-
/** No axis on (`--no-cache`, `--cache=local:`): no cache answers for any task. */
|
|
32
|
+
/** No axis on (`--no-cache`, `--cache-policy=local:`): no cache answers for any task. */
|
|
40
33
|
export function cachesNothing(p: CachePolicy): boolean {
|
|
41
34
|
return !(p.localRead || p.localWrite || p.remoteRead || p.remoteWrite)
|
|
42
35
|
}
|
|
43
36
|
|
|
44
37
|
/**
|
|
45
|
-
* Parse a `--cache=<spec>` value into a `CachePolicy`, starting from a
|
|
38
|
+
* Parse a `--cache-policy=<spec>` value into a `CachePolicy`, starting from a
|
|
46
39
|
* base (defaults to {@link FULL_CACHE_POLICY}). The spec is a
|
|
47
40
|
* comma-separated list of `layer:flags` segments where `layer` is
|
|
48
41
|
* `local` or `remote` and `flags` is any subset of `r` (read) and `w`
|
|
@@ -53,7 +46,8 @@ export function cachesNothing(p: CachePolicy): boolean {
|
|
|
53
46
|
* So `local:rw,remote:r` = remote read-only, `remote:` = remote fully
|
|
54
47
|
* off, `local:r` = local read-only with remote untouched.
|
|
55
48
|
*
|
|
56
|
-
* Throws a {@link UserError}
|
|
49
|
+
* Throws a {@link UserError}, whose message the caller prefixes with the
|
|
50
|
+
* spec's source (`--cache-policy`, `VX_CACHE_POLICY`, `cache.policy`), on an unknown layer, an unknown flag, a
|
|
57
51
|
* duplicated flag, a missing colon, or a repeated layer.
|
|
58
52
|
*/
|
|
59
53
|
export function parseCachePolicy(spec: string, base: CachePolicy = FULL_CACHE_POLICY): CachePolicy {
|
|
@@ -64,24 +58,24 @@ export function parseCachePolicy(spec: string, base: CachePolicy = FULL_CACHE_PO
|
|
|
64
58
|
if (seg.length === 0) continue
|
|
65
59
|
const colon = seg.indexOf(':')
|
|
66
60
|
if (colon < 0) {
|
|
67
|
-
throw new UserError(`invalid
|
|
61
|
+
throw new UserError(`invalid segment '${seg}': expected '<layer>:<flags>'`)
|
|
68
62
|
}
|
|
69
63
|
const layer = seg.slice(0, colon)
|
|
70
64
|
const flags = seg.slice(colon + 1)
|
|
71
65
|
if (layer !== 'local' && layer !== 'remote') {
|
|
72
|
-
throw new UserError(`invalid
|
|
66
|
+
throw new UserError(`invalid layer '${layer}': expected 'local' or 'remote'`)
|
|
73
67
|
}
|
|
74
68
|
if (seen.has(layer)) {
|
|
75
|
-
throw new UserError(
|
|
69
|
+
throw new UserError(`layer '${layer}' specified twice`)
|
|
76
70
|
}
|
|
77
71
|
seen.add(layer)
|
|
78
72
|
const flagSet = new Set<string>()
|
|
79
73
|
for (const ch of flags) {
|
|
80
74
|
if (ch !== 'r' && ch !== 'w') {
|
|
81
|
-
throw new UserError(`invalid
|
|
75
|
+
throw new UserError(`invalid flag '${ch}' for '${layer}': expected 'r' and/or 'w'`)
|
|
82
76
|
}
|
|
83
77
|
if (flagSet.has(ch)) {
|
|
84
|
-
throw new UserError(
|
|
78
|
+
throw new UserError(`flag '${ch}' repeated for '${layer}'`)
|
|
85
79
|
}
|
|
86
80
|
flagSet.add(ch)
|
|
87
81
|
}
|
|
@@ -97,15 +91,26 @@ export function parseCachePolicy(spec: string, base: CachePolicy = FULL_CACHE_PO
|
|
|
97
91
|
}
|
|
98
92
|
|
|
99
93
|
/**
|
|
100
|
-
*
|
|
101
|
-
*
|
|
102
|
-
* name is an untrusted scope. A clamp, never a widening: `--cache` still
|
|
103
|
-
* decides which axes are on. A client-side convention, not a boundary —
|
|
104
|
-
* any run holding a write credential can write a trusted key; only the
|
|
105
|
-
* cache server, by token, can refuse it (docs/security.md).
|
|
94
|
+
* The `--cache-policy` spec for `p`, both layers named, so the line
|
|
95
|
+
* `vx last` shows pastes back as the flag: `local:rw,remote:r`.
|
|
106
96
|
*/
|
|
107
|
-
export function
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
97
|
+
export function formatCachePolicy(p: CachePolicy): string {
|
|
98
|
+
const flags = (r: boolean, w: boolean): string => `${r ? 'r' : ''}${w ? 'w' : ''}`
|
|
99
|
+
return `local:${flags(p.localRead, p.localWrite)},remote:${flags(p.remoteRead, p.remoteWrite)}`
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** `--no-cache` and `--force`, which still apply over whichever spec set the base. */
|
|
103
|
+
export interface CacheFlags {
|
|
104
|
+
/** `--no-cache`: every axis off. Beats `force`. */
|
|
105
|
+
noCache?: boolean
|
|
106
|
+
/** `--force`: both reads off, writes kept. */
|
|
107
|
+
force?: boolean
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export function applyCacheFlags(policy: CachePolicy, flags: CacheFlags | undefined): CachePolicy {
|
|
111
|
+
if (flags?.noCache === true) {
|
|
112
|
+
return { localRead: false, localWrite: false, remoteRead: false, remoteWrite: false }
|
|
113
|
+
}
|
|
114
|
+
if (flags?.force === true) return { ...policy, localRead: false, remoteRead: false }
|
|
115
|
+
return policy
|
|
111
116
|
}
|
package/src/cache/run-history.ts
CHANGED
|
@@ -31,8 +31,38 @@ function insertRunsSql(n: number): string {
|
|
|
31
31
|
*/
|
|
32
32
|
const RUNS_PER_INSERT = 40
|
|
33
33
|
|
|
34
|
+
/** Rows per `task_durations` INSERT: 6 variables each, under 999. */
|
|
35
|
+
const DURATIONS_PER_INSERT = 160
|
|
36
|
+
|
|
37
|
+
/** Rows per `task_outcomes` INSERT: 6 variables each, under 999. */
|
|
38
|
+
const OUTCOMES_PER_INSERT = 160
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Whether a run says anything of its key, so the flaky record may keep it:
|
|
42
|
+
* keyed, cached, and not run behind a failure (its bytes were not the key's).
|
|
43
|
+
*/
|
|
44
|
+
const keyed = (r: RunRecord): boolean =>
|
|
45
|
+
r.hash !== undefined && r.hash !== '' && r.cached !== false && r.blockedBy == null
|
|
46
|
+
|
|
47
|
+
/** A hit, as the orchestrator marks one (`isCacheHit`, run-records.ts). */
|
|
48
|
+
const isHit = (r: RunRecord): boolean => r.cacheHit === true
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Of a JSON array of hashes, those the record holds a pass of: one index
|
|
52
|
+
* probe each (0.6 ms for 1,000; a join reading every row of each key was
|
|
53
|
+
* 5.7 ms at 20 runs of history). A hit's row is kept only when its key has
|
|
54
|
+
* none: a hit replays a pass, and the first is what a later failure
|
|
55
|
+
* relapses from (an entry adopted from its artifact, or older than the
|
|
56
|
+
* record, has no executed row). By hash alone, as the entry it replays is.
|
|
57
|
+
*/
|
|
58
|
+
const PASSED_HASHES = `
|
|
59
|
+
SELECT j.value AS hash FROM json_each(?) AS j
|
|
60
|
+
WHERE EXISTS (SELECT 1 FROM task_outcomes AS o WHERE o.hash = j.value AND o.status <> 'failed')`
|
|
61
|
+
|
|
62
|
+
/** A local run's `task_durations.source`: its branch, `''` when it had none. */
|
|
63
|
+
export const branchSource = (branch: string | null): string => `branch:${branch ?? ''}`
|
|
64
|
+
|
|
34
65
|
export class RunHistory {
|
|
35
|
-
private readonly insertRun: ReturnType<Database['prepare']>
|
|
36
66
|
private readonly insertInvocation: ReturnType<Database['prepare']>
|
|
37
67
|
|
|
38
68
|
constructor(
|
|
@@ -40,7 +70,6 @@ export class RunHistory {
|
|
|
40
70
|
/** The store's salted value digest (L-4). */
|
|
41
71
|
private readonly digestValue: (value: string) => string,
|
|
42
72
|
) {
|
|
43
|
-
this.insertRun = lazyStatement(this.db, insertRunsSql(1))
|
|
44
73
|
this.insertInvocation = lazyStatement(
|
|
45
74
|
this.db,
|
|
46
75
|
`
|
|
@@ -60,18 +89,74 @@ export class RunHistory {
|
|
|
60
89
|
}
|
|
61
90
|
|
|
62
91
|
recordRun(run: RunRecord): void {
|
|
63
|
-
this.
|
|
92
|
+
this.recordRuns([run])
|
|
64
93
|
}
|
|
65
94
|
|
|
66
95
|
recordRuns(runs: readonly RunRecord[]): void {
|
|
67
96
|
if (runs.length === 0) return
|
|
68
|
-
if (runs.length === 1) {
|
|
69
|
-
this.insertRun.run(...bindRun(runs[0]!, this.digestValue))
|
|
70
|
-
return
|
|
71
|
-
}
|
|
72
97
|
// One transaction, one fsync: for a 200-task run, one instead of 200.
|
|
73
98
|
// IMMEDIATE, as `recordRunBundle` explains.
|
|
74
|
-
this.db
|
|
99
|
+
this.db
|
|
100
|
+
.transaction(() => {
|
|
101
|
+
this.insertRuns(runs)
|
|
102
|
+
this.insertDurations(runs, branchSource(null), null)
|
|
103
|
+
this.insertOutcomes(runs)
|
|
104
|
+
})
|
|
105
|
+
.immediate()
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** One `task_durations` row per executed success: what a forecast reads. */
|
|
109
|
+
private insertDurations(runs: readonly RunRecord[], source: string, sha: string | null): void {
|
|
110
|
+
const done = runs.filter((r) => r.status === 'success' && r.cacheHit !== true)
|
|
111
|
+
for (let i = 0; i < done.length; i += DURATIONS_PER_INSERT) {
|
|
112
|
+
const block = done.slice(i, i + DURATIONS_PER_INSERT)
|
|
113
|
+
const params: SQLQueryBindings[] = []
|
|
114
|
+
for (const r of block) params.push(r.project, r.task, source, sha, r.durationMs, r.endedAt)
|
|
115
|
+
this.db
|
|
116
|
+
.query(
|
|
117
|
+
`INSERT INTO task_durations(project, task, source, sha, duration_ms, recorded_at) VALUES ${Array(block.length).fill('(?, ?, ?, ?, ?, ?)').join(', ')}`,
|
|
118
|
+
)
|
|
119
|
+
.run(...params)
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* The flaky record's rows: every executed outcome, and a key's first hit,
|
|
125
|
+
* in the order given (a batch may hold one key's history).
|
|
126
|
+
*/
|
|
127
|
+
private insertOutcomes(runs: readonly RunRecord[]): void {
|
|
128
|
+
const rows = runs.filter(
|
|
129
|
+
(r) => keyed(r) && (isHit(r) || r.status === 'success' || r.status === 'failed'),
|
|
130
|
+
)
|
|
131
|
+
const hits = rows.filter(isHit)
|
|
132
|
+
let passed: Set<string> | undefined
|
|
133
|
+
if (hits.length > 0) {
|
|
134
|
+
const found = this.db.query(PASSED_HASHES).all(JSON.stringify(hits.map((r) => r.hash))) as {
|
|
135
|
+
hash: string
|
|
136
|
+
}[]
|
|
137
|
+
passed = new Set(found.map((f) => f.hash))
|
|
138
|
+
}
|
|
139
|
+
const kept: RunRecord[] = []
|
|
140
|
+
for (const r of rows) {
|
|
141
|
+
if (isHit(r)) {
|
|
142
|
+
if (passed!.has(r.hash!)) continue
|
|
143
|
+
passed!.add(r.hash!)
|
|
144
|
+
} else if (r.status === 'success') {
|
|
145
|
+
passed?.add(r.hash!)
|
|
146
|
+
}
|
|
147
|
+
kept.push(r)
|
|
148
|
+
}
|
|
149
|
+
for (let i = 0; i < kept.length; i += OUTCOMES_PER_INSERT) {
|
|
150
|
+
const block = kept.slice(i, i + OUTCOMES_PER_INSERT)
|
|
151
|
+
const params: SQLQueryBindings[] = []
|
|
152
|
+
for (const r of block)
|
|
153
|
+
params.push(r.project, r.task, r.hash!, r.status, r.attempts ?? null, r.startedAt)
|
|
154
|
+
this.db
|
|
155
|
+
.query(
|
|
156
|
+
`INSERT INTO task_outcomes(project, task, hash, status, attempts, recorded_at) VALUES ${Array(block.length).fill('(?, ?, ?, ?, ?, ?)').join(', ')}`,
|
|
157
|
+
)
|
|
158
|
+
.run(...params)
|
|
159
|
+
}
|
|
75
160
|
}
|
|
76
161
|
|
|
77
162
|
/** `runs` as few INSERTs; each distinct forward-args list digested once. */
|
|
@@ -109,6 +194,12 @@ export class RunHistory {
|
|
|
109
194
|
.transaction(() => {
|
|
110
195
|
this.insertRuns(bundle.runs)
|
|
111
196
|
this.insertInvocation.run(...bindInvocation(bundle.invocation))
|
|
197
|
+
this.insertDurations(
|
|
198
|
+
bundle.runs,
|
|
199
|
+
branchSource(bundle.invocation.branch),
|
|
200
|
+
bundle.invocation.commitSha,
|
|
201
|
+
)
|
|
202
|
+
this.insertOutcomes(bundle.runs)
|
|
112
203
|
})
|
|
113
204
|
.immediate()
|
|
114
205
|
}
|
|
@@ -116,21 +207,20 @@ export class RunHistory {
|
|
|
116
207
|
/**
|
|
117
208
|
* Retention: the runs table grows by one row per executed task per
|
|
118
209
|
* invocation — a 2000-task repo accretes ~20k rows in days. 30 days
|
|
119
|
-
* covers `vx
|
|
210
|
+
* covers `vx info` (24 h windows) and CI-side consumers. The
|
|
120
211
|
* `invocations` header is pruned on the SAME window, or a header would
|
|
121
212
|
* outlive its rows and `vx last` would list a run whose detail is gone.
|
|
122
213
|
*/
|
|
123
214
|
pruneOlderThan(cutoff: number): void {
|
|
124
215
|
this.db.prepare('DELETE FROM runs WHERE started_at < ?').run(cutoff)
|
|
125
216
|
this.db.prepare('DELETE FROM invocations WHERE started_at < ?').run(cutoff)
|
|
217
|
+
this.db.prepare('DELETE FROM task_durations WHERE recorded_at < ?').run(cutoff)
|
|
218
|
+
this.db.prepare('DELETE FROM task_outcomes WHERE recorded_at < ?').run(cutoff)
|
|
219
|
+
this.db.prepare('DELETE FROM plugin_blobs WHERE at < ?').run(cutoff)
|
|
126
220
|
}
|
|
127
221
|
}
|
|
128
222
|
|
|
129
|
-
/**
|
|
130
|
-
* Bind a RunRecord to the positional parameters expected by the
|
|
131
|
-
* `insertRun` prepared statement (22 columns). Shared between the
|
|
132
|
-
* single and batched record paths.
|
|
133
|
-
*/
|
|
223
|
+
/** Bind a RunRecord to one `RUN_TUPLE`'s 22 positional parameters. */
|
|
134
224
|
function bindRun(run: RunRecord, digestValue: (v: string) => string): SQLQueryBindings[] {
|
|
135
225
|
return [
|
|
136
226
|
// The ONE place the no-key sentinel is applied, so the column's
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
// What core keeps of a run that writes a remote (owner 2026-10-15): a failed
|
|
2
|
+
// task's log and the files its outputs left, an uncached task's log. Names
|
|
3
|
+
// come from the run id and task (`run/<id>/<task>/log`); each key is derived
|
|
4
|
+
// in a namespace no task key or plugin entry shares. Remote only: locally
|
|
5
|
+
// `vx last --log` and the working tree already hold them. No size cap (M:
|
|
6
|
+
// the cache is free). Plugins read them by name (`ctx.cache.getRun`).
|
|
7
|
+
|
|
8
|
+
import { xxh3hex } from '../util/index.js'
|
|
9
|
+
import type { PluginCacheRemote } from './plugin-cache.js'
|
|
10
|
+
import { scopedKey } from './scoped-cache.js'
|
|
11
|
+
import { tarPackInto, tarSize } from './tar-stream.js'
|
|
12
|
+
|
|
13
|
+
/** The name core stores a task's `part` of run `runId` under. */
|
|
14
|
+
export function runEntryName(runId: string, taskId: string, part: 'log' | 'outputs'): string {
|
|
15
|
+
return `run/${runId}/${taskId}/${part}`
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const runKey = (name: string): string => xxh3hex(`vx-run\0${name}`)
|
|
19
|
+
|
|
20
|
+
/** The keys `name` is read under: the default, then the scope's, as tasks read. */
|
|
21
|
+
export function runReadKeys(name: string, scope: string | undefined): string[] {
|
|
22
|
+
const key = runKey(name)
|
|
23
|
+
return scope === undefined ? [key] : [key, scopedKey(scope, key)]
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Whether some remote takes this run's writes: only then does core keep run bytes. */
|
|
27
|
+
export function writesRunStore(remotes: readonly PluginCacheRemote[]): boolean {
|
|
28
|
+
return remotes.some((r) => r.write)
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Store `bytes` under `name` in every remote the policy writes. An entry is
|
|
33
|
+
* written once; a remote's failure leaves the others theirs, silently: what
|
|
34
|
+
* a run keeps here is a record, never its verdict.
|
|
35
|
+
*/
|
|
36
|
+
export async function putRunEntry(
|
|
37
|
+
remotes: readonly PluginCacheRemote[],
|
|
38
|
+
scope: string | undefined,
|
|
39
|
+
name: string,
|
|
40
|
+
bytes: Uint8Array,
|
|
41
|
+
): Promise<boolean> {
|
|
42
|
+
const key = scope === undefined ? runKey(name) : scopedKey(scope, runKey(name))
|
|
43
|
+
let kept = false
|
|
44
|
+
for (const { remote, write } of remotes) {
|
|
45
|
+
if (!write) continue
|
|
46
|
+
try {
|
|
47
|
+
if (!(await remote.has(key))) await remote.put(key, new Blob([bytes]), { durationMs: 0 })
|
|
48
|
+
kept = true
|
|
49
|
+
} catch {
|
|
50
|
+
// the other remotes still keep it
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return kept
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* The files at `paths` (workspace-relative) as a plain ustar archive, names
|
|
58
|
+
* as given, so any `tar` reads it; null when none is left to read.
|
|
59
|
+
*/
|
|
60
|
+
export async function packRunOutputs(
|
|
61
|
+
workspaceRoot: string,
|
|
62
|
+
paths: readonly string[],
|
|
63
|
+
): Promise<Uint8Array | null> {
|
|
64
|
+
const inputs: { name: string; size: number; body: Uint8Array }[] = []
|
|
65
|
+
for (const name of paths) {
|
|
66
|
+
try {
|
|
67
|
+
const body = await Bun.file(`${workspaceRoot}/${name}`).bytes()
|
|
68
|
+
inputs.push({ name, size: body.byteLength, body })
|
|
69
|
+
} catch {
|
|
70
|
+
// gone since the glob: nothing to keep
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
if (inputs.length === 0) return null
|
|
74
|
+
const out = new Uint8Array(tarSize(inputs))
|
|
75
|
+
tarPackInto(inputs, out)
|
|
76
|
+
return out
|
|
77
|
+
}
|