@vzn/vx 0.0.641 → 0.0.643
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -12
- package/skills/vx/SKILL.md +4 -4
- package/src/cache/archive.ts +525 -104
- package/src/cache/cache.ts +85 -11
- package/src/cache/fold-memo.ts +58 -0
- package/src/cache/git-inputs.ts +213 -31
- package/src/cache/index.ts +12 -3
- package/src/cache/inputs.ts +169 -79
- package/src/cache/key-fold.ts +90 -69
- package/src/cache/layer.ts +4 -6
- package/src/cache/layered-cache.ts +96 -10
- package/src/cache/output-index.ts +25 -4
- package/src/cache/plugin-cache.ts +23 -16
- package/src/cache/remote-lane.ts +25 -8
- package/src/cache/run-store.ts +4 -1
- package/src/cache/schema.ts +13 -3
- package/src/cache/scoped-cache.ts +74 -237
- package/src/cache/umask.ts +29 -0
- package/src/cli/docs.ts +71 -7
- package/src/cli/help.ts +6 -8
- package/src/cli/plan-format.ts +4 -1
- package/src/cli/plugin-commands.ts +11 -6
- package/src/cli/run.ts +14 -15
- package/src/cli/select.ts +9 -9
- package/src/cli/workspace-config.ts +8 -2
- package/src/config.ts +7 -16
- package/src/exec/runner.ts +5 -0
- package/src/exec/sandbox-runtime.ts +81 -1
- package/src/graph/priorities.ts +14 -1
- package/src/history/history.ts +43 -33
- package/src/index.ts +14 -0
- package/src/orchestrator/affected-tasks.ts +23 -2
- package/src/orchestrator/deferred-outputs.ts +3 -2
- package/src/orchestrator/execute-task.ts +20 -14
- package/src/orchestrator/hit-restore.ts +10 -2
- package/src/orchestrator/index.ts +5 -0
- package/src/orchestrator/lockfile-claim.ts +54 -8
- package/src/orchestrator/miss-save.ts +27 -17
- package/src/orchestrator/options.ts +9 -1
- package/src/orchestrator/output-host.ts +9 -1
- package/src/orchestrator/plan.ts +4 -0
- package/src/orchestrator/plugin-host.ts +38 -4
- package/src/orchestrator/plugin.ts +49 -5
- package/src/orchestrator/prepare.ts +141 -22
- package/src/orchestrator/projects.ts +25 -6
- package/src/orchestrator/run-context.ts +80 -0
- package/src/orchestrator/run.ts +38 -15
- package/src/orchestrator/sandbox-request.ts +1 -48
- package/src/orchestrator/save-lane.ts +17 -10
- package/src/orchestrator/signals.ts +7 -10
- package/src/orchestrator/stable-keys.ts +83 -25
- package/src/orchestrator/task-hash.ts +52 -12
- package/src/report/events.ts +12 -0
- package/src/report/framed-output.ts +2 -1
- package/src/report/logger.ts +53 -13
- package/src/report/status-line.ts +45 -4
- package/src/report/summary.ts +8 -2
- package/src/settings.ts +11 -11
- package/src/util/errors.ts +1 -1
- package/src/util/hidden-prompt.ts +74 -0
- package/src/util/index.ts +12 -2
- package/src/util/paths.ts +33 -14
- package/src/util/secrets.ts +150 -0
- package/src/workspace/affected.ts +182 -75
- package/src/workspace/config-schema.ts +81 -19
- package/src/workspace/index.ts +3 -0
- package/src/workspace/project-loader.ts +1 -0
- package/src/workspace/workspace.ts +17 -11
- package/docs/caching.md +0 -2376
- package/docs/cli.md +0 -3285
- package/docs/execution.md +0 -676
- package/docs/features.md +0 -178
- package/docs/patterns.md +0 -219
- package/docs/schema.md +0 -2468
- package/docs/security.md +0 -253
- package/src/cli/docs-corpus.ts +0 -21
- package/src/cli/docs-text.d.ts +0 -5
package/src/cli/select.ts
CHANGED
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
refIsHead,
|
|
13
13
|
applyFilters,
|
|
14
14
|
buildPackageGraph,
|
|
15
|
+
configuredBaseBranch,
|
|
15
16
|
defaultAffectedBase,
|
|
16
17
|
findWorkspaceRoot,
|
|
17
18
|
type FingerprintClaims,
|
|
@@ -36,6 +37,7 @@ import {
|
|
|
36
37
|
} from '../util/index.js'
|
|
37
38
|
import {
|
|
38
39
|
claimedAffected,
|
|
40
|
+
claimedExtraFiles,
|
|
39
41
|
fingerprintClaims,
|
|
40
42
|
gitOfDiscovery,
|
|
41
43
|
hasHook,
|
|
@@ -48,7 +50,6 @@ import {
|
|
|
48
50
|
loadCliProjects,
|
|
49
51
|
loadCliWorkspace,
|
|
50
52
|
} from './workspace-config.js'
|
|
51
|
-
import { resolveSetting } from '../settings.js'
|
|
52
53
|
|
|
53
54
|
/**
|
|
54
55
|
* The projects whose tasks declare an `inputs.workspaceFiles` glob
|
|
@@ -123,6 +124,7 @@ async function workspaceFingerprintClaims(
|
|
|
123
124
|
return {
|
|
124
125
|
files: new Set(claims.keys()),
|
|
125
126
|
affected: (change) => claimedAffected(claims.get(change.file)!, change, ctx),
|
|
127
|
+
extraFiles: (file, bytes) => claimedExtraFiles(claims.get(file)!, file, bytes),
|
|
126
128
|
}
|
|
127
129
|
}
|
|
128
130
|
|
|
@@ -242,20 +244,18 @@ function refusalOf(err: unknown): Refusal {
|
|
|
242
244
|
|
|
243
245
|
/**
|
|
244
246
|
* The filter `--affected[=<base>]` stands for: `...[<base>]`, the changed
|
|
245
|
-
* projects and their dependents. An empty base is `
|
|
246
|
-
*
|
|
247
|
-
* `defaultBase` or `TURBO_SCM_BASE`), then the guess.
|
|
247
|
+
* projects and their dependents. An empty base is a plugin's (`setBranch`:
|
|
248
|
+
* a pull request's base commit), else the merge base with `baseBranch`.
|
|
248
249
|
*/
|
|
249
250
|
export async function affectedFilterFor(cwd: string, affected: string): Promise<string | Refusal> {
|
|
250
251
|
const root = await findWorkspaceRoot(cwd)
|
|
251
252
|
let base = affected
|
|
252
253
|
if (base === '') {
|
|
253
|
-
const ws =
|
|
254
|
-
base = (resolveSetting('run.affectedBase', ws).value as string | undefined) ?? ''
|
|
255
|
-
}
|
|
256
|
-
if (base === '') {
|
|
254
|
+
const ws = await loadCliWorkspace(root)
|
|
257
255
|
try {
|
|
258
|
-
base =
|
|
256
|
+
base =
|
|
257
|
+
ws.branch.base ??
|
|
258
|
+
(await defaultAffectedBase(root, configuredBaseBranch(ws.workspaceConfig)))
|
|
259
259
|
} catch (err) {
|
|
260
260
|
if (!isUserError(err)) throw err
|
|
261
261
|
return refusalOf(err)
|
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
lockOfDiscovery,
|
|
16
16
|
lockReader,
|
|
17
17
|
} from '../orchestrator/index.js'
|
|
18
|
-
import type { VxPlugin } from '../orchestrator/index.js'
|
|
18
|
+
import type { BranchFacts, VxPlugin } from '../orchestrator/index.js'
|
|
19
19
|
import {
|
|
20
20
|
buildPackageGraph,
|
|
21
21
|
type ProjectEntry,
|
|
@@ -27,6 +27,8 @@ import {
|
|
|
27
27
|
interface CliWorkspace {
|
|
28
28
|
workspaceConfig: WorkspaceConfig | null
|
|
29
29
|
plugins: readonly VxPlugin[]
|
|
30
|
+
/** What a plugin's `config` stage said of the branch: its cache scope and base commit. */
|
|
31
|
+
branch: BranchFacts
|
|
30
32
|
cacheDir: string
|
|
31
33
|
/** The shared store a run opens `cacheDir` with (`resolveStoreRoot`), asked on load. */
|
|
32
34
|
storeRoot: Promise<string | null>
|
|
@@ -37,10 +39,14 @@ export const warnToStderr = (message: string): void => {
|
|
|
37
39
|
}
|
|
38
40
|
|
|
39
41
|
export async function loadCliWorkspace(workspaceRoot: string): Promise<CliWorkspace> {
|
|
40
|
-
const { workspaceConfig, plugins } = await loadWorkspacePlugins(
|
|
42
|
+
const { workspaceConfig, plugins, branch } = await loadWorkspacePlugins(
|
|
43
|
+
workspaceRoot,
|
|
44
|
+
warnToStderr,
|
|
45
|
+
)
|
|
41
46
|
return {
|
|
42
47
|
workspaceConfig,
|
|
43
48
|
plugins,
|
|
49
|
+
branch,
|
|
44
50
|
cacheDir: resolveCacheDir(workspaceRoot, workspaceConfig),
|
|
45
51
|
storeRoot: resolveStoreRoot(workspaceRoot, workspaceConfig),
|
|
46
52
|
}
|
package/src/config.ts
CHANGED
|
@@ -14,6 +14,13 @@ export type Duration = `${number}${'ms' | 's' | 'm' | 'h' | 'd'}`
|
|
|
14
14
|
* Nothing here is folded into a cache key.
|
|
15
15
|
*/
|
|
16
16
|
export interface WorkspaceConfig {
|
|
17
|
+
/**
|
|
18
|
+
* The repository's default branch (`'develop'`). A run on it has no
|
|
19
|
+
* cache scope and a bare `--affected` has no default; elsewhere both
|
|
20
|
+
* compare with the merge base of `origin/<baseBranch>` and HEAD.
|
|
21
|
+
* Omitted → `origin/HEAD`'s branch, then `main`, then `master`.
|
|
22
|
+
*/
|
|
23
|
+
baseBranch?: string
|
|
17
24
|
run?: RunSettings
|
|
18
25
|
cache?: CacheSettings
|
|
19
26
|
lock?: LockSettings
|
|
@@ -42,13 +49,6 @@ export interface RunSettings {
|
|
|
42
49
|
* `schedule` plugin's weights still come first; this breaks their ties.
|
|
43
50
|
*/
|
|
44
51
|
schedule?: 'most-work' | 'critical-path' | 'direct-dependents' | 'ready-order'
|
|
45
|
-
/**
|
|
46
|
-
* The git ref a bare `--affected` compares with (`origin/develop`).
|
|
47
|
-
* `--affected=<base>` overrides it. Omitted → `origin/HEAD`, then the
|
|
48
|
-
* first of `origin/main`, `origin/master`, `main`, `master` that is not
|
|
49
|
-
* HEAD, else `HEAD~1`.
|
|
50
|
-
*/
|
|
51
|
-
affectedBase?: string
|
|
52
52
|
/**
|
|
53
53
|
* A task's time limit, for tasks without their own `exec.timeout` (which
|
|
54
54
|
* beats every run-level source). A runaway-process guard: a timed-out
|
|
@@ -95,15 +95,6 @@ export interface CacheSettings {
|
|
|
95
95
|
* first, then each remote in declared order.
|
|
96
96
|
*/
|
|
97
97
|
policy?: string
|
|
98
|
-
/**
|
|
99
|
-
* Where this run's cache writes land. Omitted, a run reads and writes the
|
|
100
|
-
* default keys. A name (`'pr-123'`) reads the default key first, then the
|
|
101
|
-
* scope's, and writes only the scope's, so a PR never writes what the
|
|
102
|
-
* default branch reads. Whether a run writes at all is `policy`'s call.
|
|
103
|
-
* A client-side convention, not a security boundary: the boundary is a
|
|
104
|
-
* cache token the server limits.
|
|
105
|
-
*/
|
|
106
|
-
scope?: string
|
|
107
98
|
/** Which remote hits land on disk: `all` (the default), only the requested tasks' (`toplevel`), or `none`. */
|
|
108
99
|
download?: 'all' | 'toplevel' | 'none'
|
|
109
100
|
/** The `cache.inputStability` of every task that does not set its own. */
|
package/src/exec/runner.ts
CHANGED
|
@@ -50,6 +50,10 @@ export interface RunResult {
|
|
|
50
50
|
/** `Bun.spawn` threw: no shell ran, so its 127 is vx's and says nothing
|
|
51
51
|
* about the command (A-41). */
|
|
52
52
|
spawnFailed?: true
|
|
53
|
+
/** The shell exited while a process it started still held its
|
|
54
|
+
* stdout/stderr past the drain bound (`POST_EXIT_CUT_LINE`): that process
|
|
55
|
+
* may still write the task's outputs, so the result is not saved. */
|
|
56
|
+
leftRunning?: true
|
|
53
57
|
}
|
|
54
58
|
|
|
55
59
|
/**
|
|
@@ -1034,6 +1038,7 @@ export async function runCommand(opts: RunOptions): Promise<RunResult> {
|
|
|
1034
1038
|
stderr,
|
|
1035
1039
|
...(proc.signalCode ? { signal: proc.signalCode } : {}),
|
|
1036
1040
|
...(timeout.timedOut() ? { timedOut: true } : {}),
|
|
1041
|
+
...(cut ? { leftRunning: true } : {}),
|
|
1037
1042
|
...resourceUsageToCpuRss(proc.resourceUsage(), rssFloorFor),
|
|
1038
1043
|
}
|
|
1039
1044
|
}
|
|
@@ -1344,7 +1344,7 @@ async function wrapIn(
|
|
|
1344
1344
|
if (process.platform === 'darwin') {
|
|
1345
1345
|
const rules = [
|
|
1346
1346
|
...macProfileRules(args.config),
|
|
1347
|
-
...
|
|
1347
|
+
...darwinGrantRules(args.config, baselines.allowRead, baselines.denyRead),
|
|
1348
1348
|
]
|
|
1349
1349
|
try {
|
|
1350
1350
|
if (rules.length > 0) wrapped = injectProfileRules(wrapped, rules)
|
|
@@ -2270,6 +2270,7 @@ async function runSandboxedOnce(
|
|
|
2270
2270
|
violations,
|
|
2271
2271
|
...(proc.signalCode ? { signal: proc.signalCode } : {}),
|
|
2272
2272
|
...(timeout.timedOut() ? { timedOut: true } : {}),
|
|
2273
|
+
...(cut ? { leftRunning: true } : {}),
|
|
2273
2274
|
// Linux: the task's CPU and peak never reach this wait. bwrap runs it
|
|
2274
2275
|
// in a pid namespace (`--unshare-pid`), and what the namespace's
|
|
2275
2276
|
// processes used is not folded into bwrap's usage: a 500 ms busy loop
|
|
@@ -2570,6 +2571,85 @@ export function wallsGlobsReach(grants: readonly string[], walls: readonly strin
|
|
|
2570
2571
|
return walls.filter((w) => heads.some((h) => atOrUnder(w, h)))
|
|
2571
2572
|
}
|
|
2572
2573
|
|
|
2574
|
+
/**
|
|
2575
|
+
* macOS: the grant rules vx appends. The listings come first so a wall's
|
|
2576
|
+
* deny, later, wins: SBPL is last-match-wins.
|
|
2577
|
+
*/
|
|
2578
|
+
export function darwinGrantRules(
|
|
2579
|
+
c: Pick<ResolvedSandboxConfig, 'wallsReached' | 'allowRead' | 'allowWrite'>,
|
|
2580
|
+
baseAllowRead: readonly string[],
|
|
2581
|
+
anchors: readonly string[],
|
|
2582
|
+
): string[] {
|
|
2583
|
+
return [...darwinAncestorRules(c, anchors), ...darwinWallRules(c, baseAllowRead)]
|
|
2584
|
+
}
|
|
2585
|
+
|
|
2586
|
+
/**
|
|
2587
|
+
* macOS: each directory above a grant may be listed, as on Linux. bwrap
|
|
2588
|
+
* makes every directory above a mount, so a task granted
|
|
2589
|
+
* `../vx-*\/package.json` could glob it from `..`; seatbelt matches paths
|
|
2590
|
+
* and refused the listing (EPERM), so `@vzn/vx-docs#import` failed on
|
|
2591
|
+
* every Mac (2026-10-17). Above the glob that is each literal directory
|
|
2592
|
+
* strictly inside a deny anchor (outside one, all is readable already);
|
|
2593
|
+
* a directory the glob itself names (`vx-a/`) is matched as a regex, and
|
|
2594
|
+
* under a `**` any directory is. The rules grant directories only, never
|
|
2595
|
+
* a file, and come before the wall denies, which still win.
|
|
2596
|
+
*
|
|
2597
|
+
* A listing shows every name in the directory, where Linux's scaffold
|
|
2598
|
+
* shows only what it mounted: names, never a file's data. Not the anchor
|
|
2599
|
+
* itself: Bun, shown the workspace root's `package.json` it
|
|
2600
|
+
* may not read, failed with "Cannot read file" (EPERM).
|
|
2601
|
+
*/
|
|
2602
|
+
export function darwinAncestorRules(
|
|
2603
|
+
c: Pick<ResolvedSandboxConfig, 'allowRead' | 'allowWrite'>,
|
|
2604
|
+
anchors: readonly string[],
|
|
2605
|
+
): string[] {
|
|
2606
|
+
const literals = new Set<string>()
|
|
2607
|
+
const patterns = new Set<string>()
|
|
2608
|
+
// Grants are canonical already (`resolveSandboxConfig`), and so is each
|
|
2609
|
+
// directory above one: no realpath here, at every spawn.
|
|
2610
|
+
const inside = (d: string): boolean => anchors.some((a) => d !== a && atOrUnder(d, a))
|
|
2611
|
+
for (const g of [...c.allowRead, ...c.allowWrite]) {
|
|
2612
|
+
const named = namedPath(g)
|
|
2613
|
+
const head = g.slice(0, g.search(MOUNT_WILDCARDS))
|
|
2614
|
+
const cut = head.lastIndexOf('/')
|
|
2615
|
+
const top = named !== undefined ? path.dirname(named) : head.slice(0, cut) || '/'
|
|
2616
|
+
for (let d = top; inside(d); d = path.dirname(d)) literals.add(d)
|
|
2617
|
+
if (named !== undefined || g.includes('\\')) continue
|
|
2618
|
+
let re = top === '/' ? '' : regexLiteral(top)
|
|
2619
|
+
for (const part of g
|
|
2620
|
+
.slice(cut + 1)
|
|
2621
|
+
.split('/')
|
|
2622
|
+
.slice(0, -1)) {
|
|
2623
|
+
if (part === '**') {
|
|
2624
|
+
patterns.add(`^${re}(/.*)?$`)
|
|
2625
|
+
break
|
|
2626
|
+
}
|
|
2627
|
+
re += `/${part
|
|
2628
|
+
.replace(/[.^$+{}()|]/g, '[$&]')
|
|
2629
|
+
.replaceAll('*', '[^/]*')
|
|
2630
|
+
.replaceAll('?', '[^/]')}`
|
|
2631
|
+
patterns.add(`^${re}$`)
|
|
2632
|
+
}
|
|
2633
|
+
}
|
|
2634
|
+
// A name seatbelt's quoted string cannot hold keeps today's refusal
|
|
2635
|
+
// rather than failing the task (`sbplResolvedPath` would throw).
|
|
2636
|
+
const spellable = (p: string): boolean => !UNSPELLABLE.test(p)
|
|
2637
|
+
return [
|
|
2638
|
+
...[...literals].filter(spellable).map((d) => `(allow file-read-data (literal "${d}"))`),
|
|
2639
|
+
...[...patterns]
|
|
2640
|
+
.filter(spellable)
|
|
2641
|
+
.map((r) => `(allow file-read-data (require-all (vnode-type DIRECTORY) (regex #"${r}")))`),
|
|
2642
|
+
]
|
|
2643
|
+
}
|
|
2644
|
+
|
|
2645
|
+
// eslint-disable-next-line no-control-regex -- the control range is the point
|
|
2646
|
+
const UNSPELLABLE = /["'\\\x00-\x1f\x7f]/
|
|
2647
|
+
|
|
2648
|
+
/** A real path as a regex matching only itself: each special in a class of one. */
|
|
2649
|
+
function regexLiteral(p: string): string {
|
|
2650
|
+
return p.replace(/[.^$+{}()|*?[\]]/g, (ch) => `[${ch}]`)
|
|
2651
|
+
}
|
|
2652
|
+
|
|
2573
2653
|
/**
|
|
2574
2654
|
* macOS: the walls a glob grant reaches, denied at the profile's tail
|
|
2575
2655
|
* (SBPL is last-match-wins). SRT re-emits a wall's deny after the allows
|
package/src/graph/priorities.ts
CHANGED
|
@@ -23,7 +23,20 @@ export function tieredReverseDepCount(
|
|
|
23
23
|
restoreTier: ReadonlySet<string>,
|
|
24
24
|
): Map<string, number> {
|
|
25
25
|
const exec = new Map<string, TaskNode>()
|
|
26
|
-
|
|
26
|
+
let runsAny = false
|
|
27
|
+
for (const [id, node] of nodes) {
|
|
28
|
+
if (restoreTier.has(id)) continue
|
|
29
|
+
exec.set(id, node)
|
|
30
|
+
if (node.config.exec !== undefined) runsAny = true
|
|
31
|
+
}
|
|
32
|
+
// Nothing outside the restore tier runs a command (a warm run: restores
|
|
33
|
+
// and groups), so no order finishes any work sooner: everything ranks 0.
|
|
34
|
+
// The weights below were 10 ms of a 1,000-project warm run.
|
|
35
|
+
if (!runsAny) {
|
|
36
|
+
const zeros = new Map<string, number>()
|
|
37
|
+
for (const id of nodes.keys()) zeros.set(id, 0)
|
|
38
|
+
return zeros
|
|
39
|
+
}
|
|
27
40
|
// An exec tier with no edge inside it (a warm run's groups) counts 0
|
|
28
41
|
// everywhere: no closure to build.
|
|
29
42
|
let linked = false
|
package/src/history/history.ts
CHANGED
|
@@ -263,56 +263,66 @@ export class LocalHistoryProvider implements HistoryProvider {
|
|
|
263
263
|
* newest imported timing (`importTimings`), else the median on any
|
|
264
264
|
* branch, else none. The timings are the shared store's `task_durations`, so every
|
|
265
265
|
* worktree of the repo reads them and a new one forecasts on its first
|
|
266
|
-
* run.
|
|
267
|
-
* however old the rows, where the
|
|
268
|
-
* invocations.
|
|
266
|
+
* run. One statement per tier, an index probe per task on `(project,
|
|
267
|
+
* task, source, id)` inside it: exact however old the rows, where the
|
|
268
|
+
* runs window holds only the newest invocations.
|
|
269
269
|
*/
|
|
270
270
|
private forecasts(wanted: ReadonlySet<string>): Map<string, number> {
|
|
271
271
|
const out = new Map<string, number>()
|
|
272
272
|
this.lackedBranchHistory = false
|
|
273
273
|
if (wanted.size === 0) return out
|
|
274
|
-
// Tiers, first answer wins
|
|
275
|
-
|
|
274
|
+
// Tiers, first answer wins: a WHERE over `task_durations`, its params,
|
|
275
|
+
// and how many of the newest rows it reads.
|
|
276
|
+
type Tier = { where: string; params: string[]; limit: number }
|
|
276
277
|
const tiers: Tier[] = []
|
|
277
|
-
const newest = (where: string) =>
|
|
278
|
-
this.db.query(
|
|
279
|
-
`SELECT duration_ms AS ms FROM task_durations WHERE project = ? AND task = ? AND ${where}
|
|
280
|
-
ORDER BY id DESC LIMIT ${FORECAST_RUNS}`,
|
|
281
|
-
)
|
|
282
278
|
if (this.branch !== undefined) {
|
|
283
|
-
|
|
284
|
-
const q = newest('source = ?')
|
|
285
|
-
tiers.push((p, t) => q.all(p, t, source) as Array<{ ms: number }>)
|
|
279
|
+
tiers.push({ where: 'source = ?', params: [branchSource(this.branch)], limit: FORECAST_RUNS })
|
|
286
280
|
}
|
|
287
281
|
// One probe on the partial index first: a store with no import costs
|
|
288
|
-
// no
|
|
282
|
+
// no tier.
|
|
289
283
|
if (
|
|
290
284
|
this.db.query(`SELECT 1 FROM task_durations WHERE source = 'remote' LIMIT 1`).get() !== null
|
|
291
285
|
) {
|
|
292
|
-
|
|
293
|
-
`SELECT duration_ms AS ms FROM task_durations WHERE project = ? AND task = ? AND source = 'remote'
|
|
294
|
-
ORDER BY id DESC LIMIT 1`,
|
|
295
|
-
)
|
|
296
|
-
tiers.push((p, t) => remote.all(p, t) as Array<{ ms: number }>)
|
|
286
|
+
tiers.push({ where: `source = 'remote'`, params: [], limit: 1 })
|
|
297
287
|
}
|
|
298
|
-
|
|
299
|
-
tiers.push((p, t) => any.all(p, t) as Array<{ ms: number }>)
|
|
288
|
+
tiers.push({ where: `source LIKE 'branch:%'`, params: [], limit: FORECAST_RUNS })
|
|
300
289
|
const own = this.branch !== undefined ? tiers[0] : tiers.at(-1)
|
|
301
|
-
let
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
290
|
+
let pending = [...wanted]
|
|
291
|
+
let ownAnswered = 0
|
|
292
|
+
for (const tier of tiers) {
|
|
293
|
+
if (pending.length === 0) break
|
|
294
|
+
// One statement per tier over every task it is asked for: an index
|
|
295
|
+
// probe per task inside SQLite, where a query per task from here was
|
|
296
|
+
// 31-40 ms of 2,000 tasks against 5-8.
|
|
297
|
+
const rows = this.db
|
|
298
|
+
.query(
|
|
299
|
+
`SELECT j.key AS i, (SELECT group_concat(ms) FROM (
|
|
300
|
+
SELECT duration_ms AS ms FROM task_durations
|
|
301
|
+
WHERE project = j.value ->> 0 AND task = j.value ->> 1 AND ${tier.where}
|
|
302
|
+
ORDER BY id DESC LIMIT ${tier.limit})) AS ds
|
|
303
|
+
FROM json_each(?) AS j`,
|
|
304
|
+
)
|
|
305
|
+
.all(...tier.params, JSON.stringify(pending.map(splitTaskId))) as {
|
|
306
|
+
i: number
|
|
307
|
+
ds: string | null
|
|
308
|
+
}[]
|
|
309
|
+
const left: string[] = []
|
|
310
|
+
for (const r of rows) {
|
|
311
|
+
const id = pending[r.i]!
|
|
312
|
+
if (r.ds === null) {
|
|
313
|
+
left.push(id)
|
|
314
|
+
continue
|
|
315
|
+
}
|
|
316
|
+
const durations = r.ds
|
|
317
|
+
.split(',')
|
|
318
|
+
.map(Number)
|
|
319
|
+
.sort((x, y) => x - y)
|
|
309
320
|
out.set(id, pickPercentile(durations, 0.5))
|
|
310
|
-
|
|
311
|
-
break
|
|
321
|
+
if (tier === own) ownAnswered++
|
|
312
322
|
}
|
|
313
|
-
|
|
323
|
+
pending = left
|
|
314
324
|
}
|
|
315
|
-
this.lackedBranchHistory =
|
|
325
|
+
this.lackedBranchHistory = ownAnswered < wanted.size
|
|
316
326
|
return out
|
|
317
327
|
}
|
|
318
328
|
|
package/src/index.ts
CHANGED
|
@@ -46,6 +46,7 @@ export type {
|
|
|
46
46
|
} from './config.js'
|
|
47
47
|
export { defineProject, defineWorkspace } from './config.js'
|
|
48
48
|
export { definePlugin, refuseUnknownOptions } from './orchestrator/index.js'
|
|
49
|
+
export { branchCacheScope } from './workspace/index.js'
|
|
49
50
|
|
|
50
51
|
// Programmatic engine API: run / plan / prepare (docs/cli.md § Programmatic
|
|
51
52
|
// API). The graph primitives, the cache-key hashing seam and the git / host
|
|
@@ -83,9 +84,20 @@ export type {
|
|
|
83
84
|
CacheLayer,
|
|
84
85
|
PluginCache,
|
|
85
86
|
RemoteCacheLayer,
|
|
87
|
+
RemoteKey,
|
|
88
|
+
RemoteOpenContext,
|
|
86
89
|
RunRecord,
|
|
87
90
|
InvocationRecord,
|
|
88
91
|
} from './cache/index.js'
|
|
92
|
+
// A plugin's keychain (`ctx.secret` on `setup` and a remote's `open`), and
|
|
93
|
+
// the guard that tells a missing secret from a fault.
|
|
94
|
+
export { isSecretUnavailable } from './util/index.js'
|
|
95
|
+
export type {
|
|
96
|
+
PluginSecrets,
|
|
97
|
+
SecretRequest,
|
|
98
|
+
SecretUnavailable,
|
|
99
|
+
SecretUnavailableReason,
|
|
100
|
+
} from './util/index.js'
|
|
89
101
|
|
|
90
102
|
// Workspace discovery + the project/config catalog surface — an
|
|
91
103
|
// out-of-process service/CLI needs these. `loadProjectConfig` is the RAW
|
|
@@ -161,6 +173,8 @@ export type {
|
|
|
161
173
|
ScheduleHookContext,
|
|
162
174
|
AdmitContext,
|
|
163
175
|
WorkspaceHookContext,
|
|
176
|
+
ConfigContext,
|
|
177
|
+
BranchFacts,
|
|
164
178
|
DiscoverContext,
|
|
165
179
|
DiscoveredProject,
|
|
166
180
|
PluginSetupContext,
|
|
@@ -41,6 +41,7 @@ const KEYED_BY_EVERY_TASK: ReadonlySet<string> = new Set(PROJECT_CONFIG_FILENAME
|
|
|
41
41
|
function wholeProjects(
|
|
42
42
|
changes: AffectedChanges,
|
|
43
43
|
projects: ReadonlyMap<string, ProjectEntry>,
|
|
44
|
+
byKeys: boolean,
|
|
44
45
|
): Map<string, string | undefined> {
|
|
45
46
|
// Each whole project, with the path no task claims when one did it.
|
|
46
47
|
const whole = new Map<string, string | undefined>([...changes.whole].map((n) => [n, undefined]))
|
|
@@ -50,7 +51,7 @@ function wholeProjects(
|
|
|
50
51
|
(t) => t.exec !== undefined && t.inputs !== undefined,
|
|
51
52
|
)
|
|
52
53
|
const unclaimed = (rel: string): boolean =>
|
|
53
|
-
KEYED_BY_EVERY_TASK.has(rel) || !keyed.some((t) => declaresInput(t, rel, null))
|
|
54
|
+
KEYED_BY_EVERY_TASK.has(rel) || (!byKeys && !keyed.some((t) => declaresInput(t, rel, null)))
|
|
54
55
|
const rel = rels.find(unclaimed)
|
|
55
56
|
if (rel !== undefined) whole.set(name, rel)
|
|
56
57
|
}
|
|
@@ -99,8 +100,10 @@ export function affectedRoots(
|
|
|
99
100
|
projects: ReadonlyMap<string, ProjectEntry>,
|
|
100
101
|
packageGraph: PackageGraph,
|
|
101
102
|
explain?: AffectedExplain,
|
|
103
|
+
/** Only what moves a key: a changed path no task declares reaches no task. */
|
|
104
|
+
byKeys = false,
|
|
102
105
|
): string[] {
|
|
103
|
-
const whole = wholeProjects(changes, projects)
|
|
106
|
+
const whole = wholeProjects(changes, projects, byKeys)
|
|
104
107
|
const seeds = explain === undefined ? null : new Map<string, AffectedReason>()
|
|
105
108
|
const through = explain === undefined ? null : new Map<string, string>()
|
|
106
109
|
const inProject = (name: string, rel: string | undefined): AffectedReason => {
|
|
@@ -226,6 +229,24 @@ export function affectedRoots(
|
|
|
226
229
|
return kept
|
|
227
230
|
}
|
|
228
231
|
|
|
232
|
+
/**
|
|
233
|
+
* The tasks of `nodes` whose key the change can move (a branch's cache
|
|
234
|
+
* scope, owner 2026-10-17): a changed path one of its inputs declares, a
|
|
235
|
+
* whole project, or a task its closure holds. Unlike `--affected`, a path
|
|
236
|
+
* no task declares (a README) reaches none: its key is the default
|
|
237
|
+
* branch's, so it reads the default key.
|
|
238
|
+
*/
|
|
239
|
+
export function keyedByChange(
|
|
240
|
+
nodes: ReadonlyMap<string, TaskNode>,
|
|
241
|
+
changes: AffectedChanges,
|
|
242
|
+
projects: ReadonlyMap<string, ProjectEntry>,
|
|
243
|
+
packageGraph: PackageGraph,
|
|
244
|
+
): Set<string> {
|
|
245
|
+
return new Set(
|
|
246
|
+
affectedRoots(nodes, [...nodes.keys()], changes, projects, packageGraph, undefined, true),
|
|
247
|
+
)
|
|
248
|
+
}
|
|
249
|
+
|
|
229
250
|
/**
|
|
230
251
|
* The requested tasks `--affected` keeps, in their order: one the user
|
|
231
252
|
* named (`pkg#task`), one in a project another include selected outright
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
import {
|
|
16
16
|
cleanOutputs,
|
|
17
17
|
committedFiles,
|
|
18
|
-
|
|
18
|
+
resolveOutputTree,
|
|
19
19
|
type CacheLayer,
|
|
20
20
|
type GitFilesCache,
|
|
21
21
|
} from '../cache/index.js'
|
|
@@ -144,7 +144,7 @@ export class DeferredOutputs {
|
|
|
144
144
|
`could not fetch deferred outputs of ${taskId}: ${msg} — re-run it with --force, or use --cache-download=all`,
|
|
145
145
|
)
|
|
146
146
|
}
|
|
147
|
-
const outputFiles = await
|
|
147
|
+
const { files: outputFiles, emptyDirs } = await resolveOutputTree(cleanArgs)
|
|
148
148
|
this.args.gitFilesCache?.markOutputsChanged(
|
|
149
149
|
producer.projectDir,
|
|
150
150
|
outputFiles.map((f) => f.slice(producer.projectDir.length + 1)),
|
|
@@ -157,6 +157,7 @@ export class DeferredOutputs {
|
|
|
157
157
|
hash: entry.hash,
|
|
158
158
|
projectDir: producer.projectDir,
|
|
159
159
|
outputFiles,
|
|
160
|
+
...(emptyDirs.length > 0 ? { outputDirs: emptyDirs } : {}),
|
|
160
161
|
entry: entry.entry,
|
|
161
162
|
})
|
|
162
163
|
}
|
|
@@ -61,7 +61,6 @@ import {
|
|
|
61
61
|
type Placeholder,
|
|
62
62
|
reachedWithheld,
|
|
63
63
|
sandboxRequestFor,
|
|
64
|
-
sandboxReadable,
|
|
65
64
|
sweepPlaceholders,
|
|
66
65
|
undeclaredWriteReach,
|
|
67
66
|
untouchedPlaceholderLine,
|
|
@@ -233,7 +232,7 @@ export interface ExecuteArgs {
|
|
|
233
232
|
*/
|
|
234
233
|
outputDirSnapshots?: OutputDirSnapshot[]
|
|
235
234
|
/** The run's save lane: a miss's cache save runs off the execution slot (save-lane.ts). */
|
|
236
|
-
deferSave?: (save: () => Promise<void>) => Promise<
|
|
235
|
+
deferSave?: (save: () => Promise<void>) => Promise<boolean>
|
|
237
236
|
/**
|
|
238
237
|
* Where a deferred save's `landed` promise is parked by task id, for the
|
|
239
238
|
* in-flight join (admission.ts): a duplicate of this task in another run
|
|
@@ -1278,6 +1277,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1278
1277
|
} else if (
|
|
1279
1278
|
effectiveExitCode === 0 &&
|
|
1280
1279
|
willSave &&
|
|
1280
|
+
nothingLeftRunning() &&
|
|
1281
1281
|
(await keyStillTrue()) &&
|
|
1282
1282
|
(own = await ownOutputs()) !== undefined
|
|
1283
1283
|
) {
|
|
@@ -1302,19 +1302,11 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1302
1302
|
deferSave: args.deferSave,
|
|
1303
1303
|
// A sink listens: say what the save cost.
|
|
1304
1304
|
measure: args.explainMiss !== undefined,
|
|
1305
|
-
...(userSandbox
|
|
1306
|
-
? {
|
|
1307
|
-
readable: sandboxReadable(
|
|
1308
|
-
node,
|
|
1309
|
-
cfg.exec!.sandbox!,
|
|
1310
|
-
args.workspaceRoot,
|
|
1311
|
-
args.nestedProjectDirs,
|
|
1312
|
-
),
|
|
1313
|
-
}
|
|
1314
|
-
: {}),
|
|
1315
1305
|
})
|
|
1316
|
-
|
|
1317
|
-
|
|
1306
|
+
// A run that settles the outcome on `landed` learns there whether the
|
|
1307
|
+
// entry is in; without one, the save is taken as done.
|
|
1308
|
+
if (args.deferredSaves !== undefined) args.deferredSaves.set(node.id, landed)
|
|
1309
|
+
else saved = true
|
|
1318
1310
|
} else if (cfgCacheable && !remoteOnly && !deferralRequested && args.noDependants !== true) {
|
|
1319
1311
|
// Ran here and saves nothing — failed, a read-only policy, a tainted
|
|
1320
1312
|
// upstream, a key that no longer held — yet it wrote what a save would
|
|
@@ -1362,6 +1354,20 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1362
1354
|
return mine
|
|
1363
1355
|
}
|
|
1364
1356
|
|
|
1357
|
+
/**
|
|
1358
|
+
* A process the shell left behind still held its output when the drain
|
|
1359
|
+
* bound cut it, so it may still write outputs: a save now would file a
|
|
1360
|
+
* tree the run does not leave, and a hit elsewhere would replay it.
|
|
1361
|
+
*/
|
|
1362
|
+
function nothingLeftRunning(): boolean {
|
|
1363
|
+
if (result.leftRunning !== true) return true
|
|
1364
|
+
warnOf(log)(
|
|
1365
|
+
`[vx] ${node.id}: a process it left running may still write its outputs — ` +
|
|
1366
|
+
`the result stands, but is not saved; wait for it in the command (\`wait\`)`,
|
|
1367
|
+
)
|
|
1368
|
+
return false
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1365
1371
|
/**
|
|
1366
1372
|
* The input that no longer holds what the key folded, or one the key did
|
|
1367
1373
|
* not fold that is there now: `null` when the key re-derived just before
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
// which bytes a green run replays. Moved out of execute-task.ts
|
|
9
9
|
// 2026-09-10 as pure code motion, the mirror of miss-save.ts.
|
|
10
10
|
|
|
11
|
+
import path from 'node:path'
|
|
11
12
|
import type { CacheConfig } from '../config.js'
|
|
12
13
|
import {
|
|
13
14
|
type CacheEntry,
|
|
@@ -166,7 +167,7 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
166
167
|
actual: readonly string[],
|
|
167
168
|
exp: ReadonlyArray<{ path: string }>,
|
|
168
169
|
): boolean => {
|
|
169
|
-
const expSet = new Set(exp.map((e) => e.path))
|
|
170
|
+
const expSet = new Set(exp.map((e) => e.path).filter((p) => !p.endsWith('/')))
|
|
170
171
|
return actual.length === expSet.size && actual.every((r) => expSet.has(r))
|
|
171
172
|
}
|
|
172
173
|
// A stray a dependant's glob could have added is not a stray; a path
|
|
@@ -185,7 +186,7 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
185
186
|
exp: ReadonlyArray<{ path: string }>,
|
|
186
187
|
): boolean => {
|
|
187
188
|
const have = new Set(actual)
|
|
188
|
-
return exp.every((e) => have.has(e.path))
|
|
189
|
+
return exp.every((e) => e.path.endsWith('/') || have.has(e.path))
|
|
189
190
|
}
|
|
190
191
|
treeMatches = additive
|
|
191
192
|
? rowsPresent(actualRels, projExpected) && rowsPresent(actualWsRels, wsExpected)
|
|
@@ -289,6 +290,13 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
289
290
|
rels: staleRels,
|
|
290
291
|
outputs,
|
|
291
292
|
committed: cleanArgs.committed,
|
|
293
|
+
// A stale file in an output directory the entry holds empty takes
|
|
294
|
+
// only itself: the directory stays, as the miss left it.
|
|
295
|
+
keep: new Set(
|
|
296
|
+
heldRels
|
|
297
|
+
.filter((r) => r.endsWith('/'))
|
|
298
|
+
.map((r) => path.join(node.projectDir, r.slice(0, -1))),
|
|
299
|
+
),
|
|
292
300
|
})
|
|
293
301
|
}
|
|
294
302
|
if (staleWsRels.length > 0) {
|
|
@@ -50,6 +50,8 @@ export {
|
|
|
50
50
|
type ScheduleHookContext,
|
|
51
51
|
type AdmitContext,
|
|
52
52
|
type WorkspaceHookContext,
|
|
53
|
+
type ConfigContext,
|
|
54
|
+
type BranchFacts,
|
|
53
55
|
type DiscoverContext,
|
|
54
56
|
type DiscoveredProject,
|
|
55
57
|
type InstallPluginsArgs,
|
|
@@ -77,6 +79,7 @@ export {
|
|
|
77
79
|
applyProjectHooks,
|
|
78
80
|
applyScheduleHooks,
|
|
79
81
|
claimedAffected,
|
|
82
|
+
claimedExtraFiles,
|
|
80
83
|
fingerprintClaims,
|
|
81
84
|
hasHook,
|
|
82
85
|
REMOTE_CACHE_METHODS,
|
|
@@ -116,6 +119,8 @@ export {
|
|
|
116
119
|
captureWorkspaceIdentity,
|
|
117
120
|
currentBranch,
|
|
118
121
|
detectCi,
|
|
122
|
+
localBranchScope,
|
|
123
|
+
resolveCacheScope,
|
|
119
124
|
} from './run-context.js'
|
|
120
125
|
export type { CiContext, GitContext, HostContext, WorkspaceIdentity } from './run-context.js'
|
|
121
126
|
export {
|