@vzn/vx 0.0.643 → 0.0.645
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +15 -23
- package/package.json +5 -5
- package/schemas/info.json +0 -11
- package/schemas/init.json +5 -0
- package/skills/vx/SKILL.md +1 -1
- package/src/cache/archive.ts +89 -2
- package/src/cache/cache.ts +396 -176
- package/src/cache/file-hashes.ts +2 -14
- package/src/cache/fold-memo.ts +7 -3
- package/src/cache/index.ts +4 -2
- package/src/cache/key-fold.ts +4 -1
- package/src/cache/layer.ts +14 -10
- package/src/cache/layered-cache.ts +157 -41
- package/src/cache/plugin-cache.ts +40 -44
- package/src/cache/remote-lane.ts +59 -25
- package/src/cache/scoped-cache.ts +3 -80
- package/src/cache/tar-stream.ts +12 -7
- package/src/cli/format.ts +1 -2
- package/src/cli/help.ts +10 -4
- package/src/cli/info.ts +2 -19
- package/src/cli/init.ts +121 -20
- package/src/cli/plan-format.ts +6 -2
- package/src/cli/plugin-commands.ts +4 -5
- package/src/cli/plugin-templates.ts +1 -1
- package/src/cli/run.ts +3 -1
- package/src/cli/upgrade.ts +39 -9
- package/src/config.ts +13 -10
- package/src/exec/sandbox-paths.ts +2 -2
- package/src/graph/scheduler.ts +3 -1
- package/src/graph/task-graph.ts +15 -11
- package/src/index.ts +2 -2
- package/src/orchestrator/affected-tasks.ts +0 -18
- package/src/orchestrator/doctor.ts +0 -17
- package/src/orchestrator/execute-task.ts +1 -1
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/options.ts +6 -8
- package/src/orchestrator/plan.ts +7 -1
- package/src/orchestrator/plugin-host.ts +6 -5
- package/src/orchestrator/plugin.ts +12 -5
- package/src/orchestrator/prepare.ts +16 -46
- package/src/orchestrator/projects.ts +8 -1
- package/src/orchestrator/remote-prefetch.ts +11 -4
- package/src/orchestrator/run-context.ts +48 -5
- package/src/orchestrator/run-settings.ts +3 -0
- package/src/orchestrator/run.ts +10 -26
- package/src/orchestrator/task-hash.ts +1 -3
- package/src/report/logger.ts +0 -4
- package/src/report/summary.ts +14 -13
- package/src/settings.ts +38 -15
- package/src/util/index.ts +1 -0
- package/src/util/paths.ts +42 -33
- package/src/util/size.ts +4 -5
- package/src/workspace/affected.ts +0 -39
- package/src/workspace/config-schema.ts +0 -36
- package/src/workspace/index.ts +0 -1
|
@@ -10,7 +10,7 @@ export const PLUGIN_TEMPLATES: Readonly<Record<string, { plugin: string; test: s
|
|
|
10
10
|
},
|
|
11
11
|
cache: {
|
|
12
12
|
plugin:
|
|
13
|
-
"// `cache`: where artifacts live. A directory stands in for a remote store;\n// core puts it behind the local cache, so a remote error is a miss and a\n// warning, never a failed run.\nimport { link, mkdir, unlink } from 'node:fs/promises'\nimport path from 'node:path'\nimport { definePlugin, type RemoteCacheLayer, type VxPlugin } from '@vzn/vx'\n\nclass DirRemote implements RemoteCacheLayer {\n constructor(readonly endpoint: string) {}\n async has(hash: string): Promise<
|
|
13
|
+
"// `cache`: where artifacts live. A directory stands in for a remote store;\n// core puts it behind the local cache, so a remote error is a miss and a\n// warning, never a failed run.\nimport { link, mkdir, unlink } from 'node:fs/promises'\nimport path from 'node:path'\nimport { definePlugin, type ArtifactInfo, type RemoteCacheLayer, type VxPlugin } from '@vzn/vx'\n\nclass DirRemote implements RemoteCacheLayer {\n constructor(readonly endpoint: string) {}\n async has(hash: string): Promise<ArtifactInfo | null> {\n const file = Bun.file(path.join(this.endpoint, hash))\n return (await file.exists()) ? { size: file.size } : null\n }\n async get(hash: string) {\n const file = Bun.file(path.join(this.endpoint, hash))\n return (await file.exists()) ? { body: file, durationMs: undefined } : null\n }\n async put(hash: string, body: Blob): Promise<void> {\n await mkdir(this.endpoint, { recursive: true })\n // Written aside, then linked: on a shared directory a rename can swap\n // the file under a reader, while a link never replaces an existing one.\n // EEXIST is success: the same hash means the same bytes.\n const tmp = path.join(this.endpoint, `.${hash}.${process.pid}`)\n await Bun.write(tmp, body)\n try {\n await link(tmp, path.join(this.endpoint, hash))\n } catch (err) {\n if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err\n } finally {\n await unlink(tmp)\n }\n }\n}\n\nexport function dirCache(dir: string): VxPlugin {\n return definePlugin(import.meta, {\n cache() {\n return new DirRemote(dir)\n },\n })\n}\n",
|
|
14
14
|
test: "// A test for the example `cache` plugin beside it: a throwaway workspace\n// declares the plugin and vx's own run() drives it. `bun test` runs it.\nimport { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'\nimport os from 'node:os'\nimport path from 'node:path'\nimport { afterEach, beforeEach, expect, it } from 'bun:test'\nimport { run, type Logger } from '@vzn/vx'\n\nlet root: string\nbeforeEach(async () => {\n root = await mkdtemp(path.join(os.tmpdir(), 'vx-plugin-cache-'))\n await writeFile(path.join(root, 'package.json'), JSON.stringify({ name: 'ws', private: true }))\n await writeFile(path.join(root, 'pnpm-workspace.yaml'), 'packages:\\n - \"packages/*\"\\n')\n Bun.spawnSync({ cmd: ['git', 'init', '-q'], cwd: root })\n})\nafterEach(async () => {\n await rm(root, { recursive: true, force: true })\n})\n\n/** A package `name` whose `vx.config.mjs` is `config`. */\nasync function pkg(\n name: string,\n config: string,\n scripts?: Record<string, string>,\n): Promise<string> {\n const dir = path.join(root, 'packages', name)\n await mkdir(dir, { recursive: true })\n await writeFile(\n path.join(dir, 'package.json'),\n JSON.stringify({ name, version: '1.0.0', ...(scripts ? { scripts } : {}) }),\n )\n // Core reads no package.json: the config names the project.\n await writeFile(\n path.join(dir, 'vx.config.mjs'),\n config.replace('export default {', `export default { name: ${JSON.stringify(name)},`),\n )\n return dir\n}\n\n/**\n * A `.vx/workspace.mjs` declaring `call`, e.g. `dirCache()`, from the plugin file\n * beside this test, then `vx lock`: a run reads project configs only from the lock.\n */\nasync function workspace(call: string): Promise<void> {\n const from = JSON.stringify(path.join(import.meta.dir, 'cache.ts'))\n await Bun.write(\n path.join(root, '.vx/workspace.mjs'),\n `import { dirCache } from ${from}\\nexport default { plugins: [${call}] }\\n`,\n )\n const vx = path.join(path.dirname(new URL(import.meta.resolve('@vzn/vx')).pathname), 'bin.ts')\n const lock = Bun.spawnSync({ cmd: [process.execPath, vx, 'lock'], cwd: root })\n expect([lock.exitCode, lock.stderr.toString()]).toEqual([0, ''])\n}\n\nfunction log(order: string[] = []): Logger {\n return {\n status() {},\n taskStart(node) {\n order.push(`start ${node.id}`)\n },\n taskStdout() {},\n taskStderr() {},\n taskComplete(node) {\n order.push(`end ${node.id}`)\n },\n }\n}\n\nit('a run on an empty local cache restores from the directory', async () => {\n const dir = await pkg(\n 'a',\n `export default { tasks: { build: {\n exec: { command: 'mkdir -p dist && echo built > dist/out.txt' },\n inputs: { files: ['package.json'] }, cache: { outputs: { files: ['dist/**'] } },\n } } }`,\n )\n await workspace(`dirCache(${JSON.stringify(path.join(root, 'remote'))})`)\n const first = await run({ cwd: root, tasks: ['build'], log: log(), handleSignals: false })\n expect(first.outcomes.map((o) => o.status)).toEqual(['success'])\n await rm(path.join(root, '.vx', 'cache'), { recursive: true, force: true })\n await rm(path.join(dir, 'dist'), { recursive: true, force: true })\n const second = await run({ cwd: root, tasks: ['build'], log: log(), handleSignals: false })\n expect(second.outcomes.map((o) => o.status)).toEqual(['cache-hit-remote'])\n expect(await Bun.file(path.join(dir, 'dist', 'out.txt')).text()).toBe('built\\n')\n}, 20_000)\n",
|
|
15
15
|
},
|
|
16
16
|
commands: {
|
package/src/cli/run.ts
CHANGED
|
@@ -109,6 +109,9 @@ const SETTING_OPTIONS = [
|
|
|
109
109
|
['run.retries', 'retries'],
|
|
110
110
|
['run.outputLogs', 'outputLogs'],
|
|
111
111
|
['cache.download', 'download'],
|
|
112
|
+
['cache.concurrency.download', 'downloadConcurrency'],
|
|
113
|
+
['cache.concurrency.upload', 'uploadConcurrency'],
|
|
114
|
+
['cache.concurrency.restore', 'restoreConcurrency'],
|
|
112
115
|
] as const satisfies readonly (readonly [string, keyof RunOptions])[]
|
|
113
116
|
|
|
114
117
|
/** `verb` is the one being parsed for: `vx watch` reads `vx run`'s flags, and
|
|
@@ -515,7 +518,6 @@ export async function resolveRunOptions(
|
|
|
515
518
|
}
|
|
516
519
|
if (parsed.profile !== undefined) opts.profile = parsed.profile
|
|
517
520
|
// `--affected`'s base is the branch's for the cache scope too.
|
|
518
|
-
if (affectedFilter !== undefined) opts.scopeBase = affectedFilter.slice(4, -1)
|
|
519
521
|
|
|
520
522
|
return opts
|
|
521
523
|
}
|
package/src/cli/upgrade.ts
CHANGED
|
@@ -21,6 +21,7 @@ import { access, chmod, chown, link, mkdtemp, rename, rm, stat } from 'node:fs/p
|
|
|
21
21
|
import path from 'node:path'
|
|
22
22
|
import { flagHint, seeHelp } from './help.js'
|
|
23
23
|
import { UserError, sayNotice, ui } from '../util/index.js'
|
|
24
|
+
import { findWorkspaceRoot } from '../workspace/index.js'
|
|
24
25
|
import { VERSION } from '../version.js'
|
|
25
26
|
|
|
26
27
|
const REGISTRY = 'https://registry.npmjs.org'
|
|
@@ -474,7 +475,7 @@ function installedVersion(dir: string, pkg: string): string | undefined {
|
|
|
474
475
|
}
|
|
475
476
|
|
|
476
477
|
/** The project's package manager: its lockfile, then `packageManager`, then the tree's shape. */
|
|
477
|
-
function projectManager(dir: string, execPath: string): ManagedInstall['manager'] {
|
|
478
|
+
export function projectManager(dir: string, execPath: string): ManagedInstall['manager'] {
|
|
478
479
|
for (let d = dir; ; d = path.dirname(d)) {
|
|
479
480
|
if (existsSync(path.join(d, 'pnpm-lock.yaml'))) return 'pnpm'
|
|
480
481
|
if (existsSync(path.join(d, 'bun.lock')) || existsSync(path.join(d, 'bun.lockb'))) return 'bun'
|
|
@@ -596,11 +597,11 @@ export async function upgradeManaged(
|
|
|
596
597
|
version: string,
|
|
597
598
|
current = VERSION,
|
|
598
599
|
step = '',
|
|
599
|
-
): Promise<
|
|
600
|
+
): Promise<boolean> {
|
|
600
601
|
const plugins = install.plugins ?? []
|
|
601
602
|
if (version === current && plugins.every((p) => installedVersion(install.cwd, p) === version)) {
|
|
602
603
|
ui.print('out', ui.status('info', 'vx upgrade', `${step}already at ${current}`))
|
|
603
|
-
return
|
|
604
|
+
return false
|
|
604
605
|
}
|
|
605
606
|
const argv = upgradeArgv(install, version)
|
|
606
607
|
// PATH as it is now: `Bun.which` alone reads the one the process started with.
|
|
@@ -636,6 +637,7 @@ export async function upgradeManaged(
|
|
|
636
637
|
}`,
|
|
637
638
|
)
|
|
638
639
|
}
|
|
640
|
+
return true
|
|
639
641
|
}
|
|
640
642
|
|
|
641
643
|
/**
|
|
@@ -648,7 +650,7 @@ async function upgradeBinary(
|
|
|
648
650
|
current = VERSION,
|
|
649
651
|
step = '',
|
|
650
652
|
key: SigningKey = NPM_SIGNING_KEY,
|
|
651
|
-
): Promise<
|
|
653
|
+
): Promise<boolean> {
|
|
652
654
|
const pkg = platformPackage()
|
|
653
655
|
let installed: string | undefined
|
|
654
656
|
const replaced = await replaceBinary(
|
|
@@ -679,6 +681,7 @@ async function upgradeBinary(
|
|
|
679
681
|
? ui.status('ok', 'vx upgrade', `${step}installed ${installed}`)
|
|
680
682
|
: ui.status('info', 'vx upgrade', `${step}already at ${current}`),
|
|
681
683
|
)
|
|
684
|
+
return replaced
|
|
682
685
|
}
|
|
683
686
|
|
|
684
687
|
/** The version `bin --version` answers, without the hand-off note a handed-off run adds. */
|
|
@@ -738,9 +741,9 @@ export async function upgradeOnlyGlobal(
|
|
|
738
741
|
}
|
|
739
742
|
|
|
740
743
|
/** `run`, its refusal naming `what` after the verb: a run that moves two installs says which failed. */
|
|
741
|
-
async function named(what: string, run: () => Promise<
|
|
744
|
+
async function named<T>(what: string, run: () => Promise<T>): Promise<T> {
|
|
742
745
|
try {
|
|
743
|
-
await run()
|
|
746
|
+
return await run()
|
|
744
747
|
} catch (err) {
|
|
745
748
|
if (!(err instanceof UserError)) throw err
|
|
746
749
|
throw new UserError(err.message.replace(/^vx upgrade: /, `vx upgrade: ${what}: `), err.code)
|
|
@@ -787,19 +790,46 @@ export async function upgradeCmd(argv: readonly string[]): Promise<number> {
|
|
|
787
790
|
const from = handedOffFrom
|
|
788
791
|
if (from !== undefined) {
|
|
789
792
|
const version = await targetVersion(tag)
|
|
790
|
-
await named('project vx', () =>
|
|
793
|
+
const moved = await named('project vx', () =>
|
|
791
794
|
install === null
|
|
792
795
|
? upgradeBinary(dest, version, VERSION, 'project ')
|
|
793
796
|
: upgradeManaged(install, version, VERSION, 'project '),
|
|
794
797
|
)
|
|
795
798
|
await upgradeGlobal(from, version)
|
|
799
|
+
if (moved) await relock(newVx(install, dest))
|
|
796
800
|
return 0
|
|
797
801
|
}
|
|
798
|
-
|
|
799
|
-
|
|
802
|
+
const moved =
|
|
803
|
+
install !== null
|
|
804
|
+
? await upgradeManaged(install, await targetVersion(tag))
|
|
805
|
+
: await upgradeBinary(dest, versionSpec(tag))
|
|
806
|
+
if (moved) await relock(newVx(install, dest))
|
|
800
807
|
return 0
|
|
801
808
|
}
|
|
802
809
|
|
|
810
|
+
/**
|
|
811
|
+
* The vx an upgrade just installed: the replaced binary, or the project's
|
|
812
|
+
* `node_modules/.bin/vx` (a manager's store path names the old version).
|
|
813
|
+
*/
|
|
814
|
+
const newVx = (install: ManagedInstall | null, dest: string): string =>
|
|
815
|
+
install === null ? dest : path.join(install.cwd, 'node_modules', '.bin', 'vx')
|
|
816
|
+
|
|
817
|
+
/**
|
|
818
|
+
* Lock the workspace with the vx just installed at `bin` (M, 2026-10-18):
|
|
819
|
+
* a release may record the lock differently. `-g` never re-locks. A lock
|
|
820
|
+
* that fails leaves the upgrade done and says what to run.
|
|
821
|
+
*/
|
|
822
|
+
export async function relock(bin: string, cwd = process.cwd()): Promise<void> {
|
|
823
|
+
const root = await findWorkspaceRoot(cwd).catch(() => null)
|
|
824
|
+
if (root === null) return
|
|
825
|
+
const proc = Bun.spawnSync({
|
|
826
|
+
cmd: [bin, 'lock'],
|
|
827
|
+
cwd: root,
|
|
828
|
+
stdio: ['inherit', 'inherit', 'inherit'],
|
|
829
|
+
})
|
|
830
|
+
if (proc.exitCode !== 0) sayNotice('warn', 'vx upgrade', 'vx lock failed: run vx lock')
|
|
831
|
+
}
|
|
832
|
+
|
|
803
833
|
/**
|
|
804
834
|
* `bin --version` when it answers as vx within `timeoutMs`, else null. The
|
|
805
835
|
* answer decides whether a replaced binary is rolled back, so it is
|
package/src/config.ts
CHANGED
|
@@ -79,15 +79,6 @@ export interface CacheSettings {
|
|
|
79
79
|
* `.vx/cache` and its entries in the user's shared store.
|
|
80
80
|
*/
|
|
81
81
|
dir?: string
|
|
82
|
-
/**
|
|
83
|
-
* Evict from the local cache at the end of every run, whatever its
|
|
84
|
-
* policy (a remote hit is ingested under any), by the same policy
|
|
85
|
-
* `vx cache clean` takes as flags: entries not used for `olderThan`
|
|
86
|
-
* (`'30d'`, `'12h'`), then least-recently-used entries until the cache
|
|
87
|
-
* is under `maxSize` (`'10GB'`, `'500MB'`). Either or both. Omitted →
|
|
88
|
-
* nothing is evicted unless you run `vx cache clean`.
|
|
89
|
-
*/
|
|
90
|
-
retention?: { olderThan?: Duration; maxSize?: string }
|
|
91
82
|
/**
|
|
92
83
|
* Which layers a run reads and writes (`'local:rw,remote:r'`; a layer it
|
|
93
84
|
* leaves out stays on); `--no-cache` and `--force` still apply on top.
|
|
@@ -107,6 +98,19 @@ export interface CacheSettings {
|
|
|
107
98
|
* both (`'always'`). Only a run whose policy writes a remote keeps any.
|
|
108
99
|
*/
|
|
109
100
|
upload?: { failures?: 'ci' | 'local' | 'always' }
|
|
101
|
+
/**
|
|
102
|
+
* How the local cache keeps an entry. `embeddings` (default `'32KiB'`):
|
|
103
|
+
* an entry up to this size inside the index; `false`, none. `decompress` (default true): a larger one as its
|
|
104
|
+
* plain files, which a hit copies or clones out with no decode; off, as
|
|
105
|
+
* one `.tar.zst`.
|
|
106
|
+
*/
|
|
107
|
+
store?: { embeddings?: false | string; decompress?: boolean }
|
|
108
|
+
/**
|
|
109
|
+
* Cache transfers at once. `download` and `upload` (per remote) default
|
|
110
|
+
* to min(96, max(cores × 3, 64)); `restore`, local hits writing their
|
|
111
|
+
* outputs, to twice `run.concurrency`.
|
|
112
|
+
*/
|
|
113
|
+
concurrency?: { download?: number; upload?: number; restore?: number }
|
|
110
114
|
}
|
|
111
115
|
|
|
112
116
|
/** `WorkspaceConfig.lock`. */
|
|
@@ -786,7 +790,6 @@ export function defineWorkspace<T extends WorkspaceConfig>(
|
|
|
786
790
|
rules?: Known<At<At<T, 'run'>, 'rules'>, WorkspaceRules>
|
|
787
791
|
}
|
|
788
792
|
cache?: Known<At<T, 'cache'>, CacheSettings> & {
|
|
789
|
-
retention?: Known<At<At<T, 'cache'>, 'retention'>, NonNullable<CacheSettings['retention']>>
|
|
790
793
|
inputStability?: Known<At<At<T, 'cache'>, 'inputStability'>, InputStability>
|
|
791
794
|
}
|
|
792
795
|
lock?: Known<At<T, 'lock'>, LockSettings>
|
|
@@ -16,8 +16,8 @@ import path from 'node:path'
|
|
|
16
16
|
* move that spelling into the scan, which finds nothing before the task has
|
|
17
17
|
* written, and turn a working grant into `Read-only file system`. The two
|
|
18
18
|
* predicates answer different questions: whether a declaration must be
|
|
19
|
-
* MATCHED against other declarations (495; `GLOB_WILDCARDS
|
|
20
|
-
*
|
|
19
|
+
* MATCHED against other declarations (495; `GLOB_WILDCARDS`), and whether a
|
|
20
|
+
* grant can be mounted (here).
|
|
21
21
|
* Item 577 gave each question one spelling.
|
|
22
22
|
*/
|
|
23
23
|
export const MOUNT_WILDCARDS = /[*?[\]]/
|
package/src/graph/scheduler.ts
CHANGED
|
@@ -239,6 +239,8 @@ export type ContinueMode = 'never' | 'deps-ok' | 'always'
|
|
|
239
239
|
export interface ScheduleOptions {
|
|
240
240
|
nodes: Map<string, TaskNode>
|
|
241
241
|
concurrency: number
|
|
242
|
+
/** Restore-tier tasks at once; twice `concurrency` unset. */
|
|
243
|
+
restoreConcurrency?: number
|
|
242
244
|
/**
|
|
243
245
|
* An aborted run dispatches nothing further: every task not yet
|
|
244
246
|
* started completes `aborted` (the ones in flight are being killed by
|
|
@@ -604,7 +606,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
|
|
|
604
606
|
// the two lanes share its one slot, where a lane each let an 80 MB
|
|
605
607
|
// restore run inside another task's execution (item 1103).
|
|
606
608
|
let activeRestore = 0
|
|
607
|
-
const restoreConcurrency = 2 * concurrency
|
|
609
|
+
const restoreConcurrency = options.restoreConcurrency ?? 2 * concurrency
|
|
608
610
|
const serial = concurrency === 1
|
|
609
611
|
const execRoom = (): boolean => (serial ? active + activeRestore < 1 : active < concurrency)
|
|
610
612
|
const restoreRoom = (): boolean =>
|
package/src/graph/task-graph.ts
CHANGED
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
checkIgnoreSync,
|
|
5
5
|
ignoreProbes,
|
|
6
6
|
isLiteralPattern,
|
|
7
|
+
literalPath,
|
|
7
8
|
listedSync,
|
|
8
9
|
normalizeGlob,
|
|
9
10
|
relPosix,
|
|
@@ -915,15 +916,17 @@ export function outputsOverlap(rawA: string, rawB: string): boolean {
|
|
|
915
916
|
// A built path proves overlap only between globs as written: a literal's
|
|
916
917
|
// `/**` twin reads it as a directory, and `dist/types.d.ts` met
|
|
917
918
|
// `dist/**/*.js` at `dist/types.d.ts/x.js`, a file no one can write.
|
|
918
|
-
const bothGlobs =
|
|
919
|
+
const bothGlobs = literalPath(rawA) === null && literalPath(rawB) === null
|
|
919
920
|
for (const a of asTrees([rawA])) {
|
|
920
921
|
for (const b of asTrees([rawB])) {
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
if (
|
|
925
|
-
} else if (
|
|
926
|
-
if (taskGlob(
|
|
922
|
+
const la = literalPath(a)
|
|
923
|
+
const lb = literalPath(b)
|
|
924
|
+
if (la !== null && lb !== null) {
|
|
925
|
+
if (la === lb) return true
|
|
926
|
+
} else if (la !== null) {
|
|
927
|
+
if (taskGlob(b).match(la)) return true
|
|
928
|
+
} else if (lb !== null) {
|
|
929
|
+
if (taskGlob(a).match(lb)) return true
|
|
927
930
|
} else if (a === b || covers(a, b) || covers(b, a) || (bothGlobs && witnessed(a, b)))
|
|
928
931
|
return true
|
|
929
932
|
}
|
|
@@ -1202,13 +1205,13 @@ function collideAcross(
|
|
|
1202
1205
|
* found no match outside the head this cuts, and thousands once `\` and
|
|
1203
1206
|
* `!` were left out (item 746).
|
|
1204
1207
|
*/
|
|
1205
|
-
const GLOB_HEAD_END = /[*?{}\\!]/
|
|
1208
|
+
const GLOB_HEAD_END = /[*?{}[\]\\!]/
|
|
1206
1209
|
|
|
1207
1210
|
/** `firstSegment`'s answer for a glob that can start anywhere. */
|
|
1208
1211
|
const ANYWHERE = ''
|
|
1209
1212
|
|
|
1210
1213
|
/** Where a glob's first segment ends: a separator, or anything `GLOB_HEAD_END` stops at. */
|
|
1211
|
-
const FIRST_SEGMENT_END = /[*?{}\\!/]/
|
|
1214
|
+
const FIRST_SEGMENT_END = /[*?{}[\]\\!/]/
|
|
1212
1215
|
|
|
1213
1216
|
const firstSegments = new Map<string, string>()
|
|
1214
1217
|
|
|
@@ -1315,8 +1318,9 @@ function overlapCandidates<T>(
|
|
|
1315
1318
|
const s = sideOf?.(tasks[i]!) === true ? 1 : 0
|
|
1316
1319
|
side.push(s)
|
|
1317
1320
|
for (const tree of asTrees(globsOf(tasks[i]!) ?? [])) {
|
|
1318
|
-
|
|
1319
|
-
|
|
1321
|
+
const lit = literalPath(tree)
|
|
1322
|
+
if (lit !== null) {
|
|
1323
|
+
literals.push([i, lit])
|
|
1320
1324
|
continue
|
|
1321
1325
|
}
|
|
1322
1326
|
file(globs[s]!, tree, i)
|
package/src/index.ts
CHANGED
|
@@ -20,7 +20,7 @@ export { PLUGIN_HOOKS, type PluginHook } from './config.js'
|
|
|
20
20
|
// the second, so the cgroup walk has one implementation.
|
|
21
21
|
export { machineMemoryBytes, machineParallelism } from './util/index.js'
|
|
22
22
|
// How core reads a task glob — its spelling, and which characters are
|
|
23
|
-
// wildcards (`[` `]`
|
|
23
|
+
// wildcards (`[` `]` and an escape are) — for a plugin that turns declared
|
|
24
24
|
// outputs into something else (`@vzn/vx-reapi`'s literal `output_paths`).
|
|
25
25
|
export { isLiteralPattern, normalizeGlob, restorableOutput } from './util/index.js'
|
|
26
26
|
|
|
@@ -83,8 +83,8 @@ export { Cache } from './cache/index.js'
|
|
|
83
83
|
export type {
|
|
84
84
|
CacheLayer,
|
|
85
85
|
PluginCache,
|
|
86
|
+
ArtifactInfo,
|
|
86
87
|
RemoteCacheLayer,
|
|
87
|
-
RemoteKey,
|
|
88
88
|
RemoteOpenContext,
|
|
89
89
|
RunRecord,
|
|
90
90
|
InvocationRecord,
|
|
@@ -229,24 +229,6 @@ export function affectedRoots(
|
|
|
229
229
|
return kept
|
|
230
230
|
}
|
|
231
231
|
|
|
232
|
-
/**
|
|
233
|
-
* The tasks of `nodes` whose key the change can move (a branch's cache
|
|
234
|
-
* scope, owner 2026-10-17): a changed path one of its inputs declares, a
|
|
235
|
-
* whole project, or a task its closure holds. Unlike `--affected`, a path
|
|
236
|
-
* no task declares (a README) reaches none: its key is the default
|
|
237
|
-
* branch's, so it reads the default key.
|
|
238
|
-
*/
|
|
239
|
-
export function keyedByChange(
|
|
240
|
-
nodes: ReadonlyMap<string, TaskNode>,
|
|
241
|
-
changes: AffectedChanges,
|
|
242
|
-
projects: ReadonlyMap<string, ProjectEntry>,
|
|
243
|
-
packageGraph: PackageGraph,
|
|
244
|
-
): Set<string> {
|
|
245
|
-
return new Set(
|
|
246
|
-
affectedRoots(nodes, [...nodes.keys()], changes, projects, packageGraph, undefined, true),
|
|
247
|
-
)
|
|
248
|
-
}
|
|
249
|
-
|
|
250
232
|
/**
|
|
251
233
|
* The requested tasks `--affected` keeps, in their order: one the user
|
|
252
234
|
* named (`pkg#task`), one in a project another include selected outright
|
|
@@ -18,7 +18,6 @@ import {
|
|
|
18
18
|
isUserError,
|
|
19
19
|
machineMemoryBytes,
|
|
20
20
|
machineParallelism,
|
|
21
|
-
parseSize,
|
|
22
21
|
treeBytes,
|
|
23
22
|
} from '../util/index.js'
|
|
24
23
|
import { vxPath } from '../handoff.js'
|
|
@@ -99,8 +98,6 @@ export interface InfoFacts {
|
|
|
99
98
|
string,
|
|
100
99
|
{ value: string | number | boolean | null; source: 'env' | 'workspace' | 'default' }
|
|
101
100
|
>
|
|
102
|
-
/** `cache.retention`, with `maxSize` in bytes; null when none is set. */
|
|
103
|
-
cacheRetention: { olderThan?: string; maxSize?: string; maxBytes?: number } | null
|
|
104
101
|
orphans: { artifacts: number; bytes: number }
|
|
105
102
|
runs24h: number
|
|
106
103
|
hits24h: number
|
|
@@ -261,7 +258,6 @@ async function collectWorkspaceInfo(
|
|
|
261
258
|
cacheEntries: stats.entryCount,
|
|
262
259
|
cacheBytes: stats.totalBytes,
|
|
263
260
|
settings: settingsFact(workspaceConfig),
|
|
264
|
-
cacheRetention: retentionFact(workspaceConfig),
|
|
265
261
|
// A row-less artifact is indexed again only when its task asks for its
|
|
266
262
|
// key; the rest only `vx cache clean` reclaims (after an upgrade's
|
|
267
263
|
// schema reset, most often).
|
|
@@ -395,16 +391,3 @@ function settingsFact(config: WorkspaceConfig | null): InfoFacts['settings'] {
|
|
|
395
391
|
}
|
|
396
392
|
return out
|
|
397
393
|
}
|
|
398
|
-
|
|
399
|
-
function retentionFact(config: WorkspaceConfig | null): InfoFacts['cacheRetention'] {
|
|
400
|
-
const raw = (path: string): string | undefined =>
|
|
401
|
-
process.env[envName(path)] || (atPath(config, path) as string | undefined)
|
|
402
|
-
const olderThan = raw('cache.retention.olderThan')
|
|
403
|
-
const maxSize = raw('cache.retention.maxSize')
|
|
404
|
-
if (olderThan === undefined && maxSize === undefined) return null
|
|
405
|
-
return {
|
|
406
|
-
...(olderThan !== undefined ? { olderThan } : {}),
|
|
407
|
-
// Validated before the run (env) or at load (workspace), so it parses.
|
|
408
|
-
...(maxSize !== undefined ? { maxSize, maxBytes: parseSize(maxSize)! } : {}),
|
|
409
|
-
}
|
|
410
|
-
}
|
|
@@ -335,7 +335,7 @@ async function restoreProbed(
|
|
|
335
335
|
|
|
336
336
|
/**
|
|
337
337
|
* A hit restored, or null when its artifact was removed after the probe (a
|
|
338
|
-
*
|
|
338
|
+
* `vx cache clean`, here or for a workspace sharing the cache directory): a
|
|
339
339
|
* miss. The restore has wiped the declared outputs by then; the run path
|
|
340
340
|
* cleans them again and the task writes them.
|
|
341
341
|
*/
|
|
@@ -92,14 +92,6 @@ export interface RunOptions {
|
|
|
92
92
|
remoteRequested?: boolean
|
|
93
93
|
/** This run's cache scope, above `VX_CACHE_SCOPE`, a plugin's and the branch's (an embedder's). */
|
|
94
94
|
cacheScope?: string
|
|
95
|
-
/**
|
|
96
|
-
* What the branch changed, for a scoped run: the tasks whose key it can
|
|
97
|
-
* move read the scope's key, every other task the default key (one
|
|
98
|
-
* lookup each). Absent, the run diffs the branch from `scopeBase`.
|
|
99
|
-
*/
|
|
100
|
-
scopeChanges?: AffectedChanges
|
|
101
|
-
/** The commit a scoped run diffs the branch from (`--affected`'s base); absent, a plugin's, else the merge base with the default branch. */
|
|
102
|
-
scopeBase?: string
|
|
103
95
|
/**
|
|
104
96
|
* The default policy's remote write is off (`remote:r`): the CLI passes it
|
|
105
97
|
* off CI (`CI` unset, `0` or `false`) unless `--cache-policy` or
|
|
@@ -130,6 +122,12 @@ export interface RunOptions {
|
|
|
130
122
|
* Locally-placed tasks always write in place and ignore this.
|
|
131
123
|
*/
|
|
132
124
|
download?: 'all' | 'toplevel' | 'none'
|
|
125
|
+
/** Remote downloads at once (`cache.concurrency.download`). */
|
|
126
|
+
downloadConcurrency?: number
|
|
127
|
+
/** Uploads at once, per remote (`cache.concurrency.upload`). */
|
|
128
|
+
uploadConcurrency?: number
|
|
129
|
+
/** Local hits restoring at once (`cache.concurrency.restore`); twice `concurrency` unset. */
|
|
130
|
+
restoreConcurrency?: number
|
|
133
131
|
/**
|
|
134
132
|
* Run intent, derived by the CLI from selection flags: 'broad' iff
|
|
135
133
|
* `--all` / `--filter` / `--affected` was passed, else 'focused'.
|
package/src/orchestrator/plan.ts
CHANGED
|
@@ -54,6 +54,8 @@ export interface PlannedTask {
|
|
|
54
54
|
* eager task is the default and saying so on every line says nothing.
|
|
55
55
|
*/
|
|
56
56
|
download?: 'deferred'
|
|
57
|
+
/** A remote hit's stored size, when the remote said it: what restoring it downloads. */
|
|
58
|
+
remoteBytes?: number
|
|
57
59
|
/** Why `--affected` kept this requested task; absent without `--affected` and on a dependency it pulled in. */
|
|
58
60
|
affected?: AffectedReason
|
|
59
61
|
}
|
|
@@ -139,6 +141,7 @@ export interface PlanArgs {
|
|
|
139
141
|
*/
|
|
140
142
|
export async function plan(args: PlanArgs): Promise<RunPlan> {
|
|
141
143
|
const cacheStatusById = new Map<string, CacheStatus>()
|
|
144
|
+
const remoteBytesById = new Map<string, number>()
|
|
142
145
|
// Keyed first, probed after: a remote that can batch (`hasMany`) is
|
|
143
146
|
// asked once for every key, one that cannot is asked per key.
|
|
144
147
|
const probed = new Map<string, string>()
|
|
@@ -191,13 +194,15 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
|
|
|
191
194
|
},
|
|
192
195
|
})
|
|
193
196
|
if (probed.size > 0) {
|
|
194
|
-
await args.cache.remoteHasMany?.([...new Set(probed.values())])
|
|
197
|
+
const infos = await args.cache.remoteHasMany?.([...new Set(probed.values())])
|
|
195
198
|
for (const [id, hash] of probed) {
|
|
196
199
|
const where = await args.cache.has(hash)
|
|
197
200
|
cacheStatusById.set(
|
|
198
201
|
id,
|
|
199
202
|
where === null ? 'miss' : where === 'remote' ? 'hit-remote' : 'hit-local',
|
|
200
203
|
)
|
|
204
|
+
const size = where === 'remote' ? infos?.get(hash)?.size : undefined
|
|
205
|
+
if (size !== undefined) remoteBytesById.set(id, size)
|
|
201
206
|
}
|
|
202
207
|
}
|
|
203
208
|
|
|
@@ -225,6 +230,7 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
|
|
|
225
230
|
deps: node.deps,
|
|
226
231
|
...(executor !== undefined ? { executor } : {}),
|
|
227
232
|
...(args.downloadOf?.(id) === 'deferred' ? { download: 'deferred' as const } : {}),
|
|
233
|
+
...(remoteBytesById.has(id) ? { remoteBytes: remoteBytesById.get(id)! } : {}),
|
|
228
234
|
})
|
|
229
235
|
}
|
|
230
236
|
|
|
@@ -9,7 +9,6 @@
|
|
|
9
9
|
|
|
10
10
|
import path from 'node:path'
|
|
11
11
|
import {
|
|
12
|
-
type BranchScope,
|
|
13
12
|
type Cache,
|
|
14
13
|
type CacheLayer,
|
|
15
14
|
type CachePolicy,
|
|
@@ -556,8 +555,8 @@ export interface CacheHost extends CacheContext {
|
|
|
556
555
|
readonly policy: CachePolicy
|
|
557
556
|
/** Filled with each remote and what its policy lets it do, for the plugins' `ctx.cache`. */
|
|
558
557
|
readonly remotes?: ResolvedRemote[]
|
|
559
|
-
/**
|
|
560
|
-
readonly
|
|
558
|
+
/** Uploads at once, per remote (`cache.concurrency.upload`). */
|
|
559
|
+
readonly uploadConcurrency?: number
|
|
561
560
|
}
|
|
562
561
|
|
|
563
562
|
/**
|
|
@@ -572,7 +571,7 @@ export async function resolveCache(
|
|
|
572
571
|
plugins: readonly VxPlugin[],
|
|
573
572
|
host: CacheHost,
|
|
574
573
|
): Promise<CacheLayer> {
|
|
575
|
-
const { localCache, policy, remotes,
|
|
574
|
+
const { localCache, policy, remotes, uploadConcurrency, ...ctx } = host
|
|
576
575
|
const resolved: ResolvedRemote[] = []
|
|
577
576
|
for (const plugin of plugins) {
|
|
578
577
|
if (plugin.cache === undefined) continue
|
|
@@ -589,7 +588,9 @@ export async function resolveCache(
|
|
|
589
588
|
resolved.push(entry)
|
|
590
589
|
remotes?.push(entry)
|
|
591
590
|
}
|
|
592
|
-
return resolved.length === 0
|
|
591
|
+
return resolved.length === 0
|
|
592
|
+
? localCache
|
|
593
|
+
: new LayeredCache(localCache, resolved, uploadConcurrency)
|
|
593
594
|
}
|
|
594
595
|
|
|
595
596
|
/** The plugin each resolved executor came from, so a hook of it that throws is named. */
|
|
@@ -346,12 +346,14 @@ export interface WorkspaceHookContext {
|
|
|
346
346
|
* What a CI knows of the run's branch (owner, 2026-10-17): `scope`, the
|
|
347
347
|
* cache scope (`'vznjs/vx:feat-x'`; `VX_CACHE_SCOPE` beats it, and it
|
|
348
348
|
* beats the one git names), and `base`, the commit the branch is compared
|
|
349
|
-
* with (a pull request's base), for a bare `--affected`
|
|
350
|
-
*
|
|
349
|
+
* with (a pull request's base), for a bare `--affected`. `trusted`: the
|
|
350
|
+
* run has no scope, whatever git names (a merge queue's run, owner
|
|
351
|
+
* 2026-10-18). The scope keys plugin entries only. A later plugin's field wins.
|
|
351
352
|
*/
|
|
352
353
|
export interface BranchFacts {
|
|
353
354
|
scope?: string
|
|
354
355
|
base?: string
|
|
356
|
+
trusted?: boolean
|
|
355
357
|
}
|
|
356
358
|
|
|
357
359
|
export interface ConfigContext extends WorkspaceHookContext {
|
|
@@ -503,11 +505,16 @@ export interface ScheduleHookContext extends BaseContext {
|
|
|
503
505
|
/** A `cache` hook's context: the workspace and the run's scope, never the local store or the policy, which core owns. */
|
|
504
506
|
export interface CacheContext extends BaseContext {
|
|
505
507
|
/**
|
|
506
|
-
* The run's cache scope (`'vznjs/vx:feat-x'`), `undefined` when none
|
|
507
|
-
*
|
|
508
|
-
* claims (an OIDC branch) for every call it makes.
|
|
508
|
+
* The run's cache scope (`'vznjs/vx:feat-x'`), `undefined` when none: what
|
|
509
|
+
* plugins' `ctx.cache` entries and stored runs are keyed apart by.
|
|
509
510
|
*/
|
|
510
511
|
readonly scope: string | undefined
|
|
512
|
+
/**
|
|
513
|
+
* The refs the run reads, in order (`['refs/heads/feat-x', 'refs/heads/main']`):
|
|
514
|
+
* its own branch, the branch it merges into, the default branch. A remote
|
|
515
|
+
* keys or scopes its entries by them; core keys task entries by none.
|
|
516
|
+
*/
|
|
517
|
+
readonly refs: readonly string[]
|
|
511
518
|
}
|
|
512
519
|
|
|
513
520
|
export interface OutputContext extends BaseContext {
|