@vzn/vx-migrate 0.0.643 → 0.0.645
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +5 -3
- package/package.json +2 -2
- package/src/adoption-plugin.ts +1 -1
- package/src/glob-grammar.ts +15 -50
- package/src/migrate-nx.ts +2 -10
- package/src/nx/index.ts +0 -24
- package/src/nx/nx-map.ts +0 -17
- package/src/nx-cache/index.ts +6 -5
- package/src/turbo/index.ts +5 -54
- package/src/turbo/turbo-map.ts +7 -10
- package/src/turbo-cache/index.ts +49 -19
- package/src/turbo-cache/scope.ts +89 -0
package/LICENSE
CHANGED
package/README.md
CHANGED
|
@@ -75,7 +75,7 @@ Rules:
|
|
|
75
75
|
- `interruptible` maps to nothing: `vx watch` keeps a persistent task up across cycles and restarts it only when its config changes or it dies.
|
|
76
76
|
- A task's `tags` (Turbo main, after 2.11.5) map to nothing: Turbo keeps them out of the hash and the run.
|
|
77
77
|
- `envMode: "loose"` (top-level or in `global`, or `TURBO_ENV_MODE=loose`, which wins) is a note: Turbo hands every task the whole environment, vx only the declared names.
|
|
78
|
-
- The workspace keys vx has a home for, top level or in `global`, fill what `.vx/workspace.ts` leaves unset: `run.concurrency` (`"10"`, `"50%"` of the cores) → `run.concurrency
|
|
78
|
+
- The workspace keys vx has a home for, top level or in `global`, fill what `.vx/workspace.ts` leaves unset: `run.concurrency` (`"10"`, `"50%"` of the cores) → `run.concurrency`. `TURBO_CONCURRENCY` wins over turbo.json, as under Turbo; a set one decides, its `0` included. `cacheMaxSize` / `cacheMaxAge` map to nothing: no vx run evicts, `vx cache clean --max-size / --older-than` does.
|
|
79
79
|
- `TURBO_SCM_BASE`, the base `turbo run --affected` compares with, is the run's base commit (`setBranch({ base })`): a bare `vx run build --affected` compares with the same ref. With it unset on GitHub Actions (`GITHUB_ACTIONS`), Turbo's own base applies: a pull request's `GITHUB_BASE_REF`, else the push event's `before` from `GITHUB_EVENT_PATH`.
|
|
80
80
|
- Turbo hashes each package's `microfrontends.json` (or `.jsonc`) into every task, or the one file `VC_MICROFRONTENDS_CONFIG_FILE_NAME` names; so does `turbo()`.
|
|
81
81
|
- Turbo 2.11's `global` block (`futureFlags.globalConfiguration`) is read as the `globalDependencies`, `globalEnv` and `globalPassThroughEnv` it replaces.
|
|
@@ -173,7 +173,7 @@ Two cached tasks on one workspace path cannot both keep their cache, and the fir
|
|
|
173
173
|
|
|
174
174
|
### nx.json `maxCacheSize`
|
|
175
175
|
|
|
176
|
-
`maxCacheSize`
|
|
176
|
+
`maxCacheSize` maps to nothing: no vx run evicts, `vx cache clean --max-size` does.
|
|
177
177
|
|
|
178
178
|
### Lerna
|
|
179
179
|
|
|
@@ -259,7 +259,7 @@ Reads the root pipeline (`tasks` in Turbo 2, `pipeline` in Turbo 1), per-package
|
|
|
259
259
|
|
|
260
260
|
### Nx
|
|
261
261
|
|
|
262
|
-
Reads the **resolved** project graph only (`.nx/workspace-data/project-graph.json` when exported, else the one the workspace's own `nx graph` exports into a temp file, as `nx()` does; with no `node_modules/.bin/nx`, as in a fresh clone, it stops and names the repo manager's install), through the same mapper `nx()` runs live, executors aside (below). Targets Nx plugins infer at runtime are frozen as the snapshot saw them. `nx:run-commands` is the one shell line `nx()` runs (see [`nx:run-commands`](#nxrun-commands) above: where, parallel or in order, forwarded arguments, `env`, `readyWhen`) — storybook's `compile` is `cd ../../.. && node ./scripts/build/build-package.ts --cwd code/lib/cli`; a plain `command` is that shorthand; `nx:run-script` is the package's script body with its `pre<name>` / `post<name>` hooks folded in (or `yarn run <name>` when the body calls yarn's `run` builtin; an empty script is the placeholder with a todo), `nx:noop` is a group task; Nx 15–16's `@nrwl/workspace:run-commands` and `run-script` (and their `@nx/workspace:` names) are the `nx:` executors they re-exported; **an executor target is its `nx-exec` line**, as `nx()` runs it: the migrator translates no executor, so Nx and `@vzn/vx-migrate` stay installed until each such line is rewritten as the command it runs (a server executor Nx knows, `@nx/js:node` or a dev server, is still a persistent task). The report also says what `.vx/workspace.ts` still holds, as the Turbo migration does: the `nx()` a keep adoption declared, which keeps reading nx.json, and a lockfile with no `@vzn/vx-lockfile` plugin, where Nx keyed each project on the npm packages it depends on. A written command that still runs Nx itself (a run-commands `nx run b:build`, `npx nx test b`, a script's `nx exec -- tsc`) carries a TODO: it works only while Nx is installed. An Nx project no workspace glob lists (an integrated repo's `project.json` library) gets a `package.json` (`name`, `private`) where it has none, and a note names the directories to add to `workspaces` (or `pnpm-workspace.yaml`), so the package manager installs it and a lockfile plugin names it; the migrator never edits the root manifest. A repo with no `package.json` at all (Nx's `nx init` outside JS: Nx pinned in `.nx/installation`, projects as `project.json`) is migrated too, natively only: the root is the directory holding `nx.json`, the graph is exported by `.nx/installation`'s `nx`, each config writes the project's Nx `name`, every edge is the explicit `project#target` Nx's graph draws (a `^` edge needs a lockfile plugin, which has no lockfile to read), and `.vx/workspace.ts` declares no plugin, so nothing is installed and core runs the configs alone; `--keep` refuses, as `nx()` would have no `package.json` to install into, and an executor target's `nx-exec` line is a note to rewrite as a command. A target with `configurations` writes one task per configuration (`build`, `build:ci`). A project's Nx `tags` are written as its `tags`. Named inputs expand from `nx.json` when readable. An output path is kept as written, a `!` one too (`{projectRoot}/dist` → `dist`, `{projectRoot}/bin/tool` → `bin/tool`; one naming an unset `{options.x}` is dropped, as Nx drops it; a dotted `{options.outputPath.base}` walks the options, as Nx does; an extglob is put in vx's grammar: Next's inferred `.next/!(cache)/**/*` is `.next/*/**/*` with `!.next/cache/**/*`, `@(js|map)` is `{js,map}`, and a form with no vx spelling is dropped with a TODO): vx reads a bare path as the file or the whole tree under it, so a directory and an extensionless binary both save and restore. nx.json's `parallel
|
|
262
|
+
Reads the **resolved** project graph only (`.nx/workspace-data/project-graph.json` when exported, else the one the workspace's own `nx graph` exports into a temp file, as `nx()` does; with no `node_modules/.bin/nx`, as in a fresh clone, it stops and names the repo manager's install), through the same mapper `nx()` runs live, executors aside (below). Targets Nx plugins infer at runtime are frozen as the snapshot saw them. `nx:run-commands` is the one shell line `nx()` runs (see [`nx:run-commands`](#nxrun-commands) above: where, parallel or in order, forwarded arguments, `env`, `readyWhen`) — storybook's `compile` is `cd ../../.. && node ./scripts/build/build-package.ts --cwd code/lib/cli`; a plain `command` is that shorthand; `nx:run-script` is the package's script body with its `pre<name>` / `post<name>` hooks folded in (or `yarn run <name>` when the body calls yarn's `run` builtin; an empty script is the placeholder with a todo), `nx:noop` is a group task; Nx 15–16's `@nrwl/workspace:run-commands` and `run-script` (and their `@nx/workspace:` names) are the `nx:` executors they re-exported; **an executor target is its `nx-exec` line**, as `nx()` runs it: the migrator translates no executor, so Nx and `@vzn/vx-migrate` stay installed until each such line is rewritten as the command it runs (a server executor Nx knows, `@nx/js:node` or a dev server, is still a persistent task). The report also says what `.vx/workspace.ts` still holds, as the Turbo migration does: the `nx()` a keep adoption declared, which keeps reading nx.json, and a lockfile with no `@vzn/vx-lockfile` plugin, where Nx keyed each project on the npm packages it depends on. A written command that still runs Nx itself (a run-commands `nx run b:build`, `npx nx test b`, a script's `nx exec -- tsc`) carries a TODO: it works only while Nx is installed. An Nx project no workspace glob lists (an integrated repo's `project.json` library) gets a `package.json` (`name`, `private`) where it has none, and a note names the directories to add to `workspaces` (or `pnpm-workspace.yaml`), so the package manager installs it and a lockfile plugin names it; the migrator never edits the root manifest. A repo with no `package.json` at all (Nx's `nx init` outside JS: Nx pinned in `.nx/installation`, projects as `project.json`) is migrated too, natively only: the root is the directory holding `nx.json`, the graph is exported by `.nx/installation`'s `nx`, each config writes the project's Nx `name`, every edge is the explicit `project#target` Nx's graph draws (a `^` edge needs a lockfile plugin, which has no lockfile to read), and `.vx/workspace.ts` declares no plugin, so nothing is installed and core runs the configs alone; `--keep` refuses, as `nx()` would have no `package.json` to install into, and an executor target's `nx-exec` line is a note to rewrite as a command. A target with `configurations` writes one task per configuration (`build`, `build:ci`). A project's Nx `tags` are written as its `tags`. Named inputs expand from `nx.json` when readable. An output path is kept as written, a `!` one too (`{projectRoot}/dist` → `dist`, `{projectRoot}/bin/tool` → `bin/tool`; one naming an unset `{options.x}` is dropped, as Nx drops it; a dotted `{options.outputPath.base}` walks the options, as Nx does; an extglob is put in vx's grammar: Next's inferred `.next/!(cache)/**/*` is `.next/*/**/*` with `!.next/cache/**/*`, `@(js|map)` is `{js,map}`, and a form with no vx spelling is dropped with a TODO): vx reads a bare path as the file or the whole tree under it, so a directory and an extensionless binary both save and restore. nx.json's `parallel` and `defaultBase`, which `nx()` applies live, are each a field of the `.vx/workspace.ts` it writes (`run.concurrency`, `baseBranch`), read from nx.json alone, never the environment; beside a workspace file already there, each is a note naming the field to add. vx derives package edges from `package.json`; an Nx graph edge with no manifest path (`implicitDependencies`, a tsconfig path) becomes, for each `^target` of the dependant, an explicit `pkg#target` edge to what Nx's own walk reaches — each dependency that has the target, and through one that lacks it, its dependencies — so the order and the key are Nx's. The reverse too: where a project's manifest names a workspace package its Nx graph does not reach (an `implicitDependencies: ["!a"]` that breaks a manifest cycle), its `^target` becomes the explicit edges Nx draws, since vx's `^` would follow the manifest and bring the cycle back.
|
|
263
263
|
|
|
264
264
|
### Vite Task
|
|
265
265
|
|
|
@@ -564,6 +564,7 @@ Every option falls back to the tool's own environment variable, so a self-hosted
|
|
|
564
564
|
| `uploadTimeoutMs` | `TURBO_REMOTE_CACHE_UPLOAD_TIMEOUT`, then `remoteCache.uploadTimeout` (seconds) | PUT deadline (default 60 s; 0 none) |
|
|
565
565
|
| `retries` | — | resends of a request answered 5xx (not 501) or never connected (default 1, Turbo's); a 429 waits out the limit; 0 turns both off |
|
|
566
566
|
| `preflight` | `TURBO_PREFLIGHT` (1 or 0), then `remoteCache.preflight` | send Turbo's `OPTIONS` preflight before each artifact request and go where its `Location` points (relative to `apiUrl`); the token goes along only when the API's `Access-Control-Allow-Headers` admits `Authorization`; a redirected preflight is not followed (default off) |
|
|
567
|
+
| `scope` | — | `RemoteScope`: `'header'` (default): every read names the run's refs, own branch first, in `x-artifact-read-scopes`; `'key'`: each entry is keyed on its ref, a read asks every ref's key in one batch (a server that ignores the header) |
|
|
567
568
|
|
|
568
569
|
The signature is Turbo's current scheme (`artifact-signature:v2`: prefix, hash, team id and body, each length-prefixed, under HMAC-SHA256, base64 in `x-artifact-tag`). A signed body is written to a temp file before its tag can be checked, so one past core's artifact ceiling (2 GiB, at zstd's bound) is refused as it passes it, and is a miss.
|
|
569
570
|
|
|
@@ -620,6 +621,7 @@ With no token from an option, a variable or a config file, both plugins read the
|
|
|
620
621
|
|
|
621
622
|
## Remote-cache behaviour (both)
|
|
622
623
|
|
|
624
|
+
- Task keys carry no branch. `turboCache()` names the run's refs on every read (`HEAD`, `GET`, the batch `POST`) in `x-artifact-read-scopes`, own branch first (`refs/heads/feat-x, refs/heads/main`): from CI's env, else git's current branch, then the default branch. A `PUT` names none: where a write lands is the server's call, from the CI OIDC token checked against its trust policy. `scope: 'key'` folds the ref into each key instead, for a server that ignores the header. `nxCache()` sends plain requests. How a server implements the header: [Branch caches a server enforces](https://vznjs.github.io/vx/blog/branch-caches-a-server-enforces/).
|
|
623
625
|
- A remote error degrades to a **miss** and a warning that names the request, the artifact and the server — `[vx] plugin '@vzn/vx-migrate': upload 32248a2a7c89e241 to https://cache.example.com/v8/artifacts failed: no answer within 60000 ms` — never the token. The run never fails because of the cache.
|
|
624
626
|
- The same failure is said once per run: an unreachable server fails the probe, the download and the upload alike, and the run prints the first and, at its end, `[vx] plugin '@vzn/vx-migrate': 2 more requests failed the same way: Unable to connect. Is the computer able to access the url?` (core does the counting, for every cache plugin).
|
|
625
627
|
- A request answered `5xx` (not `501`), or one that never connected (a refused port, an unresolved host), is sent again after 2 s, `retries` times. A `429` pauses every request to that remote for its `Retry-After` (capped at 10 s a wait, 2 s without one), and the request is sent again when the pause ends, for up to 20 s from its first `429`, as Turbo's client does. A spent deadline is not resent: it has already cost its wait.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx-migrate",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.645",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "Adopt @vzn/vx from Turborepo, Nx or package.json scripts: run the repo unchanged, migrate it to vx.config.ts, and keep your Turbo or Nx remote cache",
|
|
6
6
|
"keywords": [
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
"access": "public"
|
|
41
41
|
},
|
|
42
42
|
"peerDependencies": {
|
|
43
|
-
"@vzn/vx": "^0.0.
|
|
43
|
+
"@vzn/vx": "^0.0.645"
|
|
44
44
|
},
|
|
45
45
|
"repository": {
|
|
46
46
|
"type": "git",
|
package/src/adoption-plugin.ts
CHANGED
|
@@ -60,7 +60,7 @@ export function adoptionPlugin(
|
|
|
60
60
|
mapRun: (ctx: ProjectHookContext) => Promise<AdoptionRun>,
|
|
61
61
|
reads: readonly string[] = [],
|
|
62
62
|
// `config`: a workspace-wide key of the adopted tool's (nx.json `parallel`,
|
|
63
|
-
// Turbo's `
|
|
63
|
+
// Turbo's `concurrency`) that the workspace itself leaves unset.
|
|
64
64
|
// `discover`: a project the tool has that no member glob lists.
|
|
65
65
|
stages: Pick<VxPlugin, 'config' | 'discover'> = {},
|
|
66
66
|
): VxPlugin {
|
package/src/glob-grammar.ts
CHANGED
|
@@ -1,28 +1,22 @@
|
|
|
1
|
-
// The glob grammar Nx and Turbo share and vx's own differ in
|
|
2
|
-
//
|
|
1
|
+
// The glob grammar Nx and Turbo share and vx's own differ in extglobs;
|
|
2
|
+
// both adapters translate through here (items 914, 1031).
|
|
3
3
|
|
|
4
4
|
const EXTGLOB = /([?@+*!])\(([^()]*)\)/g
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* A minimatch-family glob (Nx's minimatch, Turbo's wax) in vx's grammar,
|
|
8
|
-
* or null when it has no safe form.
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* - `[jt]` is the brace set `{j,t}`, plus the literal `[jt]` itself in a
|
|
13
|
-
* positive glob (a superset of inputs is safe; the route dir may be
|
|
14
|
-
* meant); a range within digits or one case of letters (`[a-c]`,
|
|
15
|
-
* `[0-9]`) is its members;
|
|
16
|
-
* - a negated class (`[!a]`) is `?` in a positive glob, a superset;
|
|
8
|
+
* or null when it has no safe form. Character classes pass through (vx
|
|
9
|
+
* reads them the same way); vx has no extglobs, so Nx's own default
|
|
10
|
+
* `production` negation, `?(*.)+(spec|test).[jt]s?(x)`, excluded nothing
|
|
11
|
+
* (item 914):
|
|
17
12
|
* - `?(a|b)` is `{a,b,}` and `@(a|b)` is `{a,b}`;
|
|
18
13
|
* - `+(a|b)` and `*(a|b)` narrow to one repetition, which only a
|
|
19
14
|
* NEGATION may do — it then excludes fewer files, never more;
|
|
20
|
-
* -
|
|
21
|
-
* null.
|
|
15
|
+
* - `!(…)` or nesting is null.
|
|
22
16
|
*/
|
|
23
17
|
export function minimatchToVx(glob: string, negated: boolean): string | null {
|
|
24
|
-
if (
|
|
25
|
-
|
|
18
|
+
if (!glob.includes('(')) return glob
|
|
19
|
+
const out = glob.replace(EXTGLOB, (_whole, kind: string, body: string) => {
|
|
26
20
|
const alts = body.split('|')
|
|
27
21
|
if (alts.some((a) => /[{},]/.test(a))) return '\0'
|
|
28
22
|
if (kind === '?') return `{${alts.join(',')},}`
|
|
@@ -30,36 +24,7 @@ export function minimatchToVx(glob: string, negated: boolean): string | null {
|
|
|
30
24
|
if (!negated || kind === '!') return '\0'
|
|
31
25
|
return kind === '+' ? `{${alts.join(',')}}` : `{${alts.join(',')},}`
|
|
32
26
|
})
|
|
33
|
-
|
|
34
|
-
out = out.replace(/\[([^\]]*)\]/g, (whole, body: string) => {
|
|
35
|
-
if (body === '' || /[{},/]/.test(body)) return '\0'
|
|
36
|
-
if (/^[!^]/.test(body)) return negated || body.length === 1 ? '\0' : '?'
|
|
37
|
-
const chars = classMembers(body)
|
|
38
|
-
if (chars === null) return '\0'
|
|
39
|
-
return negated ? `{${chars.join(',')}}` : `{${whole},${chars.join(',')}}`
|
|
40
|
-
})
|
|
41
|
-
return out.includes('\0') ? null : out
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
const RANGES = ['0123456789', 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ']
|
|
45
|
-
|
|
46
|
-
/** A class body's characters, ranges spelled out; null for a range across kinds or backwards. */
|
|
47
|
-
function classMembers(body: string): string[] | null {
|
|
48
|
-
const out = new Set<string>()
|
|
49
|
-
for (let i = 0; i < body.length; i++) {
|
|
50
|
-
const c = body[i]!
|
|
51
|
-
// A `-` first or last is itself, as in minimatch.
|
|
52
|
-
if (body[i + 1] === '-' && i + 2 < body.length) {
|
|
53
|
-
const end = body[i + 2]!
|
|
54
|
-
const kind = RANGES.find((r) => r.includes(c))
|
|
55
|
-
const from = kind?.indexOf(c) ?? -1
|
|
56
|
-
const to = kind?.indexOf(end) ?? -1
|
|
57
|
-
if (kind === undefined || to < from) return null
|
|
58
|
-
for (const m of kind.slice(from, to + 1)) out.add(m)
|
|
59
|
-
i += 2
|
|
60
|
-
} else out.add(c)
|
|
61
|
-
}
|
|
62
|
-
return [...out]
|
|
27
|
+
return out.includes('\0') || /[?@+*!]\(/.test(out) ? null : out
|
|
63
28
|
}
|
|
64
29
|
|
|
65
30
|
/**
|
|
@@ -73,7 +38,7 @@ function classMembers(body: string): string[] | null {
|
|
|
73
38
|
*/
|
|
74
39
|
export function withoutTakenBack<T>(entries: readonly T[]): T[] {
|
|
75
40
|
const list = [...new Set(entries)]
|
|
76
|
-
const wild = /[*?{}]/
|
|
41
|
+
const wild = /[*?{}[\]\\]/
|
|
77
42
|
const negative = list.flatMap((e) =>
|
|
78
43
|
typeof e === 'string' && e.startsWith('!') ? [e.slice(1)] : [],
|
|
79
44
|
)
|
|
@@ -90,10 +55,10 @@ export function withoutTakenBack<T>(entries: readonly T[]): T[] {
|
|
|
90
55
|
|
|
91
56
|
/**
|
|
92
57
|
* A path from the file system as a task glob that names only itself: a dir
|
|
93
|
-
* named `my{app}` read as a brace matched `myapp` instead
|
|
94
|
-
*
|
|
95
|
-
*
|
|
58
|
+
* named `my{app}` read as a brace matched `myapp` instead, and `app/[id]` a
|
|
59
|
+
* class. `*` and `?` stay, since the schema reads `\*` as a Windows
|
|
60
|
+
* separator, and they match the name too (a superset).
|
|
96
61
|
*/
|
|
97
62
|
export function literalGlob(p: string): string {
|
|
98
|
-
return p.replace(/^!|[{}]/g, '\\$&')
|
|
63
|
+
return p.replace(/^!|[{}[\]]/g, '\\$&')
|
|
99
64
|
}
|
package/src/migrate-nx.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { stat } from 'node:fs/promises'
|
|
|
5
5
|
import path from 'node:path'
|
|
6
6
|
import { type GeneratedProject, type MigrationFormat, type MigrationPlan } from '@vzn/vx'
|
|
7
7
|
import type { PackageMeta } from './packages.js'
|
|
8
|
-
import { mapNxWorkspace,
|
|
8
|
+
import { mapNxWorkspace, parseNxGraph, readNxJson } from './nx/nx-map.js'
|
|
9
9
|
import { spareTrackedOutputs, trackedFiles, trackedKinds } from './tracked-outputs.js'
|
|
10
10
|
import { adoptedToolNotes } from './workspace-notes.js'
|
|
11
11
|
import { relPosix } from './paths.js'
|
|
@@ -165,7 +165,7 @@ async function unlistedProjects(
|
|
|
165
165
|
/** A `.vx/workspace.ts` field nx.json's run settings call for. */
|
|
166
166
|
export interface NxWorkspaceField {
|
|
167
167
|
/** The setting's path: its group, then its field, or a top-level key. */
|
|
168
|
-
readonly field: 'run.concurrency' | 'baseBranch'
|
|
168
|
+
readonly field: 'run.concurrency' | 'baseBranch'
|
|
169
169
|
/** The value as TypeScript source; undefined when vx cannot read it. */
|
|
170
170
|
readonly source: string | undefined
|
|
171
171
|
/** What to tell a user whose workspace file this run does not write. */
|
|
@@ -187,7 +187,6 @@ export function nxWorkspaceFields(json: Record<string, unknown> | undefined): Nx
|
|
|
187
187
|
tasksRunnerOptions?: { default?: { options?: { parallel?: unknown } } }
|
|
188
188
|
defaultBase?: unknown
|
|
189
189
|
affected?: { defaultBase?: unknown }
|
|
190
|
-
maxCacheSize?: unknown
|
|
191
190
|
}
|
|
192
191
|
| undefined
|
|
193
192
|
const out: NxWorkspaceField[] = []
|
|
@@ -207,12 +206,5 @@ export function nxWorkspaceFields(json: Record<string, unknown> | undefined): Nx
|
|
|
207
206
|
note: `nx.json \`defaultBase\` ${JSON.stringify(base.trim())}: add \`baseBranch: ${JSON.stringify(branch)}\` to .vx/workspace.ts`,
|
|
208
207
|
})
|
|
209
208
|
}
|
|
210
|
-
const size = nxSizeText(nx?.maxCacheSize)
|
|
211
|
-
if (size !== undefined)
|
|
212
|
-
out.push({
|
|
213
|
-
field: 'cache.retention',
|
|
214
|
-
source: /^\d+[KMG]?B$/.test(size) ? `{ maxSize: ${quoted(size)} }` : undefined,
|
|
215
|
-
note: `nx.json \`maxCacheSize\`: add \`cache: { retention: { maxSize: ${JSON.stringify(size)} } }\` to .vx/workspace.ts`,
|
|
216
|
-
})
|
|
217
209
|
return out
|
|
218
210
|
}
|
package/src/nx/index.ts
CHANGED
|
@@ -39,7 +39,6 @@ import {
|
|
|
39
39
|
mapNxWorkspace,
|
|
40
40
|
type NxExecutors,
|
|
41
41
|
type NxGraph,
|
|
42
|
-
nxSizeText,
|
|
43
42
|
parseNxGraph,
|
|
44
43
|
readNxJson,
|
|
45
44
|
} from './nx-map.js'
|
|
@@ -166,11 +165,6 @@ export function nx(options: NxPluginOptions = {}): VxPlugin {
|
|
|
166
165
|
rules: { ...workspace.run?.rules, dependencies },
|
|
167
166
|
}
|
|
168
167
|
}
|
|
169
|
-
if (workspace.cache?.retention === undefined) {
|
|
170
|
-
const maxSize = await nxMaxCacheSize(root)
|
|
171
|
-
if (maxSize !== undefined)
|
|
172
|
-
workspace.cache = { ...workspace.cache, retention: { maxSize } }
|
|
173
|
-
}
|
|
174
168
|
},
|
|
175
169
|
// The package manager's members are projects, each with no vx.config
|
|
176
170
|
// written. An integrated Nx repo keeps projects out of that list
|
|
@@ -294,24 +288,6 @@ async function nxDefaultBase(root: string): Promise<string | undefined> {
|
|
|
294
288
|
return typeof b === 'string' && b.trim() !== '' ? b.trim().replace(/^origin\//, '') : undefined
|
|
295
289
|
}
|
|
296
290
|
|
|
297
|
-
/**
|
|
298
|
-
* nx.json's `maxCacheSize`, or `NX_MAX_CACHE_SIZE` above it, as core's
|
|
299
|
-
* size: Nx caps its local cache there (`resolveMaxCacheSize`), and read by
|
|
300
|
-
* nothing a capped cache grew without bound under vx, as turbo.json's
|
|
301
|
-
* `cacheMaxSize` once did (G-49). 1024-based, a fraction truncated to
|
|
302
|
-
* bytes; `0` is no cap. A value Nx would refuse is left to core's
|
|
303
|
-
* refusal, which names the field.
|
|
304
|
-
*/
|
|
305
|
-
async function nxMaxCacheSize(root: string): Promise<string | undefined> {
|
|
306
|
-
const env = process.env['NX_MAX_CACHE_SIZE']
|
|
307
|
-
const raw =
|
|
308
|
-
env !== undefined
|
|
309
|
-
? env
|
|
310
|
-
: ((await readNxJson(root).catch(() => null))?.json as { maxCacheSize?: unknown } | undefined)
|
|
311
|
-
?.maxCacheSize
|
|
312
|
-
return nxSizeText(raw)
|
|
313
|
-
}
|
|
314
|
-
|
|
315
291
|
interface LoadedGraph {
|
|
316
292
|
readonly text: string
|
|
317
293
|
readonly graph: NxGraph
|
package/src/nx/nx-map.ts
CHANGED
|
@@ -1522,20 +1522,3 @@ function followNxGraph(
|
|
|
1522
1522
|
}
|
|
1523
1523
|
}
|
|
1524
1524
|
}
|
|
1525
|
-
|
|
1526
|
-
/** Nx's cache-size grammar (`10GB`, `1.5 GB`, bare bytes) as core's size; `0` is no cap. */
|
|
1527
|
-
export function nxSizeText(raw: unknown): string | undefined {
|
|
1528
|
-
if (typeof raw !== 'string' && typeof raw !== 'number') return undefined
|
|
1529
|
-
const text = String(raw).trim()
|
|
1530
|
-
const m = /^(\d+\.?\d*|\.\d+)\s?([KMG]?B)?$/.exec(text)
|
|
1531
|
-
if (m === null) return text
|
|
1532
|
-
const units = ['B', 'KB', 'MB', 'GB']
|
|
1533
|
-
let n = Math.floor(Number(m[1]) * 1024 ** units.indexOf(m[2] ?? 'B'))
|
|
1534
|
-
if (n === 0) return undefined
|
|
1535
|
-
let u = 0
|
|
1536
|
-
while (u < units.length - 1 && n % 1024 === 0) {
|
|
1537
|
-
n /= 1024
|
|
1538
|
-
u++
|
|
1539
|
-
}
|
|
1540
|
-
return `${n}${units[u]}`
|
|
1541
|
-
}
|
package/src/nx-cache/index.ts
CHANGED
|
@@ -227,15 +227,16 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
227
227
|
return res
|
|
228
228
|
}
|
|
229
229
|
|
|
230
|
-
async has(hash: string): Promise<
|
|
230
|
+
async has(hash: string): Promise<{ size?: number } | null> {
|
|
231
231
|
const res = await this.fetch(hash)
|
|
232
|
-
if (res === null) return
|
|
232
|
+
if (res === null) return null
|
|
233
233
|
await res.body?.cancel()
|
|
234
|
-
|
|
234
|
+
const size = Number(res.headers.get('content-length') ?? NaN)
|
|
235
|
+
return Number.isFinite(size) ? { size } : {}
|
|
235
236
|
}
|
|
236
237
|
|
|
237
|
-
// Nx's record carries no duration
|
|
238
|
-
async put(hash: string, body: Blob
|
|
238
|
+
// Nx's record carries no duration.
|
|
239
|
+
async put(hash: string, body: Blob): Promise<void> {
|
|
239
240
|
if (this.disabled || this.writesRefused) return
|
|
240
241
|
const res = await this.request('PUT', hash, body)
|
|
241
242
|
if (res === undefined) return
|
package/src/turbo/index.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
import { readFileSync } from 'node:fs'
|
|
12
12
|
import { availableParallelism } from 'node:os'
|
|
13
13
|
import path from 'node:path'
|
|
14
|
-
import type { ProjectMeta, VxPlugin
|
|
14
|
+
import type { ProjectMeta, VxPlugin } from '@vzn/vx'
|
|
15
15
|
import type { PackageMeta } from '../packages.js'
|
|
16
16
|
import { memberProjects, withManifests } from '../manifests.js'
|
|
17
17
|
import { refuseUnknownOptions, type PluginOptionKinds } from '@vzn/vx'
|
|
@@ -65,14 +65,6 @@ export function turbo(options: TurboPluginOptions = {}): VxPlugin {
|
|
|
65
65
|
const keys = await workspaceKeys(options.root ?? ctx.workspaceRoot)
|
|
66
66
|
if (workspace.run?.concurrency === undefined && keys.concurrency !== undefined)
|
|
67
67
|
workspace.run = { ...workspace.run, concurrency: keys.concurrency }
|
|
68
|
-
if (workspace.cache?.retention === undefined && keys.cacheRetention !== undefined)
|
|
69
|
-
workspace.cache = {
|
|
70
|
-
...workspace.cache,
|
|
71
|
-
// `olderThan` is built with its unit below.
|
|
72
|
-
retention: keys.cacheRetention as NonNullable<
|
|
73
|
-
NonNullable<WorkspaceConfig['cache']>['retention']
|
|
74
|
-
>,
|
|
75
|
-
}
|
|
76
68
|
// `turbo run --affected` compares with TURBO_SCM_BASE when set, and
|
|
77
69
|
// on GitHub Actions with what the event names.
|
|
78
70
|
const base =
|
|
@@ -99,36 +91,6 @@ export function turbo(options: TurboPluginOptions = {}): VxPlugin {
|
|
|
99
91
|
)
|
|
100
92
|
}
|
|
101
93
|
|
|
102
|
-
/**
|
|
103
|
-
* turbo.json's workspace keys vx has a home for, top level or under
|
|
104
|
-
* `global`: `concurrency` (`"10"`, `"50%"` of the cores) and the local
|
|
105
|
-
* cache's `cacheMaxSize` / `cacheMaxAge` (`"0"` is off; weeks become days).
|
|
106
|
-
* Read by nothing, a repo that capped its cache at 10GB (formbricks) grew
|
|
107
|
-
* it without bound under vx. A value core cannot parse is left to core's
|
|
108
|
-
* refusal, which names the field.
|
|
109
|
-
*/
|
|
110
|
-
const SIZE_UNITS = ['B', 'KB', 'MB', 'GB', 'TB']
|
|
111
|
-
|
|
112
|
-
/**
|
|
113
|
-
* Turbo's size (turborepo-cache `parse_human_size`): 1024-based, a bare
|
|
114
|
-
* number is bytes, a fraction is truncated to bytes. Core takes a whole
|
|
115
|
-
* number with a unit, so `7.5GB` (langfuse) is restated as `7680MB` and
|
|
116
|
-
* `1000000` as `1000000B`; core refused both. Undefined when Turbo reads
|
|
117
|
-
* it as off (under one byte).
|
|
118
|
-
*/
|
|
119
|
-
function turboSize(v: string): string | undefined {
|
|
120
|
-
const m = /^(\d+\.?\d*|\.\d+)([KMGT]?B)?$/i.exec(v)
|
|
121
|
-
if (!m) return v
|
|
122
|
-
let n = Math.floor(Number(m[1]) * 1024 ** SIZE_UNITS.indexOf((m[2] ?? 'B').toUpperCase()))
|
|
123
|
-
if (n === 0) return undefined
|
|
124
|
-
let u = 0
|
|
125
|
-
while (u < SIZE_UNITS.length - 1 && n % 1024 === 0) {
|
|
126
|
-
n /= 1024
|
|
127
|
-
u++
|
|
128
|
-
}
|
|
129
|
-
return `${n}${SIZE_UNITS[u]}`
|
|
130
|
-
}
|
|
131
|
-
|
|
132
94
|
const UNKNOWN_SHA = '0'.repeat(40)
|
|
133
95
|
|
|
134
96
|
/**
|
|
@@ -173,9 +135,12 @@ function githubActionsBase(root: string, remoteFallback: boolean): string | unde
|
|
|
173
135
|
return before === '' ? undefined : before
|
|
174
136
|
}
|
|
175
137
|
|
|
138
|
+
/**
|
|
139
|
+
* turbo.json's workspace keys vx has a home for, top level or under
|
|
140
|
+
* `global`: `concurrency` (`"10"`, `"50%"` of the cores).
|
|
141
|
+
*/
|
|
176
142
|
async function workspaceKeys(root: string): Promise<{
|
|
177
143
|
concurrency?: number
|
|
178
|
-
cacheRetention?: { maxSize?: string; olderThan?: string }
|
|
179
144
|
remoteBaseFallback?: boolean
|
|
180
145
|
}> {
|
|
181
146
|
const file = await turboConfigFile(root)
|
|
@@ -199,7 +164,6 @@ async function workspaceKeys(root: string): Promise<{
|
|
|
199
164
|
}
|
|
200
165
|
const out: {
|
|
201
166
|
concurrency?: number
|
|
202
|
-
cacheRetention?: { maxSize?: string; olderThan?: string }
|
|
203
167
|
remoteBaseFallback?: boolean
|
|
204
168
|
} = {}
|
|
205
169
|
const flags = raw['futureFlags'] as { githubActionsRemoteBaseRefFallback?: unknown } | undefined
|
|
@@ -212,19 +176,6 @@ async function workspaceKeys(root: string): Promise<{
|
|
|
212
176
|
: Number(c)
|
|
213
177
|
if (Number.isInteger(n) && n > 0) out.concurrency = n
|
|
214
178
|
}
|
|
215
|
-
const size = read('cacheMaxSize', 'TURBO_CACHE_MAX_SIZE')
|
|
216
|
-
const maxSize = size === undefined ? undefined : turboSize(size)
|
|
217
|
-
const age = read('cacheMaxAge', 'TURBO_CACHE_MAX_AGE')
|
|
218
|
-
// Turbo reads a bare number as days (`parse_human_duration`).
|
|
219
|
-
const olderThan = age
|
|
220
|
-
?.replace(/^(\d+)w$/i, (_, w: string) => `${Number(w) * 7}d`)
|
|
221
|
-
.replace(/^\d+$/, (d) => `${d}d`)
|
|
222
|
-
if (maxSize !== undefined || olderThan !== undefined) {
|
|
223
|
-
out.cacheRetention = {
|
|
224
|
-
...(maxSize !== undefined ? { maxSize } : {}),
|
|
225
|
-
...(olderThan !== undefined ? { olderThan } : {}),
|
|
226
|
-
}
|
|
227
|
-
}
|
|
228
179
|
return out
|
|
229
180
|
}
|
|
230
181
|
|
package/src/turbo/turbo-map.ts
CHANGED
|
@@ -2023,9 +2023,9 @@ function buildTask(
|
|
|
2023
2023
|
continue
|
|
2024
2024
|
}
|
|
2025
2025
|
const neg = i.startsWith('!')
|
|
2026
|
-
// Turbo's globs have
|
|
2027
|
-
//
|
|
2028
|
-
//
|
|
2026
|
+
// Turbo's globs have extglobs and vx's do not, so
|
|
2027
|
+
// `src/**/*.@(js|ts)` keyed on nothing and an edit replayed the old
|
|
2028
|
+
// build (item 1031). A positive glob with no safe
|
|
2029
2029
|
// form widens to every package file; a negation with none is dropped
|
|
2030
2030
|
// (it would exclude fewer files, never more).
|
|
2031
2031
|
const wax = minimatchToVx(neg ? i.slice(1) : i, neg)
|
|
@@ -2103,19 +2103,16 @@ function buildTask(
|
|
|
2103
2103
|
const zeroed = new Set<string>()
|
|
2104
2104
|
const wsOutFiles: string[] = []
|
|
2105
2105
|
for (const raw of def.outputs ?? []) {
|
|
2106
|
-
//
|
|
2107
|
-
//
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
const rest = cut < 0 ? null : minimatchToVx(raw.slice(cut + 1), raw.startsWith('!'))
|
|
2111
|
-
if (cut >= 0 && rest === null) {
|
|
2106
|
+
// Turbo's grammar: an extglob in `dist/**/*.@(js|mjs)` matched
|
|
2107
|
+
// nothing, so a hit restored nothing (item 1031).
|
|
2108
|
+
const mapped = minimatchToVx(raw, raw.startsWith('!'))
|
|
2109
|
+
if (mapped === null) {
|
|
2112
2110
|
todos.push(
|
|
2113
2111
|
`output ${JSON.stringify(raw)}: glob syntax vx cannot take — task runs uncached; ` +
|
|
2114
2112
|
'declare the exact outputs in a vx.config to cache it',
|
|
2115
2113
|
)
|
|
2116
2114
|
return { name, todos, task, uses }
|
|
2117
2115
|
}
|
|
2118
|
-
const mapped = cut < 0 ? raw : `${raw.slice(0, cut + 1)}${rest}`
|
|
2119
2116
|
// A `!` output takes a path back, as Turbo's does (A-44): Next's
|
|
2120
2117
|
// `.next/**` minus `!.next/cache/**` saved the cache and cleaned it
|
|
2121
2118
|
// before every run while vx outputs could not exclude.
|
package/src/turbo-cache/index.ts
CHANGED
|
@@ -16,6 +16,7 @@ import { unlink } from 'node:fs/promises'
|
|
|
16
16
|
import { tmpdir } from 'node:os'
|
|
17
17
|
import path from 'node:path'
|
|
18
18
|
import {
|
|
19
|
+
type ArtifactInfo,
|
|
19
20
|
definePlugin,
|
|
20
21
|
type RemoteCacheLayer,
|
|
21
22
|
type RemoteOpenContext,
|
|
@@ -24,6 +25,7 @@ import {
|
|
|
24
25
|
type PluginOptionKinds,
|
|
25
26
|
} from '@vzn/vx'
|
|
26
27
|
import { deadline } from '../remote-deadline.js'
|
|
28
|
+
import { keyedByRef, readScopes, remoteScope, type RemoteScope } from './scope.js'
|
|
27
29
|
import { OutageBreaker } from '../remote-breaker.js'
|
|
28
30
|
import { Cooldown, withRetry } from '../remote-retry.js'
|
|
29
31
|
import {
|
|
@@ -80,6 +82,14 @@ export interface TurboCacheOptions {
|
|
|
80
82
|
* `remoteCache.preflight`; default off.
|
|
81
83
|
*/
|
|
82
84
|
preflight?: boolean
|
|
85
|
+
/**
|
|
86
|
+
* How branches stay apart (default `'header'`): every read names the run's
|
|
87
|
+
* refs in `x-artifact-read-scopes`, for a server that files writes by the
|
|
88
|
+
* CI token. `'key'` also folds the run's ref into each key and reads each
|
|
89
|
+
* ref's key in order, for a server that ignores the header. The server's
|
|
90
|
+
* OIDC policy alone decides where a write lands.
|
|
91
|
+
*/
|
|
92
|
+
scope?: RemoteScope
|
|
83
93
|
}
|
|
84
94
|
|
|
85
95
|
/** turbo.json's `remoteCache` block: the source below Turbo's environment, as in Turbo. */
|
|
@@ -126,6 +136,17 @@ export const MIN_SIGNATURE_KEY_LENGTH = 32
|
|
|
126
136
|
// Bun's own hasher and the global `crypto`, not `node:crypto`: importing that
|
|
127
137
|
// module cost every `@vzn/vx-migrate` user 6.6 ms at startup, remote cache
|
|
128
138
|
// or not (G-121).
|
|
139
|
+
/** A size and a duration as the wire spells them (a header string, a JSON number), kept when they are counts. */
|
|
140
|
+
function infoOf(size: unknown, durationMs: unknown): ArtifactInfo {
|
|
141
|
+
const count = (v: unknown): number | undefined => {
|
|
142
|
+
const n = typeof v === 'string' && v !== '' ? Number(v) : v
|
|
143
|
+
return typeof n === 'number' && Number.isFinite(n) && n >= 0 ? n : undefined
|
|
144
|
+
}
|
|
145
|
+
const s = count(size)
|
|
146
|
+
const d = count(durationMs)
|
|
147
|
+
return { ...(s === undefined ? {} : { size: s }), ...(d === undefined ? {} : { durationMs: d }) }
|
|
148
|
+
}
|
|
149
|
+
|
|
129
150
|
function updateLength(mac: Bun.CryptoHasher, byteLength: number): void {
|
|
130
151
|
const len = Buffer.alloc(8)
|
|
131
152
|
len.writeBigUInt64LE(BigInt(byteLength))
|
|
@@ -448,6 +469,8 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
448
469
|
private readonly keychainRemote: KeychainRemote
|
|
449
470
|
/** The remote's rate limit, which every request of this client waits out. */
|
|
450
471
|
private readonly cooldown = new Cooldown()
|
|
472
|
+
/** Headers every read carries: the refs it may be answered from (`readScopes`). */
|
|
473
|
+
reads: Record<string, string> = {}
|
|
451
474
|
constructor(
|
|
452
475
|
private readonly config: TurboCacheConfig,
|
|
453
476
|
private readonly fetchImpl: typeof fetch = fetch,
|
|
@@ -614,25 +637,26 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
614
637
|
await revokeOidc(this.config.oidc!, token, this.fetchImpl, this.wait)
|
|
615
638
|
}
|
|
616
639
|
|
|
617
|
-
async has(hash: string): Promise<
|
|
618
|
-
if (this.disabled) return
|
|
619
|
-
const res = await this.request('HEAD', `/${hash}
|
|
620
|
-
if (res === undefined) return
|
|
621
|
-
if (res.status === 200)
|
|
640
|
+
async has(hash: string): Promise<ArtifactInfo | null> {
|
|
641
|
+
if (this.disabled) return null
|
|
642
|
+
const res = await this.request('HEAD', `/${hash}`, { headers: this.reads })
|
|
643
|
+
if (res === undefined) return null
|
|
644
|
+
if (res.status === 200)
|
|
645
|
+
return infoOf(res.headers.get('content-length'), res.headers.get('x-artifact-duration'))
|
|
622
646
|
if (res.status === 404) {
|
|
623
647
|
this.absent.add(hash)
|
|
624
|
-
return
|
|
648
|
+
return null
|
|
625
649
|
}
|
|
626
650
|
throw new Error(`HTTP ${res.status}`)
|
|
627
651
|
}
|
|
628
652
|
|
|
629
|
-
async hasMany(hashes: readonly string[]): Promise<
|
|
630
|
-
if (this.disabled || hashes.length === 0) return this.disabled ? new
|
|
653
|
+
async hasMany(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
|
|
654
|
+
if (this.disabled || hashes.length === 0) return this.disabled ? new Map() : null
|
|
631
655
|
const res = await this.request('POST', '', {
|
|
632
656
|
body: JSON.stringify({ hashes }),
|
|
633
|
-
headers: { 'Content-Type': 'application/json' },
|
|
657
|
+
headers: { 'Content-Type': 'application/json', ...this.reads },
|
|
634
658
|
})
|
|
635
|
-
if (res === undefined) return new
|
|
659
|
+
if (res === undefined) return new Map()
|
|
636
660
|
if (res.status !== 200) return null
|
|
637
661
|
// An answer per hash is a few hundred bytes of ArtifactInfo; a reply
|
|
638
662
|
// past this bound is no answer, read no further than the bound (L-9).
|
|
@@ -643,13 +667,15 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
643
667
|
// Each hash answers ArtifactInfo, null, or `{ error: { message } }`; an
|
|
644
668
|
// error entry is not a stored artifact, and it was counted as one (a GET
|
|
645
669
|
// spent on a 404, item 929).
|
|
646
|
-
const held = new
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
670
|
+
const held = new Map<string, ArtifactInfo>()
|
|
671
|
+
for (const h of hashes) {
|
|
672
|
+
const v = info[h]
|
|
673
|
+
if (typeof v !== 'object' || v === null || 'error' in v) this.absent.add(h)
|
|
674
|
+
else {
|
|
675
|
+
const { size, taskDurationMs } = v as { size?: unknown; taskDurationMs?: unknown }
|
|
676
|
+
held.set(h, infoOf(size, taskDurationMs))
|
|
677
|
+
}
|
|
678
|
+
}
|
|
653
679
|
return held
|
|
654
680
|
}
|
|
655
681
|
|
|
@@ -658,7 +684,7 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
658
684
|
// A gateway that keys binary media on Accept base64-encodes a `*/*`
|
|
659
685
|
// response, and every hit read as a corrupt artifact (nx#33092's class).
|
|
660
686
|
const res = await this.request('GET', `/${hash}`, {
|
|
661
|
-
headers: { Accept: 'application/octet-stream' },
|
|
687
|
+
headers: { Accept: 'application/octet-stream', ...this.reads },
|
|
662
688
|
})
|
|
663
689
|
if (res === undefined) return null
|
|
664
690
|
if (res.status !== 200) {
|
|
@@ -723,6 +749,7 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
723
749
|
// uploads: a lost store costs more. A server the HEAD cannot reach
|
|
724
750
|
// fails the put, as the PUT would.
|
|
725
751
|
const head = await this.request('HEAD', `/${hash}`, {
|
|
752
|
+
headers: this.reads,
|
|
726
753
|
timeoutMs: this.config.uploadTimeoutMs,
|
|
727
754
|
})
|
|
728
755
|
if (head === undefined || head.status === 200) return
|
|
@@ -801,6 +828,7 @@ const TURBO_CACHE_KEYS: PluginOptionKinds<TurboCacheOptions> = {
|
|
|
801
828
|
retries: 'number',
|
|
802
829
|
preflight: 'boolean',
|
|
803
830
|
oidc: 'any',
|
|
831
|
+
scope: 'string',
|
|
804
832
|
}
|
|
805
833
|
|
|
806
834
|
/**
|
|
@@ -814,6 +842,7 @@ const TURBO_CACHE_KEYS: PluginOptionKinds<TurboCacheOptions> = {
|
|
|
814
842
|
*/
|
|
815
843
|
export function turboCache(options: TurboCacheOptions = {}): VxPlugin {
|
|
816
844
|
refuseUnknownOptions('turboCache()', options, TURBO_CACHE_KEYS)
|
|
845
|
+
const scope = remoteScope('turbo-cache', options.scope)
|
|
817
846
|
// The run's layer and warn, for `teardown` to revoke an exchanged token.
|
|
818
847
|
let opened: { layer: TurboRemoteCache; warn: (m: string) => void } | undefined
|
|
819
848
|
return definePlugin(import.meta, {
|
|
@@ -829,8 +858,9 @@ export function turboCache(options: TurboCacheOptions = {}): VxPlugin {
|
|
|
829
858
|
// cache directory, which the sandbox walls, not the shared temp dir a
|
|
830
859
|
// sandboxed task may read (L-45).
|
|
831
860
|
const layer = new TurboRemoteCache(config, fetch, ctx.cacheDir)
|
|
861
|
+
layer.reads = readScopes(ctx.refs)
|
|
832
862
|
if (config.oidc !== undefined) opened = { layer, warn: ctx.warn }
|
|
833
|
-
return layer
|
|
863
|
+
return scope === 'key' ? keyedByRef(layer, ctx.refs) : layer
|
|
834
864
|
},
|
|
835
865
|
async teardown() {
|
|
836
866
|
const run = opened
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import type { ArtifactInfo, RemoteCacheLayer } from '@vzn/vx'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* How a remote keeps branches apart (2026-10-20). `'header'`: plain keys,
|
|
5
|
+
* and every read names the refs it may be answered from, in order, in
|
|
6
|
+
* `x-artifact-read-scopes`; a server that files writes by the caller's CI
|
|
7
|
+
* token reads them, any other ignores it. `'key'`: each entry's key also
|
|
8
|
+
* folds a ref, for a server that keeps one bucket; a read asks each ref's
|
|
9
|
+
* key and the first hit wins.
|
|
10
|
+
*/
|
|
11
|
+
export type RemoteScope = 'header' | 'key'
|
|
12
|
+
|
|
13
|
+
export function remoteScope(name: string, scope: unknown): RemoteScope {
|
|
14
|
+
if (scope === undefined || scope === 'header' || scope === 'key') return scope ?? 'header'
|
|
15
|
+
throw new Error(`vx/${name}: scope must be 'header' or 'key', got ${JSON.stringify(scope)}`)
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** A server caps the read header at this many refs (the server rules), so vx never sends more. */
|
|
19
|
+
const MAX_READ_SCOPES = 8
|
|
20
|
+
|
|
21
|
+
/** The read header: the refs in order. Writes carry none: the server files them by the token. */
|
|
22
|
+
export function readScopes(refs: readonly string[]): Record<string, string> {
|
|
23
|
+
return refs.length === 0
|
|
24
|
+
? {}
|
|
25
|
+
: { 'x-artifact-read-scopes': refs.slice(0, MAX_READ_SCOPES).join(', ') }
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** The key `ref` stores `hash` under: 16 hex, task-key shaped, so every wire takes it. */
|
|
29
|
+
export function refKey(ref: string, hash: string): string {
|
|
30
|
+
return Bun.hash.xxHash3(`vx-cache-ref\0${ref}\0${hash}`).toString(16).padStart(16, '0')
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const QUEUE = /^refs\/heads\/gh-readonly-queue\//
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* `inner` keyed by ref: a write lands under the run's own ref's key (a
|
|
37
|
+
* merge queue's, its base branch's, which its results become), a read asks
|
|
38
|
+
* each ref's key in order. A merge queue's own ref holds nothing, so it is
|
|
39
|
+
* never asked.
|
|
40
|
+
*/
|
|
41
|
+
export function keyedByRef(inner: RemoteCacheLayer, allRefs: readonly string[]): RemoteCacheLayer {
|
|
42
|
+
const refs = QUEUE.test(allRefs[0] ?? '') ? allRefs.slice(1) : allRefs
|
|
43
|
+
const write = refs[0]
|
|
44
|
+
if (write === undefined) return inner
|
|
45
|
+
/** Task key → the stored key a batch probe found it under. */
|
|
46
|
+
const found = new Map<string, string>()
|
|
47
|
+
const hasMany = inner.hasMany
|
|
48
|
+
const firstHeld = async (hash: string): Promise<ArtifactInfo | null> => {
|
|
49
|
+
const known = found.get(hash)
|
|
50
|
+
if (known !== undefined) return inner.has(known)
|
|
51
|
+
for (const ref of refs) {
|
|
52
|
+
const info = await inner.has(refKey(ref, hash))
|
|
53
|
+
if (info !== null) return info
|
|
54
|
+
}
|
|
55
|
+
return null
|
|
56
|
+
}
|
|
57
|
+
return {
|
|
58
|
+
...(inner.endpoint !== undefined ? { endpoint: inner.endpoint } : {}),
|
|
59
|
+
...(inner.open !== undefined ? { open: (ctx) => inner.open!(ctx) } : {}),
|
|
60
|
+
has: firstHeld,
|
|
61
|
+
...(hasMany !== undefined
|
|
62
|
+
? {
|
|
63
|
+
hasMany: async (hashes) => {
|
|
64
|
+
const keys = hashes.map((h) => refs.map((r) => refKey(r, h)))
|
|
65
|
+
const held = await hasMany.call(inner, keys.flat())
|
|
66
|
+
if (held === null) return null
|
|
67
|
+
const out = new Map<string, ArtifactInfo>()
|
|
68
|
+
hashes.forEach((h, i) => {
|
|
69
|
+
const key = keys[i]!.find((k) => held.has(k))
|
|
70
|
+
if (key === undefined) return
|
|
71
|
+
found.set(h, key)
|
|
72
|
+
out.set(h, held.get(key)!)
|
|
73
|
+
})
|
|
74
|
+
return out
|
|
75
|
+
},
|
|
76
|
+
}
|
|
77
|
+
: {}),
|
|
78
|
+
get: async (hash) => {
|
|
79
|
+
const known = found.get(hash)
|
|
80
|
+
if (known !== undefined) return inner.get(known)
|
|
81
|
+
for (const ref of refs) {
|
|
82
|
+
const hit = await inner.get(refKey(ref, hash))
|
|
83
|
+
if (hit !== null) return hit
|
|
84
|
+
}
|
|
85
|
+
return null
|
|
86
|
+
},
|
|
87
|
+
put: (hash, body, meta) => inner.put(refKey(write, hash), body, meta),
|
|
88
|
+
}
|
|
89
|
+
}
|