@vzn/vx-migrate 0.0.640 → 0.0.642
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -14
- package/package.json +2 -2
- package/src/adopt.ts +15 -0
- package/src/migrate-turbo.ts +3 -0
- package/src/migrate.ts +26 -15
- package/src/nx/nx-map.ts +41 -4
- package/src/nx-cache/index.ts +46 -11
- package/src/remote-cache.ts +1 -1
- package/src/remote-retry.ts +52 -12
- package/src/remote-token.ts +48 -0
- package/src/tsc-buildinfo.ts +168 -0
- package/src/turbo/index.ts +11 -2
- package/src/turbo/turbo-map.ts +20 -0
- package/src/turbo-cache/index.ts +141 -64
- package/src/turbo-cache/oidc.ts +171 -0
package/README.md
CHANGED
|
@@ -62,7 +62,7 @@ Rules:
|
|
|
62
62
|
- A transit node (Turbo's documented pattern, its with-vitest example: `transit: { dependsOn: ["^transit"] }` with no script anywhere, and `test: { dependsOn: ["transit"] }`) is a key-only task in each package that defines it: a keyed group (no command, nothing spawns) keyed on its inputs, with its `^transit` edge, so a dependant's `test` re-keys on its dependencies' sources as Turbo re-hashes it. A `^` task that some package has a script for, or that nothing depends on, is no transit node. A package without the script of a `^` task others run gets the same key-only task, as Turbo hashes a no-op node there: an edit to it re-keys its dependants. A `build` whose only edge is `^build` is not written: core gives every project with no `build` that node (solid's three script-less packages each got a `build` of their own). A dependency without the script whose own edges lack `^x` gets it too, so a `^x` stops there as Turbo's does (opencode's `build: { dependsOn: [] }`: core walked past to the builds below, which Turbo's `^build` never reaches).
|
|
63
63
|
- `with` (tasks Turbo runs alongside, `web#dev` with `api#dev`): an edge to each sidecar that is persistent, so vx starts the task once the sidecar has spawned and runs the sidecar only when the task runs; a sidecar that ends would be waited for, so it is a todo instead, and a pair that names each other keeps one edge (two would be a cycle). A task with no script whose `with` names persistent sidecars (Turbo's with-tailwind example: `ui` has no `dev` script, its `dev` starts `dev:styles` and `dev:components`) is a group task that depends on them.
|
|
64
64
|
- `inputs`: a structured entry (Turbo 2.11) is its `globs`, plus `**/*` with `withDefaults`, for `startup` and `jit` alike; `dependencyOutputs` adds none where the producer is cached (vx folds each dependency's key); from a `cache: false` producer (vercel/vercel's `//#generate:cache-keys`, which records the host) the named files, or its outputs, are a `inputs.workspaceRuntime` probe, and `inputs.tasks` leaves the producer out, so core answers the probe after it ran. Then: absent or `[]` → `**/*` (Turbo's default); `$TURBO_DEFAULT$` → `**/*`; exclusions alone narrow `**/*`; `$TURBO_ROOT$/<path>` → `inputs.workspaceFiles` (negation kept); any other `$TURBO_ROOT$` use is a todo. `globalDependencies` and Turbo 1's `globalDotEnv` land in `workspaceFiles` too, and a task's Turbo 1 `dotEnv` in its `files`. Turbo 1's `$NAME` entries, in `globalDependencies` or a task's `dependsOn`, are env vars: they join `inputs.env` and `exec.env.passThrough`. A package nested in the task's package (cal.com's apps inside `@calcom/app-store`) is hashed by Turbo with its parent, and core's globs stop at a project: each glob that reaches one is also listed in `inputs.workspaceFiles`. Turbo's glob grammar is translated as Nx's is (`*.[jt]s` is `*.{[jt],j,t}s`); an input with no safe form widens to `**/*` with a todo. A wildcard `X/**` input, `!` or not, also lists `X`, as outputs do: Turbo's `**` matches zero segments, so `*/**` keys on the package-root files too. A `.env`-shaped input (`.env*`, `.env.local`, a task's Turbo 1 `dotEnv`, create-turbo's `globalDependencies: ["**/.env.*local"]`) is gitignored as a rule, so it is keyed by a probe that hashes the `.env` files it can name (`inputs.runtime`) or, for a root entry, the files its root-relative globs name, hidden ones too, as Turbo's `*` matches them (`workspaceRuntime`; `**/<name>` is a `find -name` that skips `node_modules`, and another `**` walks the workspace), rather than as a file glob git never reports. A package whose `.env` globs all sit at its root gets a one-shell probe of that directory; one below the root walks the package.
|
|
65
|
-
- `outputs`: `$TURBO_ROOT$/<path>` → `cache.outputs.workspaceFiles`; a negated output rides beside its positives and takes its paths back from the clean, the save and the restore (Next's `.next/**` minus `!.next/cache/**` keeps the cache), and one with no positive beside it takes back nothing and is dropped. Past its first segment an output takes Turbo's grammar (`dist/**/*.[cm]js`); the first stays a literal (a route directory). A wildcard `X/**` also lists `X`: Turbo's `**` matches zero segments, so medusa's `*/**` caches the package-root `tsconfig.tsbuildinfo` too (its `package.json` is taken back, below); a literal `dist/**` needs no `dist`. An output whose first segment is a wildcard (`**/*.d.ts`, medusa's `*/**`) stays cached, as in `nx()` (`turbo()` 2026-10-14): Turbo never cleans an output, vx cleans it before every run and every restore, and such a glob can reach the sources; core never cleans a committed file, and every untracked file it reaches is deleted, which `vx lock`'s warnings name (`output "*/**": … every untracked file this glob reaches is deleted`). Negations beside it (`!src/**`) keep what they name. Without git the files cannot be told apart, and the task runs uncached with a todo. A glob that reaches no tracked kind gets no warning: one segment with a literal extension the package tracks no file of (`*.xml`), `**/<dir>/**` when the package tracks nothing under a directory of that name (vercel/ai's `**/dist/**`), and a first segment no tracked top-level entry matches (tldraw's `dist-*/**`). An output that covers the package's own `package.json` (trpc's client build lists it: the build rewrites `exports`) takes the manifest back with `!package.json` and keeps the cache, in `turbo()` as in `nx()` (owner, 2026-10-03): vx would otherwise delete it before every run. The build rewrites a file it may list as an input (`@vzn/vx-lockfile` lists `package.json`), so the Turbo mapping also sets `inputStability: { strategy: 'mutable' }` and an identical rewrite counts as unchanged. So does one that covers the package's vx.config; `turbo()` and `nx()` check the config the package has, and `vx-migrate` the one it writes, so sanity's `*.js` shims stay cached beside none. `nx()` keeps a wildcard-first output cached the same way (owner, 2026-10-03). A committed file under an output (typescript-eslint's `data/sponsors.json` in a cached `data`) needs nothing: core never cleans a file git tracks, so it survives and the task keeps its cache.
|
|
65
|
+
- `outputs`: `$TURBO_ROOT$/<path>` → `cache.outputs.workspaceFiles`; a negated output rides beside its positives and takes its paths back from the clean, the save and the restore (Next's `.next/**` minus `!.next/cache/**` keeps the cache), and one with no positive beside it takes back nothing and is dropped. Past its first segment an output takes Turbo's grammar (`dist/**/*.[cm]js`); the first stays a literal (a route directory). A wildcard `X/**` also lists `X`: Turbo's `**` matches zero segments, so medusa's `*/**` caches the package-root `tsconfig.tsbuildinfo` too (its `package.json` is taken back, below); a literal `dist/**` needs no `dist`. An output whose first segment is a wildcard (`**/*.d.ts`, medusa's `*/**`) stays cached, as in `nx()` (`turbo()` 2026-10-14): Turbo never cleans an output, vx cleans it before every run and every restore, and such a glob can reach the sources; core never cleans a committed file, and every untracked file it reaches is deleted, which `vx lock`'s warnings name (`output "*/**": … every untracked file this glob reaches is deleted`). Negations beside it (`!src/**`) keep what they name. Without git the files cannot be told apart, and the task runs uncached with a todo. A glob that reaches no tracked kind gets no warning: one segment with a literal extension the package tracks no file of (`*.xml`), `**/<dir>/**` when the package tracks nothing under a directory of that name (vercel/ai's `**/dist/**`), and a first segment no tracked top-level entry matches (tldraw's `dist-*/**`). An output that covers the package's own `package.json` (trpc's client build lists it: the build rewrites `exports`) takes the manifest back with `!package.json` and keeps the cache, in `turbo()` as in `nx()` (owner, 2026-10-03): vx would otherwise delete it before every run. The build rewrites a file it may list as an input (`@vzn/vx-lockfile` lists `package.json`), so the Turbo mapping also sets `inputStability: { strategy: 'mutable' }` and an identical rewrite counts as unchanged. So does one that covers the package's vx.config; `turbo()` and `nx()` check the config the package has, and `vx-migrate` the one it writes, so sanity's `*.js` shims stay cached beside none. `nx()` keeps a wildcard-first output cached the same way (owner, 2026-10-03). A committed file under an output (typescript-eslint's `data/sponsors.json` in a cached `data`) needs nothing: core never cleans a file git tracks, so it survives and the task keeps its cache. A task whose command runs `tsc` that writes a `.tsbuildinfo` (`-b`, `composite` or `incremental`, read through the tsconfig's `extends` and, for `-b`, its `references`) outside its outputs also outputs that file: tsc re-emits only what its buildinfo calls changed, so with `rootDir: "src"` and `outDir: "dist"` its `tsconfig.tsbuildinfo` beside the tsconfig survived the clean before a miss and the miss cached a `dist/` holding only the edited file. Nx infers the file as an output; Turbo never cleans `dist/`.
|
|
66
66
|
- `env` / `passThroughEnv`: explicit names go to `inputs.env` (env only) and `exec.env.passThrough` (both, plus both globals); a wildcard expands over the run's environment under `turbo()`, which maps where the tasks run (a new variable name such a wildcard matches maps afresh); the CLI, which writes for another environment, lists the names the task's package and its workspace dependencies spell (a root task: the whole repo; clerk's `E2E_*`), noted once per entry, and one nothing spells is a todo; `*` is the only wildcard, as in Turbo, so `\*`, a leading `\!`, `?` and `[` are literal: unkey's `NEXT_PUBLIC_\*` names one variable, which vx cannot key, so it is dropped and reported once for the workspace with the count of tasks naming it (openstatus's root `build` env `\*` had written one todo into each of 45 configs; `turbo()`: only when the run's environment sets it, as Turbo hashes nothing otherwise); and a `!` entry (openstatus' `!NEXT_PUBLIC_VERCEL_URL`) removes the names it matches from its list, with no todo, since vx matches no name it is not given. A name both a global list and the task's own list carry is listed once. A name that is no shell variable name (`p-q`) is keyed but left out of `exec.env.passThrough`, as `sh` drops it before the task runs.
|
|
67
67
|
- Framework inference (Turbo hashes and passes Next's `NEXT_PUBLIC_*`, Vite's `VITE_*`, … for a package that depends on the framework): `turbo()` expands the prefix over the run's environment; the CLI writes the names the tracked source and `.env.example` of the package and its workspace dependencies spell (the bare prefix, as a `startsWith` test spells it, is none) (Next bundles theirs: cal.com's web, 23 alone, 58 with them), and a note asks for any only an installed dependency reads.
|
|
68
68
|
|
|
@@ -231,7 +231,7 @@ It is the one command a repo needs (`pnpx @vzn/vx-migrate` in a pnpm repo works
|
|
|
231
231
|
- In a Turbo or Nx repo, in a terminal, it asks which adoption you want: **native** (the default, a `vx.config.ts` per package) or **keep** (a workspace file declaring `turbo()` or `nx()`). `--native` / `--keep` answer it; with no terminal (CI, a pipe) it is native. Elsewhere both are refused. A repo already set up (a workspace file declaring neither, and project configs) is said to be, exit 0, unless `--force`.
|
|
232
232
|
- Native or keep, a remote cache the repo shows is declared: `turboCache()` for turbo.json's enabled `remoteCache`, a `turbo link`ed `.turbo/config.json` or a CI file (`.github/workflows/*`, `.gitlab-ci.yml`, `.circleci/config.yml`) setting `TURBO_TOKEN`; `nxCache()` for one setting `NX_SELF_HOSTED_REMOTE_CACHE_SERVER`. The report names the file that showed it. Nx Cloud (`nxCloudId`, `nxCloudAccessToken`, the `nx-cloud` runner) is named instead: vx does not speak its wire.
|
|
233
233
|
- With no `.vx/workspace.*` yet, it writes one declaring the plugins the repo calls for (`src/workspace-plugins.ts`): the remote cache above, the `@vzn/vx-lockfile` factory for the lockfile (`pnpm()`, `bun()`, `npm()`, `yarn()`), `scheduleHistoryPlugin()`, and `github()` when `.github/workflows` exists, from `@vzn/vx-ci`, each installed beside `@vzn/vx` at vx-migrate's own version. Keep adds them to the file it writes, or to one already in that shape; any other workspace file is the user's and left alone. Native over a file vx wrote (a keep adoption's) rewrites it: `turbo()` and `nx()` go, since the configs written replace them, and the other plugins stay.
|
|
234
|
-
- It installs what the written files import with the repo's own manager (`packageManager`, else the lockfile): `@vzn/vx`, the
|
|
234
|
+
- It installs what the written files import with the repo's own manager (`packageManager`, else the lockfile): `@vzn/vx`, every `@vzn/vx-*` package the workspace file imports (one an earlier init wrote included), and `@vzn/vx-migrate` for keep or when a written task runs its `nx-exec` or `nx-env` bin (`pnpm add -D -w`, `yarn add -D -W` on Yarn 1 and without `-W` on Yarn 2+, `bun add -d`, `npm install -D`). A package the root both lists and has installed at vx-migrate's own version is left alone; any other is installed at that version, and so is every other `@vzn/vx-*` plugin the root lists from the registry (one install moves them together, as `vx upgrade` does). Native removes a listed `@vzn/vx-migrate` when nothing written needs it (no `nx-exec` / `nx-env` task, no `turboCache()` or `nxCache()` left in the workspace file): `pnpm remove -w`, `yarn remove` (`-W` on Yarn 1), `bun remove`, `npm uninstall`. `--no-install` skips both. vx-migrate runs only with the `@vzn/vx` of its own release, which it installs: one of another release (a `bun x` install directory keeps the one it had) is refused, naming where it was loaded.
|
|
235
235
|
- Native, once the files are written, it ends with `vx lock`: `.vx/projects.lock.json` is written and the graph rules are checked, its line the report's `lock` row. A refusal prints above the report and the configs stay written (exit 0). `--no-install` skips it, since the plugins the workspace file declares are not installed yet, and `next` then starts with `vx lock`.
|
|
236
236
|
|
|
237
237
|
A Lerna repo is an Nx one: Lerna 6+ runs `lerna run` on Nx's task runner over the graph `nx graph` exports, nx.json or not, so a `lerna.json` with no nx.json and no turbo.json is mapped from that graph, and keep writes the workspace file declaring `nx()`. Lerna is the installed one, else the root manifest's range; with neither, the lerna.json is another tool's (lerna-lite reads it too, and runs no Nx). A root whose scripts never `lerna run` runs its tasks another way and publishes with Lerna (webdriverio: `run-s`, `pnpm -r`), so its scripts stay the source. Not where Lerna runs its own runner: `useNx: false`, or Lerna 5 without `useNx: true`; those map their scripts. Beside turbo.json, Turbo runs the tasks and Lerna only publishes: turbo.json is the source.
|
|
@@ -550,18 +550,19 @@ export default defineWorkspace({
|
|
|
550
550
|
|
|
551
551
|
Off CI a run reads the remote and does not write it unless its policy names the remote. Give the write token only to the default branch's CI and a read-only one to laptops and pull request jobs: the server's token is the boundary (`docs/security.md` § Cache poisoning). The same holds for `nxCache()`.
|
|
552
552
|
|
|
553
|
-
Every option falls back to the tool's own environment variable, so a self-hosted setup carries over unchanged. A token with no `apiUrl` means Vercel's hosted Remote Cache (`https://vercel.com/api`), exactly as it does for `turbo` — so `npx turbo login && npx turbo link`, then `turboCache()` with `TURBO_TOKEN` / `TURBO_TEAM` set, is the whole hosted setup. Below the environment, in Turbo's order: a Vercel build's `VERCEL_ARTIFACTS_TOKEN` and `VERCEL_ARTIFACTS_OWNER` (the team id, where `TURBO_TOKEN` with a team is not set), then the repo's `.turbo/config.json` (what `turbo link` writes: `apiUrl`, `teamId`, `teamSlug`, `token`), then the root `turbo.json`'s `remoteCache` (`apiUrl`, `teamId`, `teamSlug`), whose `enabled: false` declines unless the options name a cache. With no token the plugin **declines** and the run stays local.
|
|
553
|
+
Every option falls back to the tool's own environment variable, so a self-hosted setup carries over unchanged. A token with no `apiUrl` means Vercel's hosted Remote Cache (`https://vercel.com/api`), exactly as it does for `turbo` — so `npx turbo login && npx turbo link`, then `turboCache()` with `TURBO_TOKEN` / `TURBO_TEAM` set, is the whole hosted setup. Below the environment, in Turbo's order: a Vercel build's `VERCEL_ARTIFACTS_TOKEN` and `VERCEL_ARTIFACTS_OWNER` (the team id, where `TURBO_TOKEN` with a team is not set), then the repo's `.turbo/config.json` (what `turbo link` writes: `apiUrl`, `teamId`, `teamSlug`, `token`), then the root `turbo.json`'s `remoteCache` (`apiUrl`, `teamId`, `teamSlug`), whose `enabled: false` declines unless the options name a cache. A team with no `apiUrl` also means Vercel's. With a URL or a team but no token, the token comes from the OS keychain ([Tokens](#tokens-and-the-os-keychain)); with none of the three the plugin **declines** and the run stays local.
|
|
554
554
|
|
|
555
555
|
| Option | Environment variable | Meaning |
|
|
556
556
|
| ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
557
557
|
| `apiUrl` | `TURBO_API` | base URL of the cache server; default with a token: `https://vercel.com/api`; a `user:pass@` in it is refused |
|
|
558
|
-
| `token` | `TURBO_TOKEN` | Bearer token on every request
|
|
558
|
+
| `token` | `TURBO_TOKEN` | Bearer token on every request; then OIDC, `.turbo/config.json`, the OS keychain |
|
|
559
|
+
| `oidc` | `ACTIONS_ID_TOKEN_REQUEST_URL`, `ACTIONS_ID_TOKEN_REQUEST_TOKEN` (GitHub sets them) | `true` or `{ tokenUrl, revokeUrl, audience, params }` (`TurboOidcOptions`): exchange the GitHub Actions job's OIDC token for a short-lived one ([OIDC](#oidc-on-github-actions)) |
|
|
559
560
|
| `teamId` | `TURBO_TEAMID` | `teamId` query parameter; required with `signatureKey` |
|
|
560
561
|
| `teamSlug` | `TURBO_TEAM` | `slug` query parameter |
|
|
561
562
|
| `signatureKey` | `TURBO_REMOTE_CACHE_SIGNATURE_KEY`, read only under turbo.json's `remoteCache.signature: true` (or `TURBO_SIGNATURE`), as Turbo reads it | HMAC-SHA256 key (≥ 32 bytes, used raw); a download whose tag does not verify is a miss |
|
|
562
563
|
| `timeoutMs` | `TURBO_REMOTE_CACHE_TIMEOUT`, then `remoteCache.timeout` (seconds) | HEAD/GET/POST deadline (default 30 s; 0 none) |
|
|
563
564
|
| `uploadTimeoutMs` | `TURBO_REMOTE_CACHE_UPLOAD_TIMEOUT`, then `remoteCache.uploadTimeout` (seconds) | PUT deadline (default 60 s; 0 none) |
|
|
564
|
-
| `retries` | — | resends of a request answered
|
|
565
|
+
| `retries` | — | resends of a request answered 5xx (not 501) or never connected (default 1, Turbo's); a 429 waits out the limit; 0 turns both off |
|
|
565
566
|
| `preflight` | `TURBO_PREFLIGHT` (1 or 0), then `remoteCache.preflight` | send Turbo's `OPTIONS` preflight before each artifact request and go where its `Location` points (relative to `apiUrl`); the token goes along only when the API's `Access-Control-Allow-Headers` admits `Authorization`; a redirected preflight is not followed (default off) |
|
|
566
567
|
|
|
567
568
|
The signature is Turbo's current scheme (`artifact-signature:v2`: prefix, hash, team id and body, each length-prefixed, under HMAC-SHA256, base64 in `x-artifact-tag`). A signed body is written to a temp file before its tag can be checked, so one past core's artifact ceiling (2 GiB, at zstd's bound) is refused as it passes it, and is a miss.
|
|
@@ -581,26 +582,51 @@ export default defineWorkspace({
|
|
|
581
582
|
})
|
|
582
583
|
```
|
|
583
584
|
|
|
584
|
-
Every option falls back to the tool's own environment variable; with
|
|
585
|
+
Every option falls back to the tool's own environment variable, and a missing token to the OS keychain ([Tokens](#tokens-and-the-os-keychain)); with no server the plugin **declines** and the run stays local.
|
|
585
586
|
|
|
586
|
-
| Option | Environment variable | Meaning
|
|
587
|
-
| ------------- | ------------------------------------------ |
|
|
588
|
-
| `server` | `NX_SELF_HOSTED_REMOTE_CACHE_SERVER` | base URL of the cache server; a `user:pass@` in it is refused
|
|
589
|
-
| `accessToken` | `NX_SELF_HOSTED_REMOTE_CACHE_ACCESS_TOKEN` | Bearer token;
|
|
590
|
-
| `timeoutMs` | — | per-request deadline (default 30 s; a positive number of ms)
|
|
591
|
-
| `retries` | — | resends, as `turboCache()`'s (default 1)
|
|
587
|
+
| Option | Environment variable | Meaning |
|
|
588
|
+
| ------------- | ------------------------------------------ | ----------------------------------------------------------------------------- |
|
|
589
|
+
| `server` | `NX_SELF_HOSTED_REMOTE_CACHE_SERVER` | base URL of the cache server; a `user:pass@` in it is refused |
|
|
590
|
+
| `accessToken` | `NX_SELF_HOSTED_REMOTE_CACHE_ACCESS_TOKEN` | Bearer token, else the OS keychain; `''` (option or variable): an open server |
|
|
591
|
+
| `timeoutMs` | — | per-request deadline (default 30 s; a positive number of ms) |
|
|
592
|
+
| `retries` | — | resends, as `turboCache()`'s (default 1) |
|
|
592
593
|
|
|
593
594
|
The Nx spec has no existence probe, so `has` (the `--dry` prediction; the prefetch pass calls `get`) is a `GET` whose body is cancelled before it answers. The wire carries no producing-task duration, so a remote hit reports none.
|
|
594
595
|
|
|
596
|
+
## OIDC on GitHub Actions
|
|
597
|
+
|
|
598
|
+
`turboCache({ oidc: true })` needs no `TURBO_TOKEN` secret on GitHub Actions: before the first task, when the run's policy uses the remote, it asks GitHub for the job's OIDC token (`ACTIONS_ID_TOKEN_REQUEST_URL`, with `audience` when given), exchanges it at the token endpoint (RFC 8693) for a short-lived cache token, and revokes that (RFC 7009) when the run ends. With Vercel's hosted cache the endpoints, Vercel's OAuth client id and `team_id_or_slug` (the team id, else slug; one is required) are the defaults, as Vercel's own action sends them; `params` adds fields such as `policy_id`. Any other server names its `tokenUrl` (and `revokeUrl`, or none); `params` carries its extras. `revokeUrl: false` skips the revoke.
|
|
599
|
+
|
|
600
|
+
```yaml
|
|
601
|
+
permissions:
|
|
602
|
+
id-token: write
|
|
603
|
+
```
|
|
604
|
+
|
|
605
|
+
- Order: the `token` option, `TURBO_TOKEN`, then OIDC, then `.turbo/config.json`, then the OS keychain.
|
|
606
|
+
- A request is tried three times on a `429`, a `5xx` or no answer, 30 s each. A failed exchange is one warning and a local-only run; a failed revoke is a warning (the token expires on its own). No message quotes a token.
|
|
607
|
+
- A job without `id-token: write` gets no OIDC variables: one warning, and the sources below OIDC answer. A pull request from a fork, and a Dependabot one, never gets an OIDC token.
|
|
608
|
+
- `nxCache()` has no OIDC: Nx's cache spec names no exchange.
|
|
609
|
+
|
|
610
|
+
## Tokens and the OS keychain
|
|
611
|
+
|
|
612
|
+
With no token from an option, a variable or a config file, both plugins read the OS keychain through core's `ctx.secret` (`Bun.secrets`: macOS Keychain, libsecret, Windows Credential Manager), service `@vzn/vx-migrate`, entry named by the endpoint and team: `https://remote.cachely.dev/v8/artifacts#cachely` (`turboCache()`, team id else slug; no `#…` without a team) or `https://cache.example.com/v1/cache` (`nxCache()`). So separate caches never share a token.
|
|
613
|
+
|
|
614
|
+
- Read once per run, only by a run that uses the remote: before its first task, when its policy reads or writes the remote. `--no-cache`, `--cache-policy=remote:`, `--dry`, `vx show` and `vx info` never touch the keychain. Off CI, remote writes are already off by default ([`cli.md` § Cache control](../vx/docs/cli.md#cache-control---cache-policy---no-cache---force)).
|
|
615
|
+
- Missing, on a terminal (stdin and stderr TTYs, `CI` unset): asked once, input hidden — `Token for remote.cachely.dev (cachely), saved to the OS keychain: ` — and saved. Later runs read it without asking.
|
|
616
|
+
- Missing anywhere else (CI, piped), or an empty answer (Ctrl-C stops the run): one warning naming the remote and the fixes (set `TURBO_TOKEN` / `NX_SELF_HOSTED_REMOTE_CACHE_ACCESS_TOKEN`, run in a terminal, or `--cache-policy=remote:` to skip the remote), and the run goes on with the local cache.
|
|
617
|
+
- A keychain token the server refuses (`401` / `403`) is deleted, with the warning, so the next run asks again. A token from an option or a variable is never deleted.
|
|
618
|
+
- No keychain (headless Linux without libsecret) reads as empty; it never breaks a run.
|
|
619
|
+
- The token never reaches a task: the task environment does not pass `TURBO_TOKEN`, `VERCEL_ARTIFACTS_TOKEN` or Nx's variable.
|
|
620
|
+
|
|
595
621
|
## Remote-cache behaviour (both)
|
|
596
622
|
|
|
597
623
|
- A remote error degrades to a **miss** and a warning that names the request, the artifact and the server — `[vx] plugin '@vzn/vx-migrate': upload 32248a2a7c89e241 to https://cache.example.com/v8/artifacts failed: no answer within 60000 ms` — never the token. The run never fails because of the cache.
|
|
598
624
|
- The same failure is said once per run: an unreachable server fails the probe, the download and the upload alike, and the run prints the first and, at its end, `[vx] plugin '@vzn/vx-migrate': 2 more requests failed the same way: Unable to connect. Is the computer able to access the url?` (core does the counting, for every cache plugin).
|
|
599
|
-
- A request answered `
|
|
625
|
+
- A request answered `5xx` (not `501`), or one that never connected (a refused port, an unresolved host), is sent again after 2 s, `retries` times. A `429` pauses every request to that remote for its `Retry-After` (capped at 10 s a wait, 2 s without one), and the request is sent again when the pause ends, for up to 20 s from its first `429`, as Turbo's client does. A spent deadline is not resent: it has already cost its wait.
|
|
600
626
|
- Three outages in a row (no connection, no answer within the deadline, or a `5xx` once the resends are spent) open Turbo's outage breaker: no request is sent for 30 s, each lookup is a miss at once, then one request probes and its answer closes or reopens it. A hit is judged when its body ends, so a server that sends headers and then stalls the body counts the same way. A hung server costs three deadlines, not one per task.
|
|
601
627
|
- A refused token (`401`/`403`) warns **once** and turns the layer off for the rest of the process (a `403` on an upload is a read-only token — Nx's spec, or turborepo-remote-cache's `READ_ONLY` and write-less JWTs: it turns off uploads alone, and reads go on) — including the requests already in flight when the refusal lands, which degrade in silence rather than repeating it (a six-project run printed five identical lines before 2026-09-20).
|
|
602
628
|
- Policy (`--cache-policy=remote:r`, `VX_CACHE_POLICY`, `cache.policy`) is enforced by core, which layers each plugin's remote over the local cache — a read-only token pairs naturally with `remote:r`.
|
|
603
|
-
-
|
|
629
|
+
- Core's policy alone decides reads and writes: the plugins read none of Turbo's or Nx's own switches (`TURBO_CACHE`, `TURBO_REMOTE_CACHE_READ_ONLY`, `NX_SKIP_REMOTE_CACHE`, `NX_DISABLE_REMOTE_CACHE`). A CI that keeps untrusted pull requests off the shared cache sets `cache.policy` or `--cache-policy=local:rw,remote:r` for them.
|
|
604
630
|
|
|
605
631
|
## Testing
|
|
606
632
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx-migrate",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.642",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "Adopt @vzn/vx from Turborepo, Nx or package.json scripts: run the repo unchanged, migrate it to vx.config.ts, and keep your Turbo or Nx remote cache",
|
|
6
6
|
"keywords": [
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
"access": "public"
|
|
41
41
|
},
|
|
42
42
|
"peerDependencies": {
|
|
43
|
-
"@vzn/vx": "^0.0.
|
|
43
|
+
"@vzn/vx": "^0.0.642"
|
|
44
44
|
},
|
|
45
45
|
"repository": {
|
|
46
46
|
"type": "git",
|
package/src/adopt.ts
CHANGED
|
@@ -175,6 +175,21 @@ export function listedPlugins(root: string): string[] {
|
|
|
175
175
|
return [...plugins].sort()
|
|
176
176
|
}
|
|
177
177
|
|
|
178
|
+
/**
|
|
179
|
+
* The `@vzn/vx*` packages a workspace file imports, parsed, not matched (a
|
|
180
|
+
* formatter's quotes hid a match). Each is installed with core: a file an
|
|
181
|
+
* earlier init wrote imports @vzn/vx-lockfile that package.json need not
|
|
182
|
+
* list, and the lock could not find it (M, 2026-10-16).
|
|
183
|
+
*/
|
|
184
|
+
export function importedPackages(text: string): string[] {
|
|
185
|
+
const out = new Set<string>()
|
|
186
|
+
for (const i of new Bun.Transpiler({ loader: 'ts' }).scanImports(text)) {
|
|
187
|
+
const m = /^(@vzn\/vx(?:-[^/]+)?)(?:\/|$)/.exec(i.path)
|
|
188
|
+
if (m !== null) out.add(m[1]!)
|
|
189
|
+
}
|
|
190
|
+
return [...out].sort()
|
|
191
|
+
}
|
|
192
|
+
|
|
178
193
|
/** Install `packages` (at `version`, given one) with the repo's manager; its output only on failure. */
|
|
179
194
|
export function install(
|
|
180
195
|
root: string,
|
package/src/migrate-turbo.ts
CHANGED
|
@@ -24,6 +24,7 @@ import { DOTENV_GLOBS_HEAD, DOTENV_PROBE, DOTENV_PROBE_TOP } from './dotenv-prob
|
|
|
24
24
|
import { adoptedToolNotes } from './workspace-notes.js'
|
|
25
25
|
import { spelledNames } from './spelled-env.js'
|
|
26
26
|
import { sealsConfig } from './sealed-tsconfig.js'
|
|
27
|
+
import { onceReader, tscBuildInfo } from './tsc-buildinfo.js'
|
|
27
28
|
|
|
28
29
|
/** What a task's `npm_package_*` read: the manifest, so a bump reaches them. */
|
|
29
30
|
const MANIFEST_IMPORT = "import pkg from './package.json' with { type: 'json' }"
|
|
@@ -46,6 +47,7 @@ export async function migrateTurbo(
|
|
|
46
47
|
format: MigrationFormat = 'ts',
|
|
47
48
|
): Promise<MigrationPlan> {
|
|
48
49
|
const tracked = await trackedFiles(root)
|
|
50
|
+
const tsc = onceReader()
|
|
49
51
|
const mapping = await mapTurboWorkspace(root, await withRootProject(root, metas), {
|
|
50
52
|
splice: (kind) => [{ raw: `...${PRESET_NAMES[kind]}` }],
|
|
51
53
|
persistentTodo: PERSISTENT_TODO,
|
|
@@ -55,6 +57,7 @@ export async function migrateTurbo(
|
|
|
55
57
|
ownConfig: () => `vx.config.${format}`,
|
|
56
58
|
sourceNames: (dirs) => spelledNames(root, dirs, tracked),
|
|
57
59
|
ignored: (rels) => gitIgnored(root, rels),
|
|
60
|
+
buildInfo: (rel, command) => tscBuildInfo(tsc.read, path.join(root, rel), command),
|
|
58
61
|
})
|
|
59
62
|
// A reader whose `!` hides a committed file under another task's output
|
|
60
63
|
// runs uncached, as under turbo().
|
package/src/migrate.ts
CHANGED
|
@@ -45,6 +45,7 @@ import { exportGraph, nxBin } from './nx/export-graph.js'
|
|
|
45
45
|
import { readNxJson } from './nx/nx-map.js'
|
|
46
46
|
import {
|
|
47
47
|
type AdoptionMode,
|
|
48
|
+
importedPackages,
|
|
48
49
|
install,
|
|
49
50
|
installArgv,
|
|
50
51
|
listedPackages,
|
|
@@ -358,11 +359,8 @@ export async function migrateCmd(args: readonly string[]): Promise<number> {
|
|
|
358
359
|
const finalWorkspace = workspaceText ?? readFileSync(path.join(root, existingWorkspace!), 'utf8')
|
|
359
360
|
// Imports parsed, not matched: a formatter's double quotes hid it, and
|
|
360
361
|
// the file still imported the package removed (owner, 2026-10-16).
|
|
361
|
-
const
|
|
362
|
-
|
|
363
|
-
new Bun.Transpiler({ loader: 'ts' })
|
|
364
|
-
.scanImports(finalWorkspace)
|
|
365
|
-
.some((i) => i.path === '@vzn/vx-migrate')
|
|
362
|
+
const imported = importedPackages(finalWorkspace)
|
|
363
|
+
const needsSelf = runsBins || imported.includes('@vzn/vx-migrate')
|
|
366
364
|
const cacheNote =
|
|
367
365
|
runner === 'vite-task'
|
|
368
366
|
? undefined
|
|
@@ -381,6 +379,7 @@ export async function migrateCmd(args: readonly string[]): Promise<number> {
|
|
|
381
379
|
'@vzn/vx',
|
|
382
380
|
...(needsSelf ? ['@vzn/vx-migrate'] : []),
|
|
383
381
|
...plugins.map((p) => p.pkg),
|
|
382
|
+
...imported,
|
|
384
383
|
]),
|
|
385
384
|
],
|
|
386
385
|
parsed,
|
|
@@ -544,7 +543,14 @@ async function keep(
|
|
|
544
543
|
const declares = before === undefined || extended !== null
|
|
545
544
|
const steps = await prepareRepo(
|
|
546
545
|
root,
|
|
547
|
-
[
|
|
546
|
+
[
|
|
547
|
+
...new Set([
|
|
548
|
+
'@vzn/vx',
|
|
549
|
+
'@vzn/vx-migrate',
|
|
550
|
+
...(declares ? all.map((p) => p.pkg) : []),
|
|
551
|
+
...importedPackages(text),
|
|
552
|
+
]),
|
|
553
|
+
],
|
|
548
554
|
args,
|
|
549
555
|
)
|
|
550
556
|
if (args.json)
|
|
@@ -633,8 +639,8 @@ function lockRow(lock: { code: number; said: string }): ui.FieldRow {
|
|
|
633
639
|
]
|
|
634
640
|
}
|
|
635
641
|
|
|
636
|
-
/** `.vx/.gitignore`'s text, as core writes it: the machine-local cache
|
|
637
|
-
const DOT_VX_GITIGNORE = '/cache\n'
|
|
642
|
+
/** `.vx/.gitignore`'s text, as core writes it: the machine-local cache and restore staging stay out of git. */
|
|
643
|
+
const DOT_VX_GITIGNORE = '/cache\n/tmp\n'
|
|
638
644
|
|
|
639
645
|
// npx and bunx name the package: their bare `vx` is a stranger's npm package.
|
|
640
646
|
const EXEC: Record<PackageManager, string> = {
|
|
@@ -668,18 +674,23 @@ export function adoptionNext(
|
|
|
668
674
|
return `${installArgv(root, pm, pinned).join(' ')} && ${vx} lock && ${run}`
|
|
669
675
|
}
|
|
670
676
|
|
|
671
|
-
/**
|
|
677
|
+
/**
|
|
678
|
+
* `build` when the runner's config names it, else the first task name it
|
|
679
|
+
* names. A key with `:` is an Nx executor (`@angular/build:application`)
|
|
680
|
+
* and one with `#` a Turbo package task: neither is a task to run `--all`.
|
|
681
|
+
*/
|
|
672
682
|
function firstTask(file: string, runner: 'turbo' | 'nx'): string {
|
|
673
|
-
let
|
|
683
|
+
let json: Record<string, unknown> | null
|
|
674
684
|
try {
|
|
675
|
-
|
|
685
|
+
json = Bun.JSONC.parse(readFileSync(file, 'utf8')) as Record<string, unknown> | null
|
|
676
686
|
} catch {
|
|
677
687
|
return 'build'
|
|
678
688
|
}
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
const
|
|
682
|
-
|
|
689
|
+
const field =
|
|
690
|
+
runner === 'turbo' ? (json?.['tasks'] ?? json?.['pipeline']) : json?.['targetDefaults']
|
|
691
|
+
const names = Object.keys(typeof field === 'object' && field !== null ? field : {})
|
|
692
|
+
if (names.includes('build')) return 'build'
|
|
693
|
+
return names.find((n) => !n.includes(':') && !n.includes('#')) ?? 'build'
|
|
683
694
|
}
|
|
684
695
|
|
|
685
696
|
/**
|
package/src/nx/nx-map.ts
CHANGED
|
@@ -563,6 +563,11 @@ export async function mapNxWorkspace(
|
|
|
563
563
|
followNxGraph(t.task, nodeName, meta.name, nxEdges, metaByNode, emittedIds, reached)
|
|
564
564
|
}
|
|
565
565
|
dropUnheldDeps(t, emitted, emittedIds)
|
|
566
|
+
const reader = dependentReaders.get(t)
|
|
567
|
+
if (reader !== undefined && !reachesProducer(t, meta.name, emittedIds, reached)) {
|
|
568
|
+
delete t.task!['cache']
|
|
569
|
+
t.todos.push(orphanReaderTodo(reader.globs, reader.dependsOn))
|
|
570
|
+
}
|
|
566
571
|
}
|
|
567
572
|
const tags = nxTags(nodeName === undefined ? undefined : nodeMap[nodeName])
|
|
568
573
|
projects.push({
|
|
@@ -895,9 +900,7 @@ function buildTask(
|
|
|
895
900
|
const orphanReader =
|
|
896
901
|
inputs.dependentOutputs.length > 0 && !deps.some((d) => !d.includes('#nx-input:'))
|
|
897
902
|
if (orphanReader && cacheWanted && !persistent) {
|
|
898
|
-
todos.push(
|
|
899
|
-
`reads ${inputs.dependentOutputs.map((g) => `\`${g}\``).join(', ')} from the outputs of the tasks it depends on, and no task it depends on is migrated (dependsOn ${JSON.stringify(target.dependsOn ?? [])}) — uncached here, so it never replays output generated from an older input; migrate the producer and depend on it to cache this task`,
|
|
900
|
-
)
|
|
903
|
+
todos.push(orphanReaderTodo(inputs.dependentOutputs, target.dependsOn))
|
|
901
904
|
}
|
|
902
905
|
const cacheEnabled = !persistent && cacheWanted && wild === undefined && !orphanReader
|
|
903
906
|
if (persistent && cacheWanted) {
|
|
@@ -1007,7 +1010,41 @@ function buildTask(
|
|
|
1007
1010
|
task.cache = { outputs }
|
|
1008
1011
|
}
|
|
1009
1012
|
|
|
1010
|
-
|
|
1013
|
+
const generated: GeneratedTask = { name: variant.name, todos, task }
|
|
1014
|
+
if (cacheEnabled && inputs.dependentOutputs.length > 0)
|
|
1015
|
+
dependentReaders.set(generated, { globs: inputs.dependentOutputs, dependsOn: target.dependsOn })
|
|
1016
|
+
return generated
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
/**
|
|
1020
|
+
* The cached readers of `dependentTasksOutputFiles`, judged again once the
|
|
1021
|
+
* graph has its final edges: a `^build` kept because another project builds
|
|
1022
|
+
* reached no producer of this one (nx-examples' codegen, its .NET
|
|
1023
|
+
* ProductsApi no node once @nx/dotnet failed to load), and the key saw
|
|
1024
|
+
* nothing the task reads (Growth, nx-codegen-dropped-producer.sh).
|
|
1025
|
+
*/
|
|
1026
|
+
const dependentReaders = new WeakMap<GeneratedTask, { globs: string[]; dependsOn: unknown }>()
|
|
1027
|
+
|
|
1028
|
+
function orphanReaderTodo(globs: readonly string[], dependsOn: unknown): string {
|
|
1029
|
+
return `reads ${globs.map((g) => `\`${g}\``).join(', ')} from the outputs of the tasks it depends on, and no task it depends on is migrated (dependsOn ${JSON.stringify(dependsOn ?? [])}) — uncached here, so it never replays output generated from an older input; migrate the producer and depend on it to cache this task`
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
/** Whether one of `t`'s final edges reaches a task vx runs. */
|
|
1033
|
+
function reachesProducer(
|
|
1034
|
+
t: GeneratedTask,
|
|
1035
|
+
project: string,
|
|
1036
|
+
emittedIds: ReadonlySet<string>,
|
|
1037
|
+
reached: (name: string) => ReadonlySet<string>,
|
|
1038
|
+
): boolean {
|
|
1039
|
+
const deps = t.task?.['dependsOn']
|
|
1040
|
+
if (!Array.isArray(deps)) return false
|
|
1041
|
+
return deps.some((d) => {
|
|
1042
|
+
if (typeof d !== 'string' || d.includes('*')) return true
|
|
1043
|
+
if (d.includes('#nx-input:')) return false
|
|
1044
|
+
if (d.startsWith('^'))
|
|
1045
|
+
return [...reached(project)].some((p) => emittedIds.has(`${p}#${d.slice(1)}`))
|
|
1046
|
+
return d.includes('#') || emittedIds.has(`${project}#${d}`)
|
|
1047
|
+
})
|
|
1011
1048
|
}
|
|
1012
1049
|
|
|
1013
1050
|
/**
|
package/src/nx-cache/index.ts
CHANGED
|
@@ -14,19 +14,29 @@
|
|
|
14
14
|
import {
|
|
15
15
|
definePlugin,
|
|
16
16
|
type RemoteCacheLayer,
|
|
17
|
+
type RemoteOpenContext,
|
|
17
18
|
type VxPlugin,
|
|
18
19
|
refuseUnknownOptions,
|
|
19
20
|
type PluginOptionKinds,
|
|
20
21
|
} from '@vzn/vx'
|
|
21
22
|
import { deadline } from '../remote-deadline.js'
|
|
22
23
|
import { OutageBreaker } from '../remote-breaker.js'
|
|
23
|
-
import { withRetry } from '../remote-retry.js'
|
|
24
|
-
import {
|
|
24
|
+
import { Cooldown, withRetry } from '../remote-retry.js'
|
|
25
|
+
import {
|
|
26
|
+
headerValueFault,
|
|
27
|
+
type KeychainRemote,
|
|
28
|
+
keychainToken,
|
|
29
|
+
remoteLabel,
|
|
30
|
+
} from '../remote-token.js'
|
|
25
31
|
|
|
26
32
|
export interface NxCacheOptions {
|
|
27
33
|
/** Base URL of the cache server, or `NX_SELF_HOSTED_REMOTE_CACHE_SERVER`. */
|
|
28
34
|
server?: string
|
|
29
|
-
/**
|
|
35
|
+
/**
|
|
36
|
+
* Bearer token, or `NX_SELF_HOSTED_REMOTE_CACHE_ACCESS_TOKEN`, then the OS
|
|
37
|
+
* keychain. `''` (option or variable) is a server that runs open: no
|
|
38
|
+
* token, nothing asked.
|
|
39
|
+
*/
|
|
30
40
|
accessToken?: string
|
|
31
41
|
/** Per-request deadline (default 30 s). */
|
|
32
42
|
timeoutMs?: number
|
|
@@ -36,6 +46,7 @@ export interface NxCacheOptions {
|
|
|
36
46
|
|
|
37
47
|
export interface NxCacheConfig {
|
|
38
48
|
server: string
|
|
49
|
+
/** Undefined: the layer's `open` asks the OS keychain; `''`: an open server. */
|
|
39
50
|
accessToken?: string
|
|
40
51
|
timeoutMs: number
|
|
41
52
|
retries: number
|
|
@@ -46,10 +57,6 @@ export function resolveNxCacheConfig(
|
|
|
46
57
|
options: NxCacheOptions,
|
|
47
58
|
env: Record<string, string | undefined> = Bun.env,
|
|
48
59
|
): NxCacheConfig | undefined {
|
|
49
|
-
// Nx's own off switches (`skipRemoteCache`): a CI that sets one for an
|
|
50
|
-
// untrusted pull request kept Nx off the shared cache and not vx.
|
|
51
|
-
if (env['NX_SKIP_REMOTE_CACHE'] === 'true' || env['NX_DISABLE_REMOTE_CACHE'] === 'true')
|
|
52
|
-
return undefined
|
|
53
60
|
const server = (options.server ?? env['NX_SELF_HOSTED_REMOTE_CACHE_SERVER'])?.replace(/\/+$/, '')
|
|
54
61
|
if (!server) return undefined
|
|
55
62
|
// The URL is printed in every refusal line, so a `user:pass@` in it would
|
|
@@ -62,7 +69,7 @@ export function resolveNxCacheConfig(
|
|
|
62
69
|
)
|
|
63
70
|
}
|
|
64
71
|
const accessToken = options.accessToken ?? env['NX_SELF_HOSTED_REMOTE_CACHE_ACCESS_TOKEN']
|
|
65
|
-
const fault = accessToken ? headerValueFault(accessToken)
|
|
72
|
+
const fault = accessToken === undefined ? null : headerValueFault(accessToken)
|
|
66
73
|
if (fault !== null)
|
|
67
74
|
throw new Error(
|
|
68
75
|
`vx/nx-cache: the access token holds ${fault}, which no HTTP header can carry — check the secret (it is not printed)`,
|
|
@@ -79,7 +86,7 @@ export function resolveNxCacheConfig(
|
|
|
79
86
|
)
|
|
80
87
|
return {
|
|
81
88
|
server,
|
|
82
|
-
...(accessToken ? { accessToken } : {}),
|
|
89
|
+
...(accessToken !== undefined ? { accessToken } : {}),
|
|
83
90
|
timeoutMs: options.timeoutMs ?? 30_000,
|
|
84
91
|
retries,
|
|
85
92
|
}
|
|
@@ -110,12 +117,31 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
110
117
|
*/
|
|
111
118
|
private writesRefused = false
|
|
112
119
|
readonly endpoint: string
|
|
120
|
+
/** The remote's rate limit, which every request of this client waits out. */
|
|
121
|
+
private readonly cooldown = new Cooldown()
|
|
122
|
+
/** Undefined until `open` reads the keychain; a layer never opened (a plan) asks the server nothing. */
|
|
123
|
+
private token: string | undefined
|
|
124
|
+
/** Set by `open` when the token came from the keychain: a refusal deletes it. */
|
|
125
|
+
private forget: (() => Promise<void>) | undefined
|
|
126
|
+
private readonly keychainRemote: KeychainRemote
|
|
113
127
|
constructor(
|
|
114
128
|
private readonly config: NxCacheConfig,
|
|
115
129
|
private readonly fetchImpl: typeof fetch = fetch,
|
|
116
130
|
private readonly wait: (ms: number) => Promise<void> = Bun.sleep,
|
|
117
131
|
) {
|
|
118
132
|
this.endpoint = `${config.server}/v1/cache`
|
|
133
|
+
this.token = config.accessToken
|
|
134
|
+
this.keychainRemote = {
|
|
135
|
+
key: this.endpoint,
|
|
136
|
+
label: remoteLabel(config.server, undefined),
|
|
137
|
+
env: 'NX_SELF_HOSTED_REMOTE_CACHE_ACCESS_TOKEN',
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
async open(ctx: RemoteOpenContext): Promise<void> {
|
|
142
|
+
if (this.token !== undefined) return
|
|
143
|
+
this.token = await keychainToken(ctx, this.keychainRemote)
|
|
144
|
+
this.forget = () => ctx.forgetSecret({ key: this.keychainRemote.key })
|
|
119
145
|
}
|
|
120
146
|
|
|
121
147
|
/**
|
|
@@ -129,8 +155,9 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
129
155
|
hash: string,
|
|
130
156
|
body?: Blob,
|
|
131
157
|
): Promise<Response | undefined> {
|
|
158
|
+
if (this.token === undefined) return undefined
|
|
132
159
|
const headers: Record<string, string> = {}
|
|
133
|
-
if (this.
|
|
160
|
+
if (this.token !== '') headers['Authorization'] = `Bearer ${this.token}`
|
|
134
161
|
if (body !== undefined) {
|
|
135
162
|
headers['Content-Type'] = 'application/octet-stream'
|
|
136
163
|
headers['Content-Length'] = String(body.size)
|
|
@@ -152,6 +179,7 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
152
179
|
}),
|
|
153
180
|
this.config.retries,
|
|
154
181
|
this.wait,
|
|
182
|
+
this.cooldown,
|
|
155
183
|
),
|
|
156
184
|
method === 'GET',
|
|
157
185
|
)
|
|
@@ -167,8 +195,15 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
167
195
|
const first = !this.disabled
|
|
168
196
|
this.disabled = true
|
|
169
197
|
if (!first) return undefined
|
|
198
|
+
const why =
|
|
199
|
+
res.status === 401
|
|
200
|
+
? 'missing or invalid token'
|
|
201
|
+
: 'access forbidden (the token may not read)'
|
|
202
|
+
if (this.forget === undefined)
|
|
203
|
+
throw new Error(`HTTP ${res.status}: ${why}; remote cache off for this run`)
|
|
204
|
+
await this.forget()
|
|
170
205
|
throw new Error(
|
|
171
|
-
`HTTP ${res.status}: ${
|
|
206
|
+
`HTTP ${res.status}: ${why}; removed it from the OS keychain, so the next run asks again; remote cache off for this run`,
|
|
172
207
|
)
|
|
173
208
|
}
|
|
174
209
|
return res
|
package/src/remote-cache.ts
CHANGED
|
@@ -109,7 +109,7 @@ export function remoteCacheNote(
|
|
|
109
109
|
const cache = REMOTE_CACHE[runner]
|
|
110
110
|
if (signal !== undefined) {
|
|
111
111
|
return declared
|
|
112
|
-
? `${cache}(): ${signal}, so vx shares that remote cache (
|
|
112
|
+
? `${cache}(): ${signal}, so vx shares that remote cache (token: the variable, else the OS keychain).`
|
|
113
113
|
: `${signal}: add ${cache}() from @vzn/vx-migrate to its plugins and vx shares that remote cache.`
|
|
114
114
|
}
|
|
115
115
|
return runner === 'nx' && usesNxCloud(root)
|
package/src/remote-retry.ts
CHANGED
|
@@ -1,40 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* When a remote's rate limit ends, shared by every request to that remote:
|
|
3
|
+
* Turbo's `RateLimit` (`turborepo-api-client/src/retry.rs`; tests
|
|
4
|
+
* `waits_for_known_cooldown_before_initial_send`,
|
|
5
|
+
* `scopes_artifact_cooldowns_to_client_team_and_token`). A request that
|
|
6
|
+
* starts inside it waits it out before its first send instead of spending
|
|
7
|
+
* a try on a 429. One per wire instance: a client, its server and token.
|
|
8
|
+
*/
|
|
9
|
+
export class Cooldown {
|
|
10
|
+
until = 0
|
|
11
|
+
constructor(private readonly now: () => number = Date.now) {}
|
|
12
|
+
|
|
13
|
+
/** Ms left of the limit, 0 when there is none. */
|
|
14
|
+
left(): number {
|
|
15
|
+
return Math.max(0, this.until - this.now())
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
extend(ms: number): void {
|
|
19
|
+
this.until = Math.max(this.until, this.now() + ms)
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
1
23
|
/**
|
|
2
24
|
* One request with Turbo's retry rule (`turborepo-api-client/src/retry.rs`):
|
|
3
|
-
* a
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
25
|
+
* a 5xx other than 501, or a request that never connected, is sent again
|
|
26
|
+
* after 2 s, `retries` times. A 429 opens the remote's `cooldown` for its
|
|
27
|
+
* `Retry-After` (capped at 10 s a wait, 2 s without one), and the request
|
|
28
|
+
* is resent once it ends, as often as fits in 20 s from its first 429: a
|
|
29
|
+
* limit longer than one capped wait lost every hit before (Growth's
|
|
30
|
+
* parity run, 2026-10-16). A spent deadline is not retried, unlike
|
|
31
|
+
* Turbo's reads: it has already cost its wait, and a hung server would
|
|
32
|
+
* cost it twice. Every request both wires make is idempotent
|
|
33
|
+
* (content-addressed, and Nx's second write is a 409 the caller takes as
|
|
34
|
+
* done), so a resend applies nothing twice. Shared by `turboCache()` and
|
|
35
|
+
* `nxCache()`.
|
|
11
36
|
*/
|
|
12
37
|
export async function withRetry(
|
|
13
38
|
send: () => Promise<Response>,
|
|
14
39
|
retries: number,
|
|
15
40
|
wait: (ms: number) => Promise<void> = Bun.sleep,
|
|
41
|
+
cooldown: Cooldown = new Cooldown(),
|
|
42
|
+
now: () => number = Date.now,
|
|
16
43
|
): Promise<Response> {
|
|
17
|
-
|
|
44
|
+
let firstLimit: number | undefined
|
|
45
|
+
for (let attempt = 0; ;) {
|
|
46
|
+
const held = cooldown.left()
|
|
47
|
+
if (held > 0) await wait(held)
|
|
18
48
|
let res: Response
|
|
19
49
|
try {
|
|
20
50
|
res = await send()
|
|
21
51
|
} catch (err) {
|
|
22
52
|
if (attempt >= retries || !neverConnected(err)) throw err
|
|
53
|
+
attempt++
|
|
23
54
|
await wait(BACKOFF_MS)
|
|
24
55
|
continue
|
|
25
56
|
}
|
|
57
|
+
if (res.status === 429) {
|
|
58
|
+
firstLimit ??= now()
|
|
59
|
+
const delay = retryAfterMs(res) ?? BACKOFF_MS
|
|
60
|
+
if (retries === 0 || now() + delay - firstLimit > RATE_LIMIT_BUDGET_MS) return res
|
|
61
|
+
cooldown.extend(delay)
|
|
62
|
+
await res.body?.cancel()
|
|
63
|
+
continue
|
|
64
|
+
}
|
|
26
65
|
if (attempt >= retries || !retryable(res.status)) return res
|
|
27
|
-
|
|
66
|
+
attempt++
|
|
28
67
|
await res.body?.cancel()
|
|
29
|
-
await wait(
|
|
68
|
+
await wait(BACKOFF_MS)
|
|
30
69
|
}
|
|
31
70
|
}
|
|
32
71
|
|
|
33
72
|
const BACKOFF_MS = 2_000
|
|
34
73
|
const MAX_RETRY_AFTER_MS = 10_000
|
|
74
|
+
const RATE_LIMIT_BUDGET_MS = 20_000
|
|
35
75
|
|
|
36
76
|
function retryable(status: number): boolean {
|
|
37
|
-
return status
|
|
77
|
+
return status >= 500 && status !== 501
|
|
38
78
|
}
|
|
39
79
|
|
|
40
80
|
/** A refused port or an unresolved host, as Bun's `fetch` codes them (measured on 1.4.2). */
|