@vzn/vx-migrate 0.0.0 → 0.0.485
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +306 -0
- package/index.ts +4 -0
- package/package.json +44 -6
- package/src/adoption-plugin.ts +125 -0
- package/src/bin.ts +18 -0
- package/src/dotenv-probe.ts +86 -0
- package/src/glob-grammar.ts +63 -0
- package/src/index.ts +15 -0
- package/src/mapping-cache.ts +101 -0
- package/src/migrate-nx.ts +154 -0
- package/src/migrate-turbo.ts +400 -0
- package/src/migrate.ts +165 -0
- package/src/nx/export-graph.ts +29 -0
- package/src/nx/index.ts +638 -0
- package/src/nx/nx-deps.ts +272 -0
- package/src/nx/nx-dotenv.ts +106 -0
- package/src/nx/nx-inputs.ts +210 -0
- package/src/nx/nx-map.ts +1430 -0
- package/src/nx/nx-outputs.ts +231 -0
- package/src/nx/nx-upstream.ts +356 -0
- package/src/nx-cache/index.ts +236 -0
- package/src/nx-command.ts +458 -0
- package/src/nx-dotenv.cjs +41 -0
- package/src/nx-env.cjs +152 -0
- package/src/nx-exec.cjs +294 -0
- package/src/paths.ts +36 -0
- package/src/plugin-gaps.ts +48 -0
- package/src/remote-deadline.ts +11 -0
- package/src/remote-retry.ts +52 -0
- package/src/remote-token.ts +18 -0
- package/src/script-command.ts +69 -0
- package/src/shared-outputs.ts +373 -0
- package/src/tracked-outputs.ts +248 -0
- package/src/turbo/index.ts +291 -0
- package/src/turbo/turbo-map.ts +1976 -0
- package/src/turbo-cache/index.ts +661 -0
- package/src/workspace-notes.ts +54 -0
|
@@ -0,0 +1,661 @@
|
|
|
1
|
+
// @vzn/vx-migrate (turboCache) — a vx `cache` plugin that stores artifacts in any
|
|
2
|
+
// server speaking Turborepo's remote cache API (`/v8/artifacts`): Vercel's
|
|
3
|
+
// hosted cache, or a self-hosted implementation of the published OpenAPI
|
|
4
|
+
// spec. The wire is Turbo's; the bytes are vx's own artifacts under vx's
|
|
5
|
+
// own keys, so the server is storage — a Turbo binary cannot read them.
|
|
6
|
+
//
|
|
7
|
+
// Nothing is on by default: declare `turboCache()` in `vx.workspace.ts`
|
|
8
|
+
// (the local store is the floor beneath it) and give it a URL and a token (options, or
|
|
9
|
+
// Turbo's own environment variables and turbo.json's `remoteCache` so a
|
|
10
|
+
// self-hosted setup carries over).
|
|
11
|
+
// With neither the plugin DECLINES and the run stays local.
|
|
12
|
+
//
|
|
13
|
+
// Imports core only through the public `@vzn/vx` specifier.
|
|
14
|
+
import { readFileSync, unlinkSync } from 'node:fs'
|
|
15
|
+
import { unlink } from 'node:fs/promises'
|
|
16
|
+
import { tmpdir } from 'node:os'
|
|
17
|
+
import path from 'node:path'
|
|
18
|
+
import {
|
|
19
|
+
definePlugin,
|
|
20
|
+
LayeredCache,
|
|
21
|
+
type CacheLayer,
|
|
22
|
+
type RemoteCacheLayer,
|
|
23
|
+
type VxPlugin,
|
|
24
|
+
refuseUnknownOptions,
|
|
25
|
+
type PluginOptionKinds,
|
|
26
|
+
} from '@vzn/vx'
|
|
27
|
+
import { deadlineNamed } from '../remote-deadline.js'
|
|
28
|
+
import { withRetry } from '../remote-retry.js'
|
|
29
|
+
import { headerValueFault } from '../remote-token.js'
|
|
30
|
+
|
|
31
|
+
export interface TurboCacheOptions {
|
|
32
|
+
/** Base URL of the cache server (`https://cache.example.com`), or `TURBO_API`; with a token and neither, Vercel's hosted cache, as for `turbo`. */
|
|
33
|
+
apiUrl?: string
|
|
34
|
+
/** Bearer token every request carries, or `TURBO_TOKEN`. */
|
|
35
|
+
token?: string
|
|
36
|
+
/** `teamId` query parameter, or `TURBO_TEAMID`. Required with `signatureKey`. */
|
|
37
|
+
teamId?: string
|
|
38
|
+
/** `slug` query parameter (a team slug), or `TURBO_TEAM`. */
|
|
39
|
+
teamSlug?: string
|
|
40
|
+
/**
|
|
41
|
+
* Sign uploads and verify downloads with Turbo's artifact signature
|
|
42
|
+
* (HMAC-SHA256, `x-artifact-tag`), or `TURBO_REMOTE_CACHE_SIGNATURE_KEY`
|
|
43
|
+
* where turbo.json sets `remoteCache.signature: true`, as Turbo reads it.
|
|
44
|
+
* At least 32 bytes, used raw. A download whose tag does not verify is a
|
|
45
|
+
* miss, never a restore.
|
|
46
|
+
*/
|
|
47
|
+
signatureKey?: string
|
|
48
|
+
/**
|
|
49
|
+
* Per-request deadline for HEAD/GET/POST, or `TURBO_REMOTE_CACHE_TIMEOUT`
|
|
50
|
+
* / turbo.json's `remoteCache.timeout` (whole seconds); default 30 s.
|
|
51
|
+
* 0 is no deadline, as in Turbo.
|
|
52
|
+
*/
|
|
53
|
+
timeoutMs?: number
|
|
54
|
+
/** … and for PUT, or `TURBO_REMOTE_CACHE_UPLOAD_TIMEOUT` / `remoteCache.uploadTimeout`; default 60 s. */
|
|
55
|
+
uploadTimeoutMs?: number
|
|
56
|
+
/** Resends of a request answered 429 / 5xx or never connected (default 1, Turbo's); 0 turns them off. */
|
|
57
|
+
retries?: number
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** turbo.json's `remoteCache` block: the source below Turbo's environment, as in Turbo. */
|
|
61
|
+
export interface TurboJsonRemoteCache {
|
|
62
|
+
apiUrl?: unknown
|
|
63
|
+
signature?: unknown
|
|
64
|
+
teamId?: unknown
|
|
65
|
+
teamSlug?: unknown
|
|
66
|
+
enabled?: unknown
|
|
67
|
+
timeout?: unknown
|
|
68
|
+
uploadTimeout?: unknown
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** `.turbo/config.json`'s fields, under any of the spellings Turbo accepts. */
|
|
72
|
+
export interface TurboLocalConfig {
|
|
73
|
+
apiUrl?: unknown
|
|
74
|
+
teamId?: unknown
|
|
75
|
+
teamSlug?: unknown
|
|
76
|
+
token?: unknown
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export interface TurboCacheConfig {
|
|
80
|
+
apiUrl: string
|
|
81
|
+
token: string
|
|
82
|
+
teamId?: string
|
|
83
|
+
teamSlug?: string
|
|
84
|
+
signatureKey?: string
|
|
85
|
+
timeoutMs: number
|
|
86
|
+
uploadTimeoutMs: number
|
|
87
|
+
retries: number
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Turbo's signature message prefix (`crates/turborepo-cache/src/signature_authentication.rs`). */
|
|
91
|
+
const SIGNATURE_MESSAGE_PREFIX = 'artifact-signature:v2'
|
|
92
|
+
export const MIN_SIGNATURE_KEY_LENGTH = 32
|
|
93
|
+
|
|
94
|
+
// Bun's own hasher and the global `crypto`, not `node:crypto`: importing that
|
|
95
|
+
// module cost every `@vzn/vx-migrate` user 6.6 ms at startup, remote cache
|
|
96
|
+
// or not (G-121).
|
|
97
|
+
function updateLength(mac: Bun.CryptoHasher, byteLength: number): void {
|
|
98
|
+
const len = Buffer.alloc(8)
|
|
99
|
+
len.writeBigUInt64LE(BigInt(byteLength))
|
|
100
|
+
mac.update(len)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* `x-artifact-tag`: base64(HMAC-SHA256(key, fields)) where every field is
|
|
105
|
+
* prefixed with its byte length as a little-endian u64 — prefix, hash,
|
|
106
|
+
* team id, body — exactly as Turbo generates and verifies it. The body is
|
|
107
|
+
* read as a stream, so a file-backed Blob is signed without being held.
|
|
108
|
+
*/
|
|
109
|
+
export async function artifactTag(
|
|
110
|
+
key: Uint8Array,
|
|
111
|
+
hash: string,
|
|
112
|
+
teamId: string,
|
|
113
|
+
body: Blob,
|
|
114
|
+
): Promise<string> {
|
|
115
|
+
const mac = new Bun.CryptoHasher('sha256', key)
|
|
116
|
+
for (const field of [
|
|
117
|
+
Buffer.from(SIGNATURE_MESSAGE_PREFIX),
|
|
118
|
+
Buffer.from(hash),
|
|
119
|
+
Buffer.from(teamId),
|
|
120
|
+
]) {
|
|
121
|
+
updateLength(mac, field.byteLength)
|
|
122
|
+
mac.update(field)
|
|
123
|
+
}
|
|
124
|
+
updateLength(mac, body.size)
|
|
125
|
+
for await (const chunk of body.stream()) mac.update(chunk)
|
|
126
|
+
return mac.digest('base64')
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* The temps not yet removed. Core's signal exit is `process.exit`, which
|
|
131
|
+
* awaits nothing, so a Ctrl-C between the download and the last read left
|
|
132
|
+
* the artifact's temp behind (item 848); the process's `exit` event
|
|
133
|
+
* removes what is still listed.
|
|
134
|
+
*/
|
|
135
|
+
const liveTemps = new Set<string>()
|
|
136
|
+
let exitHooked = false
|
|
137
|
+
function trackTemp(file: string): void {
|
|
138
|
+
liveTemps.add(file)
|
|
139
|
+
if (exitHooked) return
|
|
140
|
+
exitHooked = true
|
|
141
|
+
process.on('exit', () => {
|
|
142
|
+
for (const f of liveTemps) {
|
|
143
|
+
try {
|
|
144
|
+
unlinkSync(f)
|
|
145
|
+
} catch {
|
|
146
|
+
// never written, or gone
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
})
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
async function removeTemp(file: string): Promise<void> {
|
|
153
|
+
// Listed until the unlink lands: an exit in between would otherwise find
|
|
154
|
+
// nothing to remove and leave the temp (item 868).
|
|
155
|
+
await unlink(file).catch(() => undefined)
|
|
156
|
+
liveTemps.delete(file)
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** The verified temp as a body that deletes the temp once read to the end or cancelled. */
|
|
160
|
+
function unlinkingStream(file: string): ReadableStream<Uint8Array> {
|
|
161
|
+
const reader = Bun.file(file).stream().getReader()
|
|
162
|
+
const remove = () => removeTemp(file)
|
|
163
|
+
return new ReadableStream<Uint8Array>({
|
|
164
|
+
async pull(controller) {
|
|
165
|
+
try {
|
|
166
|
+
const next = await reader.read()
|
|
167
|
+
if (next.done) {
|
|
168
|
+
await remove()
|
|
169
|
+
controller.close()
|
|
170
|
+
} else {
|
|
171
|
+
controller.enqueue(next.value)
|
|
172
|
+
}
|
|
173
|
+
} catch (err) {
|
|
174
|
+
await remove()
|
|
175
|
+
controller.error(err)
|
|
176
|
+
}
|
|
177
|
+
},
|
|
178
|
+
async cancel(reason) {
|
|
179
|
+
await reader.cancel(reason)
|
|
180
|
+
await remove()
|
|
181
|
+
},
|
|
182
|
+
})
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function tagsEqual(expected: string, actual: string): boolean {
|
|
186
|
+
const a = Buffer.from(expected, 'base64')
|
|
187
|
+
const b = Buffer.from(actual, 'base64')
|
|
188
|
+
if (a.byteLength !== b.byteLength) return false
|
|
189
|
+
// Constant time over the bytes, as timingSafeEqual is.
|
|
190
|
+
let diff = 0
|
|
191
|
+
for (let i = 0; i < a.byteLength; i++) diff |= a[i]! ^ b[i]!
|
|
192
|
+
return diff === 0
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** Where `turbo` itself sends a token with no `apiUrl`: Vercel's hosted Remote Cache. */
|
|
196
|
+
export const VERCEL_API = 'https://vercel.com/api'
|
|
197
|
+
|
|
198
|
+
/** Resolve options over Turbo's environment; `undefined` = not configured. */
|
|
199
|
+
export function resolveTurboCacheConfig(
|
|
200
|
+
options: TurboCacheOptions,
|
|
201
|
+
env: Record<string, string | undefined> = Bun.env,
|
|
202
|
+
file: TurboJsonRemoteCache = {},
|
|
203
|
+
/** The repo's `.turbo/config.json` (`turbo link` writes it): below the environment, above turbo.json. */
|
|
204
|
+
local: TurboLocalConfig = {},
|
|
205
|
+
): TurboCacheConfig | undefined {
|
|
206
|
+
// `enabled: false` turns Turbo's remote cache off whatever the env says;
|
|
207
|
+
// options name a cache for vx alone, and win.
|
|
208
|
+
if (file.enabled === false && options.apiUrl === undefined && options.token === undefined)
|
|
209
|
+
return undefined
|
|
210
|
+
const fromFile = (v: unknown): string | undefined => (typeof v === 'string' ? v : undefined)
|
|
211
|
+
// A Vercel build hands Turbo its cache as VERCEL_ARTIFACTS_TOKEN and
|
|
212
|
+
// _OWNER (the team id), taken where TURBO_TOKEN with a team is not set
|
|
213
|
+
// (Turbo's `override_env.rs`); unread, turboCache() declined on Vercel.
|
|
214
|
+
const turboPair = env['TURBO_TOKEN'] && (env['TURBO_TEAMID'] || env['TURBO_TEAM'])
|
|
215
|
+
const vercel =
|
|
216
|
+
!turboPair && env['VERCEL_ARTIFACTS_TOKEN'] && env['VERCEL_ARTIFACTS_OWNER']
|
|
217
|
+
? { token: env['VERCEL_ARTIFACTS_TOKEN'], teamId: env['VERCEL_ARTIFACTS_OWNER'] }
|
|
218
|
+
: { token: undefined, teamId: turboPair ? undefined : env['VERCEL_ARTIFACTS_OWNER'] }
|
|
219
|
+
const token = options.token ?? env['TURBO_TOKEN'] ?? vercel.token ?? fromFile(local.token)
|
|
220
|
+
// Turbo's own default when a token is set and no `apiUrl` is: Vercel's
|
|
221
|
+
// hosted Remote Cache. A token alone is a configured cache, as it is for
|
|
222
|
+
// `turbo`; no token at all is the declined, local run.
|
|
223
|
+
const apiUrl = (
|
|
224
|
+
options.apiUrl ??
|
|
225
|
+
env['TURBO_API'] ??
|
|
226
|
+
fromFile(local.apiUrl) ??
|
|
227
|
+
fromFile(file.apiUrl) ??
|
|
228
|
+
(token ? VERCEL_API : undefined)
|
|
229
|
+
)?.replace(/\/+$/, '')
|
|
230
|
+
if (!apiUrl || !token) return undefined
|
|
231
|
+
// The URL is printed in every refusal line, so a `user:pass@` in it would
|
|
232
|
+
// leak to the log. Credentials go in the token.
|
|
233
|
+
if (URL.canParse(apiUrl)) {
|
|
234
|
+
const u = new URL(apiUrl)
|
|
235
|
+
if (u.username !== '' || u.password !== '')
|
|
236
|
+
throw new Error(
|
|
237
|
+
'vx/turbo-cache: apiUrl carries credentials (user:pass@); pass them as the token instead',
|
|
238
|
+
)
|
|
239
|
+
}
|
|
240
|
+
const fault = headerValueFault(token)
|
|
241
|
+
if (fault !== null)
|
|
242
|
+
throw new Error(
|
|
243
|
+
`vx/turbo-cache: the token holds ${fault}, which no HTTP header can carry — check the secret (it is not printed)`,
|
|
244
|
+
)
|
|
245
|
+
const teamId =
|
|
246
|
+
options.teamId ??
|
|
247
|
+
env['TURBO_TEAMID'] ??
|
|
248
|
+
vercel.teamId ??
|
|
249
|
+
fromFile(local.teamId) ??
|
|
250
|
+
fromFile(file.teamId)
|
|
251
|
+
const teamSlug =
|
|
252
|
+
options.teamSlug ?? env['TURBO_TEAM'] ?? fromFile(local.teamSlug) ?? fromFile(file.teamSlug)
|
|
253
|
+
// Turbo signs only under `remoteCache.signature: true`; the env key alone
|
|
254
|
+
// signs nothing there, and a short one here refused the whole cache.
|
|
255
|
+
// `TURBO_SIGNATURE` (1/true, 0/false) sits above turbo.json, as in Turbo.
|
|
256
|
+
const signEnv = env['TURBO_SIGNATURE']
|
|
257
|
+
let signing = file.signature === true
|
|
258
|
+
if (signEnv !== undefined && signEnv !== '') {
|
|
259
|
+
if (!/^(1|0|true|false)$/.test(signEnv))
|
|
260
|
+
throw new Error(
|
|
261
|
+
`vx/turbo-cache: TURBO_SIGNATURE should be 1 or 0, got ${JSON.stringify(signEnv)}`,
|
|
262
|
+
)
|
|
263
|
+
signing = signEnv === '1' || signEnv === 'true'
|
|
264
|
+
}
|
|
265
|
+
const signatureKey =
|
|
266
|
+
options.signatureKey ?? (signing ? env['TURBO_REMOTE_CACHE_SIGNATURE_KEY'] : undefined)
|
|
267
|
+
if (signatureKey !== undefined) {
|
|
268
|
+
if (Buffer.byteLength(signatureKey) < MIN_SIGNATURE_KEY_LENGTH) {
|
|
269
|
+
throw new Error(
|
|
270
|
+
`vx/turbo-cache: signatureKey must be at least ${MIN_SIGNATURE_KEY_LENGTH} bytes (Turbo's minimum)`,
|
|
271
|
+
)
|
|
272
|
+
}
|
|
273
|
+
if (!teamId)
|
|
274
|
+
throw new Error(
|
|
275
|
+
'vx/turbo-cache: signatureKey needs teamId — the team id is part of the signed message',
|
|
276
|
+
)
|
|
277
|
+
}
|
|
278
|
+
const retries = options.retries ?? 1
|
|
279
|
+
// A NaN never reaches the bound, and the request would be resent forever.
|
|
280
|
+
if (!Number.isInteger(retries) || retries < 0)
|
|
281
|
+
throw new Error(`vx/turbo-cache: retries must be a whole number ≥ 0, got ${retries}`)
|
|
282
|
+
// 0 is no deadline, as Turbo reads it; a negative or non-number one made
|
|
283
|
+
// AbortSignal.timeout throw on every request.
|
|
284
|
+
for (const name of ['timeoutMs', 'uploadTimeoutMs'] as const) {
|
|
285
|
+
const ms = options[name]
|
|
286
|
+
if (ms !== undefined && !(typeof ms === 'number' && Number.isFinite(ms) && ms >= 0))
|
|
287
|
+
throw new Error(`vx/turbo-cache: ${name} must be ms ≥ 0 (0: none), got ${JSON.stringify(ms)}`)
|
|
288
|
+
}
|
|
289
|
+
return {
|
|
290
|
+
apiUrl,
|
|
291
|
+
token,
|
|
292
|
+
...(teamId ? { teamId } : {}),
|
|
293
|
+
...(teamSlug ? { teamSlug } : {}),
|
|
294
|
+
...(signatureKey !== undefined ? { signatureKey } : {}),
|
|
295
|
+
timeoutMs:
|
|
296
|
+
options.timeoutMs ??
|
|
297
|
+
seconds(env['TURBO_REMOTE_CACHE_TIMEOUT'], 'TURBO_REMOTE_CACHE_TIMEOUT') ??
|
|
298
|
+
seconds(file.timeout, 'remoteCache.timeout') ??
|
|
299
|
+
30_000,
|
|
300
|
+
uploadTimeoutMs:
|
|
301
|
+
options.uploadTimeoutMs ??
|
|
302
|
+
seconds(env['TURBO_REMOTE_CACHE_UPLOAD_TIMEOUT'], 'TURBO_REMOTE_CACHE_UPLOAD_TIMEOUT') ??
|
|
303
|
+
seconds(file.uploadTimeout, 'remoteCache.uploadTimeout') ??
|
|
304
|
+
60_000,
|
|
305
|
+
retries,
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* Turbo's timeout, whole seconds, as ms; `undefined` when unset. A value
|
|
311
|
+
* Turbo would refuse is refused here rather than read as some other wait.
|
|
312
|
+
*/
|
|
313
|
+
function seconds(v: unknown, name: string): number | undefined {
|
|
314
|
+
if (v === undefined || v === null || v === '') return undefined
|
|
315
|
+
const n = typeof v === 'string' ? Number(v) : v
|
|
316
|
+
if (typeof n !== 'number' || !Number.isInteger(n) || n < 0)
|
|
317
|
+
throw new Error(`vx/turbo-cache: ${name} must be whole seconds ≥ 0, got ${JSON.stringify(v)}`)
|
|
318
|
+
return n * 1000
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/** A response's text, or null (the rest cancelled) once it passes `max` bytes. */
|
|
322
|
+
async function boundedText(res: Response, max: number): Promise<string | null> {
|
|
323
|
+
if (Number(res.headers.get('content-length') ?? 0) > max) {
|
|
324
|
+
await res.body?.cancel()
|
|
325
|
+
return null
|
|
326
|
+
}
|
|
327
|
+
const reader = res.body?.getReader()
|
|
328
|
+
if (reader === undefined) return ''
|
|
329
|
+
const parts: Uint8Array[] = []
|
|
330
|
+
let n = 0
|
|
331
|
+
for (;;) {
|
|
332
|
+
const { done, value } = await reader.read()
|
|
333
|
+
if (done) break
|
|
334
|
+
n += value.byteLength
|
|
335
|
+
if (n > max) {
|
|
336
|
+
await reader.cancel()
|
|
337
|
+
return null
|
|
338
|
+
}
|
|
339
|
+
parts.push(value)
|
|
340
|
+
}
|
|
341
|
+
return Buffer.concat(parts).toString('utf8')
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* A signed body is written whole before its tag can be checked, so a
|
|
346
|
+
* server that never ended one filled the temp's disk (L-8). Core refuses
|
|
347
|
+
* an artifact past its 2 GiB ceiling (zstd's bound on it here) anyway, so
|
|
348
|
+
* nothing larger is ever worth keeping.
|
|
349
|
+
*/
|
|
350
|
+
const MAX_SIGNED_BODY = 2 * 1024 ** 3 + ((2 * 1024 ** 3) >> 8) + 64 * 1024
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* The seam implementation: `has` is HEAD, `hasMany` is the batch query,
|
|
354
|
+
* `get`/`put` carry `x-artifact-duration` (and the tag when signing). An
|
|
355
|
+
* auth failure (401/403) throws ONCE — LayeredCache reports it — and then
|
|
356
|
+
* turns the layer off for the rest of the process, so a bad token costs one
|
|
357
|
+
* line, not one per task — the requests already in flight when it lands
|
|
358
|
+
* degrade in silence rather than repeating it.
|
|
359
|
+
*
|
|
360
|
+
* Bodies stream both ways: `put` sends the Blob core hands it, `get`
|
|
361
|
+
* returns the `fetch` Response. With a signature key the tag must verify
|
|
362
|
+
* before core sees a byte, so a signed download lands in a temp under
|
|
363
|
+
* `tempDir` first and is handed over only once it has.
|
|
364
|
+
*/
|
|
365
|
+
export class TurboRemoteCache implements RemoteCacheLayer {
|
|
366
|
+
private disabled = false
|
|
367
|
+
private readonly key: Uint8Array | undefined
|
|
368
|
+
readonly endpoint: string
|
|
369
|
+
constructor(
|
|
370
|
+
private readonly config: TurboCacheConfig,
|
|
371
|
+
private readonly fetchImpl: typeof fetch = fetch,
|
|
372
|
+
private readonly tempDir: string = tmpdir(),
|
|
373
|
+
private readonly wait: (ms: number) => Promise<void> = Bun.sleep,
|
|
374
|
+
/** The most a signed body may hold before its tag is checked (a test lowers it). */
|
|
375
|
+
private readonly maxSignedBody: number = MAX_SIGNED_BODY,
|
|
376
|
+
) {
|
|
377
|
+
this.key = config.signatureKey === undefined ? undefined : Buffer.from(config.signatureKey)
|
|
378
|
+
this.endpoint = `${config.apiUrl}/v8/artifacts`
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
private url(pathname: string): string {
|
|
382
|
+
const u = new URL(`${this.config.apiUrl}/v8/artifacts${pathname}`)
|
|
383
|
+
// Turbo sends a team id only in Vercel's `team_` form (its API client's
|
|
384
|
+
// `add_team_params`); a personal account's owner id is not one, and the
|
|
385
|
+
// signature, which folds the id as given, is unaffected.
|
|
386
|
+
if (this.config.teamId?.startsWith('team_')) u.searchParams.set('teamId', this.config.teamId)
|
|
387
|
+
if (this.config.teamSlug) u.searchParams.set('slug', this.config.teamSlug)
|
|
388
|
+
return u.toString()
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
private headers(extra: Record<string, string> = {}): Record<string, string> {
|
|
392
|
+
return {
|
|
393
|
+
Authorization: `Bearer ${this.config.token}`,
|
|
394
|
+
'x-artifact-client-interactive': process.stdout.isTTY ? '1' : '0',
|
|
395
|
+
...extra,
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* `undefined` = the token was refused and the refusal is ALREADY
|
|
401
|
+
* reported, so the caller degrades to its miss value in silence. Only the
|
|
402
|
+
* first refusal throws, and a run's calls are concurrent: six projects
|
|
403
|
+
* under a bad token printed five identical lines before this (2026-09-20).
|
|
404
|
+
*/
|
|
405
|
+
private async request(
|
|
406
|
+
method: string,
|
|
407
|
+
pathname: string,
|
|
408
|
+
init: { body?: Blob | string; headers?: Record<string, string>; timeoutMs?: number } = {},
|
|
409
|
+
): Promise<Response | undefined> {
|
|
410
|
+
const timeoutMs = init.timeoutMs ?? this.config.timeoutMs
|
|
411
|
+
const res = await withRetry(
|
|
412
|
+
() =>
|
|
413
|
+
this.fetchImpl(this.url(pathname), {
|
|
414
|
+
method,
|
|
415
|
+
headers: this.headers(init.headers),
|
|
416
|
+
...(init.body === undefined ? {} : { body: init.body }),
|
|
417
|
+
// 0 is Turbo's "no deadline".
|
|
418
|
+
...(timeoutMs === 0 ? {} : { signal: AbortSignal.timeout(timeoutMs) }),
|
|
419
|
+
}),
|
|
420
|
+
this.config.retries,
|
|
421
|
+
this.wait,
|
|
422
|
+
).catch((err: unknown) => {
|
|
423
|
+
throw deadlineNamed(err, timeoutMs)
|
|
424
|
+
})
|
|
425
|
+
if (res.status === 401 || res.status === 403) {
|
|
426
|
+
const first = !this.disabled
|
|
427
|
+
this.disabled = true
|
|
428
|
+
if (!first) return undefined
|
|
429
|
+
throw new Error(`HTTP ${res.status}: the token was refused; remote cache off for this run`)
|
|
430
|
+
}
|
|
431
|
+
return res
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
async has(hash: string): Promise<boolean> {
|
|
435
|
+
if (this.disabled) return false
|
|
436
|
+
const res = await this.request('HEAD', `/${hash}`)
|
|
437
|
+
if (res === undefined) return false
|
|
438
|
+
if (res.status === 200) return true
|
|
439
|
+
if (res.status === 404) return false
|
|
440
|
+
throw new Error(`HTTP ${res.status}`)
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
async hasMany(hashes: readonly string[]): Promise<Set<string> | null> {
|
|
444
|
+
if (this.disabled || hashes.length === 0) return this.disabled ? new Set() : null
|
|
445
|
+
const res = await this.request('POST', '', {
|
|
446
|
+
body: JSON.stringify({ hashes }),
|
|
447
|
+
headers: { 'Content-Type': 'application/json' },
|
|
448
|
+
})
|
|
449
|
+
if (res === undefined) return new Set()
|
|
450
|
+
if (res.status !== 200) return null
|
|
451
|
+
// An answer per hash is a few hundred bytes of ArtifactInfo; a reply
|
|
452
|
+
// past this bound is no answer, read no further than the bound (L-9).
|
|
453
|
+
const bound = hashes.length * 4096 + 64 * 1024
|
|
454
|
+
const text = await boundedText(res, bound)
|
|
455
|
+
if (text === null) return null
|
|
456
|
+
const info = JSON.parse(text) as Record<string, unknown>
|
|
457
|
+
// Each hash answers ArtifactInfo, null, or `{ error: { message } }`; an
|
|
458
|
+
// error entry is not a stored artifact, and it was counted as one (a GET
|
|
459
|
+
// spent on a 404, item 929).
|
|
460
|
+
return new Set(
|
|
461
|
+
hashes.filter((h) => {
|
|
462
|
+
const v = info[h]
|
|
463
|
+
return typeof v === 'object' && v !== null && !('error' in v)
|
|
464
|
+
}),
|
|
465
|
+
)
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
async get(hash: string): Promise<{ body: Response; durationMs: number | undefined } | null> {
|
|
469
|
+
if (this.disabled) return null
|
|
470
|
+
// A gateway that keys binary media on Accept base64-encodes a `*/*`
|
|
471
|
+
// response, and every hit read as a corrupt artifact (nx#33092's class).
|
|
472
|
+
const res = await this.request('GET', `/${hash}`, {
|
|
473
|
+
headers: { Accept: 'application/octet-stream' },
|
|
474
|
+
})
|
|
475
|
+
if (res === undefined) return null
|
|
476
|
+
if (res.status === 404) return null
|
|
477
|
+
if (res.status !== 200) throw new Error(`HTTP ${res.status}`)
|
|
478
|
+
const duration = Number(res.headers.get('x-artifact-duration'))
|
|
479
|
+
const durationMs = Number.isFinite(duration) && duration > 0 ? duration : undefined
|
|
480
|
+
if (this.key === undefined) return { body: res, durationMs }
|
|
481
|
+
return { body: await this.verified(this.key, hash, res), durationMs }
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* The tag's message carries the body's length BEFORE its bytes, and a
|
|
486
|
+
* chunked response declares no length, so the body is written to the temp
|
|
487
|
+
* first and signed from there: two passes over a file, never one in memory.
|
|
488
|
+
*/
|
|
489
|
+
private async verified(key: Uint8Array, hash: string, res: Response): Promise<Response> {
|
|
490
|
+
const refused = () => new Error('the artifact signature did not verify — treated as a miss')
|
|
491
|
+
const tag = res.headers.get('x-artifact-tag')
|
|
492
|
+
if (tag === null) {
|
|
493
|
+
await res.body?.cancel()
|
|
494
|
+
throw refused()
|
|
495
|
+
}
|
|
496
|
+
const temp = path.join(this.tempDir, `vx-turbo-${hash}-${crypto.randomUUID()}`)
|
|
497
|
+
trackTemp(temp)
|
|
498
|
+
try {
|
|
499
|
+
const max = this.maxSignedBody
|
|
500
|
+
const past = () => new Error(`the signed artifact runs past ${max} bytes — treated as a miss`)
|
|
501
|
+
if (Number(res.headers.get('content-length') ?? 0) > max) throw past()
|
|
502
|
+
let n = 0
|
|
503
|
+
const counted = res.body?.pipeThrough(
|
|
504
|
+
new TransformStream<Uint8Array, Uint8Array>({
|
|
505
|
+
transform(chunk, controller) {
|
|
506
|
+
n += chunk.byteLength
|
|
507
|
+
if (n > max) controller.error(past())
|
|
508
|
+
else controller.enqueue(chunk)
|
|
509
|
+
},
|
|
510
|
+
}),
|
|
511
|
+
)
|
|
512
|
+
await Bun.write(temp, new Response(counted ?? null))
|
|
513
|
+
const expected = await artifactTag(key, hash, this.config.teamId ?? '', Bun.file(temp))
|
|
514
|
+
if (!tagsEqual(expected, tag)) throw refused()
|
|
515
|
+
} catch (err) {
|
|
516
|
+
await removeTemp(temp)
|
|
517
|
+
throw err
|
|
518
|
+
}
|
|
519
|
+
return new Response(unlinkingStream(temp))
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
async put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void> {
|
|
523
|
+
if (this.disabled) return
|
|
524
|
+
const headers: Record<string, string> = {
|
|
525
|
+
'Content-Type': 'application/octet-stream',
|
|
526
|
+
'Content-Length': String(body.size),
|
|
527
|
+
'x-artifact-duration': String(Math.max(0, Math.round(meta.durationMs))),
|
|
528
|
+
}
|
|
529
|
+
if (this.key !== undefined) {
|
|
530
|
+
headers['x-artifact-tag'] = await artifactTag(this.key, hash, this.config.teamId ?? '', body)
|
|
531
|
+
}
|
|
532
|
+
const res = await this.request('PUT', `/${hash}`, {
|
|
533
|
+
body,
|
|
534
|
+
headers,
|
|
535
|
+
timeoutMs: this.config.uploadTimeoutMs,
|
|
536
|
+
})
|
|
537
|
+
if (res === undefined) return
|
|
538
|
+
if (res.status !== 200 && res.status !== 202) throw new Error(`HTTP ${res.status}`)
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/**
|
|
543
|
+
* What Turbo's own switches leave of the remote cache: `TURBO_CACHE`
|
|
544
|
+
* (`--cache`'s syntax, where an omitted source is off) and
|
|
545
|
+
* `TURBO_REMOTE_CACHE_READ_ONLY`. A CI that sets either for an untrusted
|
|
546
|
+
* pull request kept Turbo from writing the shared cache and not vx. They
|
|
547
|
+
* only narrow what `--cache` allows, never widen it.
|
|
548
|
+
*/
|
|
549
|
+
export function turboRemoteAccess(env: Record<string, string | undefined>): {
|
|
550
|
+
read: boolean
|
|
551
|
+
write: boolean
|
|
552
|
+
} {
|
|
553
|
+
let read = true
|
|
554
|
+
let write = true
|
|
555
|
+
const spec = env['TURBO_CACHE']
|
|
556
|
+
if (spec !== undefined && spec !== '') {
|
|
557
|
+
read = write = false
|
|
558
|
+
for (const part of spec.split(',')) {
|
|
559
|
+
const m = /^(local|remote):([rw]*)$/.exec(part.trim())
|
|
560
|
+
if (m === null)
|
|
561
|
+
throw new Error(
|
|
562
|
+
`vx/turbo-cache: TURBO_CACHE ${JSON.stringify(spec)} is not --cache's syntax (e.g. local:rw,remote:r)`,
|
|
563
|
+
)
|
|
564
|
+
if (m[1] !== 'remote') continue
|
|
565
|
+
read = m[2]!.includes('r')
|
|
566
|
+
write = m[2]!.includes('w')
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
if (/^(1|true)$/i.test(env['TURBO_REMOTE_CACHE_READ_ONLY'] ?? '')) write = false
|
|
570
|
+
return { read, write }
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* The repo's `.turbo/config.json`, by Turbo's field names and their
|
|
575
|
+
* aliases (`apiurl`, `ApiUrl`, `APIURL`, …); `{}` when absent or unreadable.
|
|
576
|
+
*/
|
|
577
|
+
function localConfigOf(root: string): TurboLocalConfig {
|
|
578
|
+
let raw: unknown
|
|
579
|
+
try {
|
|
580
|
+
raw = JSON.parse(readFileSync(path.join(root, '.turbo', 'config.json'), 'utf8'))
|
|
581
|
+
} catch {
|
|
582
|
+
return {}
|
|
583
|
+
}
|
|
584
|
+
if (typeof raw !== 'object' || raw === null) return {}
|
|
585
|
+
const byLower = new Map(Object.entries(raw).map(([k, v]) => [k.toLowerCase(), v]))
|
|
586
|
+
return {
|
|
587
|
+
apiUrl: byLower.get('apiurl'),
|
|
588
|
+
teamId: byLower.get('teamid'),
|
|
589
|
+
teamSlug: byLower.get('teamslug'),
|
|
590
|
+
token: byLower.get('token'),
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
/** The root turbo.json's (or turbo.jsonc's) `remoteCache`; `{}` when there is none or it does not parse. */
|
|
595
|
+
function remoteCacheOf(root: string): TurboJsonRemoteCache {
|
|
596
|
+
for (const name of ['turbo.json', 'turbo.jsonc']) {
|
|
597
|
+
let text: string
|
|
598
|
+
try {
|
|
599
|
+
text = readFileSync(path.join(root, name), 'utf8')
|
|
600
|
+
} catch {
|
|
601
|
+
continue
|
|
602
|
+
}
|
|
603
|
+
try {
|
|
604
|
+
const rc = (Bun.JSONC.parse(text) as { remoteCache?: unknown } | null)?.remoteCache
|
|
605
|
+
return typeof rc === 'object' && rc !== null ? (rc as TurboJsonRemoteCache) : {}
|
|
606
|
+
} catch {
|
|
607
|
+
return {}
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
return {}
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
/** Each option `TurboCacheOptions` names, with its kind: derived from the type, so the two cannot drift. */
|
|
614
|
+
const TURBO_CACHE_KEYS: PluginOptionKinds<TurboCacheOptions> = {
|
|
615
|
+
apiUrl: 'string',
|
|
616
|
+
token: 'string',
|
|
617
|
+
teamId: 'string',
|
|
618
|
+
teamSlug: 'string',
|
|
619
|
+
signatureKey: 'string',
|
|
620
|
+
timeoutMs: 'number',
|
|
621
|
+
uploadTimeoutMs: 'number',
|
|
622
|
+
retries: 'number',
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/**
|
|
626
|
+
* Declare in `vx.workspace.ts`; the local store stays the floor beneath it:
|
|
627
|
+
*
|
|
628
|
+
* ```ts
|
|
629
|
+
* plugins: [turboCache({ apiUrl: 'https://cache.example.com', token: process.env.CACHE_TOKEN })]
|
|
630
|
+
* ```
|
|
631
|
+
*
|
|
632
|
+
* Declines without a URL and a token, so it is safe to leave declared.
|
|
633
|
+
*/
|
|
634
|
+
export function turboCache(options: TurboCacheOptions = {}): VxPlugin {
|
|
635
|
+
refuseUnknownOptions('turboCache()', options, TURBO_CACHE_KEYS)
|
|
636
|
+
return definePlugin(import.meta, {
|
|
637
|
+
cache(ctx): CacheLayer | undefined {
|
|
638
|
+
const config = resolveTurboCacheConfig(
|
|
639
|
+
options,
|
|
640
|
+
Bun.env,
|
|
641
|
+
remoteCacheOf(ctx.workspaceRoot),
|
|
642
|
+
localConfigOf(ctx.workspaceRoot),
|
|
643
|
+
)
|
|
644
|
+
if (config === undefined) return undefined
|
|
645
|
+
const remote = turboRemoteAccess(Bun.env)
|
|
646
|
+
const policy = {
|
|
647
|
+
...ctx.policy,
|
|
648
|
+
remoteRead: ctx.policy.remoteRead && remote.read,
|
|
649
|
+
remoteWrite: ctx.policy.remoteWrite && remote.write,
|
|
650
|
+
}
|
|
651
|
+
if (!policy.remoteRead && !policy.remoteWrite) return undefined
|
|
652
|
+
// A signed download waits in a temp until its tag checks: in vx's
|
|
653
|
+
// cache directory, which the sandbox walls, not the shared temp dir a
|
|
654
|
+
// sandboxed task may read (L-45).
|
|
655
|
+
return new LayeredCache(ctx.localCache, new TurboRemoteCache(config, fetch, ctx.cacheDir), {
|
|
656
|
+
policy,
|
|
657
|
+
onRemoteError: (err) => ctx.warn(`vx/turbo-cache: ${err.message}`),
|
|
658
|
+
})
|
|
659
|
+
},
|
|
660
|
+
})
|
|
661
|
+
}
|