@vzn/vx-migrate 0.0.645 → 0.0.648
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -73
- package/package.json +3 -3
- package/src/adopt.ts +53 -12
- package/src/adoption-plugin.ts +9 -11
- package/src/bin.ts +21 -1
- package/src/glob-grammar.ts +82 -0
- package/src/index.ts +36 -10
- package/src/mapping-cache.ts +14 -6
- package/src/migrate-nx.ts +7 -4
- package/src/migrate-scripts.ts +10 -102
- package/src/migrate-turbo.ts +2 -6
- package/src/migrate.ts +57 -38
- package/src/nx/index.ts +19 -5
- package/src/nx/nx-inputs.ts +44 -9
- package/src/nx/nx-map.ts +65 -25
- package/src/nx/nx-outputs.ts +16 -6
- package/src/nx/nx-upstream.ts +15 -7
- package/src/nx-cache/index.ts +60 -50
- package/src/nx-exec.cjs +2 -2
- package/src/package-fs.ts +6 -1
- package/src/packages.ts +9 -7
- package/src/plugin-gaps.ts +32 -6
- package/src/remote-cache.ts +5 -1
- package/src/remote-deadline.ts +7 -1
- package/src/remote-refusal.ts +31 -0
- package/src/remote-retry.ts +9 -4
- package/src/remote-url.ts +28 -0
- package/src/script-command.ts +110 -4
- package/src/shared-outputs.ts +24 -12
- package/src/tracked-outputs.ts +11 -7
- package/src/turbo/index.ts +33 -17
- package/src/turbo/turbo-map.ts +97 -192
- package/src/turbo/turbo-reads.ts +183 -0
- package/src/turbo-cache/index.ts +261 -102
- package/src/turbo-cache/oidc.ts +52 -26
- package/src/turbo-cache/scope.ts +5 -3
- package/src/vite-task/vite-task-map.ts +11 -6
- package/src/workspace-plugins.ts +5 -2
package/src/turbo-cache/index.ts
CHANGED
|
@@ -18,13 +18,17 @@ import path from 'node:path'
|
|
|
18
18
|
import {
|
|
19
19
|
type ArtifactInfo,
|
|
20
20
|
definePlugin,
|
|
21
|
+
type LentAccess,
|
|
21
22
|
type RemoteCacheLayer,
|
|
22
23
|
type RemoteOpenContext,
|
|
23
24
|
type VxPlugin,
|
|
24
25
|
refuseUnknownOptions,
|
|
26
|
+
UserError,
|
|
25
27
|
type PluginOptionKinds,
|
|
26
28
|
} from '@vzn/vx'
|
|
27
|
-
import { deadline } from '../remote-deadline.js'
|
|
29
|
+
import { deadline, MAX_TIMER_MS } from '../remote-deadline.js'
|
|
30
|
+
import { checkRemoteUrl } from '../remote-url.js'
|
|
31
|
+
import { refusal } from '../remote-refusal.js'
|
|
28
32
|
import { keyedByRef, readScopes, remoteScope, type RemoteScope } from './scope.js'
|
|
29
33
|
import { OutageBreaker } from '../remote-breaker.js'
|
|
30
34
|
import { Cooldown, withRetry } from '../remote-retry.js'
|
|
@@ -44,7 +48,7 @@ import {
|
|
|
44
48
|
|
|
45
49
|
export interface TurboCacheOptions {
|
|
46
50
|
/** Base URL of the cache server (`https://cache.example.com`), or `TURBO_API`; with a token and neither, Vercel's hosted cache, as for `turbo`. */
|
|
47
|
-
|
|
51
|
+
url?: string
|
|
48
52
|
/** Bearer token every request carries, or `TURBO_TOKEN`, OIDC, `.turbo/config.json`, then the OS keychain. */
|
|
49
53
|
token?: string
|
|
50
54
|
/**
|
|
@@ -54,7 +58,7 @@ export interface TurboCacheOptions {
|
|
|
54
58
|
* its `tokenUrl`. The job needs `permissions: id-token: write`.
|
|
55
59
|
*/
|
|
56
60
|
oidc?: true | TurboOidcOptions
|
|
57
|
-
/** `teamId` query parameter, or `TURBO_TEAMID
|
|
61
|
+
/** `teamId` query parameter, or `TURBO_TEAMID`; the signature folds it, `''` when unset, as Turbo's does. */
|
|
58
62
|
teamId?: string
|
|
59
63
|
/** `slug` query parameter (a team slug), or `TURBO_TEAM`. */
|
|
60
64
|
teamSlug?: string
|
|
@@ -62,8 +66,10 @@ export interface TurboCacheOptions {
|
|
|
62
66
|
* Sign uploads and verify downloads with Turbo's artifact signature
|
|
63
67
|
* (HMAC-SHA256, `x-artifact-tag`), or `TURBO_REMOTE_CACHE_SIGNATURE_KEY`
|
|
64
68
|
* where turbo.json sets `remoteCache.signature: true`, as Turbo reads it.
|
|
65
|
-
*
|
|
66
|
-
*
|
|
69
|
+
* The option needs at least 32 bytes; the env key warns below that,
|
|
70
|
+
* refused only under turbo.json's `futureFlags.longerSignatureKey`, as
|
|
71
|
+
* in Turbo. Used raw. A download whose tag does not verify is a miss,
|
|
72
|
+
* never a restore.
|
|
67
73
|
*/
|
|
68
74
|
signatureKey?: string
|
|
69
75
|
/**
|
|
@@ -102,6 +108,8 @@ export interface TurboJsonRemoteCache {
|
|
|
102
108
|
timeout?: unknown
|
|
103
109
|
uploadTimeout?: unknown
|
|
104
110
|
preflight?: unknown
|
|
111
|
+
/** turbo.json's `futureFlags.longerSignatureKey`, which `remoteCacheOf` hoists here. */
|
|
112
|
+
longerSignatureKey?: unknown
|
|
105
113
|
}
|
|
106
114
|
|
|
107
115
|
/** `.turbo/config.json`'s fields, under any of the spellings Turbo accepts. */
|
|
@@ -113,7 +121,7 @@ export interface TurboLocalConfig {
|
|
|
113
121
|
}
|
|
114
122
|
|
|
115
123
|
export interface TurboCacheConfig {
|
|
116
|
-
|
|
124
|
+
url: string
|
|
117
125
|
/** Undefined: the layer's `open` exchanges OIDC or asks the OS keychain. */
|
|
118
126
|
token?: string
|
|
119
127
|
/** The job's OIDC exchange, when `oidc` is set and GitHub hands the job a token. */
|
|
@@ -195,6 +203,7 @@ function trackTemp(file: string): void {
|
|
|
195
203
|
for (const f of liveTemps) {
|
|
196
204
|
try {
|
|
197
205
|
unlinkSync(f)
|
|
206
|
+
// quiet: a temp file already gone at exit
|
|
198
207
|
} catch {
|
|
199
208
|
// never written, or gone
|
|
200
209
|
}
|
|
@@ -205,6 +214,7 @@ function trackTemp(file: string): void {
|
|
|
205
214
|
async function removeTemp(file: string): Promise<void> {
|
|
206
215
|
// Listed until the unlink lands: an exit in between would otherwise find
|
|
207
216
|
// nothing to remove and leave the temp (item 868).
|
|
217
|
+
// quiet: a temp file already gone
|
|
208
218
|
await unlink(file).catch(() => undefined)
|
|
209
219
|
liveTemps.delete(file)
|
|
210
220
|
}
|
|
@@ -245,7 +255,7 @@ function tagsEqual(expected: string, actual: string): boolean {
|
|
|
245
255
|
return diff === 0
|
|
246
256
|
}
|
|
247
257
|
|
|
248
|
-
/** Where `turbo` itself sends a token with no `
|
|
258
|
+
/** Where `turbo` itself sends a token with no `url`: Vercel's hosted Remote Cache. */
|
|
249
259
|
export const VERCEL_API = 'https://vercel.com/api'
|
|
250
260
|
|
|
251
261
|
/** Resolve options over Turbo's environment; `undefined` = not configured. */
|
|
@@ -255,25 +265,28 @@ export function resolveTurboCacheConfig(
|
|
|
255
265
|
file: TurboJsonRemoteCache = {},
|
|
256
266
|
/** The repo's `.turbo/config.json` (`turbo link` writes it): below the environment, above turbo.json. */
|
|
257
267
|
local: TurboLocalConfig = {},
|
|
268
|
+
warn: (message: string) => void = (m) => console.error(`[vx] plugin '@vzn/vx-migrate': ${m}`),
|
|
258
269
|
): TurboCacheConfig | undefined {
|
|
259
270
|
// `enabled: false` turns Turbo's remote cache off whatever the env says;
|
|
260
271
|
// options name a cache for vx alone, and win.
|
|
261
|
-
if (file.enabled === false && options.
|
|
272
|
+
if (file.enabled === false && options.url === undefined && options.token === undefined)
|
|
262
273
|
return undefined
|
|
263
274
|
const fromFile = (v: unknown): string | undefined => (typeof v === 'string' ? v : undefined)
|
|
275
|
+
// Empty is unset, as Turbo reads it: a fork's pull request gets `''` for a
|
|
276
|
+
// secret, and `${{ vars.TURBO_API }}` is `''` for an unset variable.
|
|
277
|
+
const set = (name: string): string | undefined => env[name] || undefined
|
|
264
278
|
// A Vercel build hands Turbo its cache as VERCEL_ARTIFACTS_TOKEN and
|
|
265
279
|
// _OWNER (the team id), taken where TURBO_TOKEN with a team is not set
|
|
266
280
|
// (Turbo's `override_env.rs`); unread, turboCache() declined on Vercel.
|
|
267
|
-
const turboPair =
|
|
281
|
+
const turboPair = set('TURBO_TOKEN') && (set('TURBO_TEAMID') || set('TURBO_TEAM'))
|
|
268
282
|
const vercel =
|
|
269
|
-
!turboPair &&
|
|
270
|
-
? { token:
|
|
271
|
-
: { token: undefined, teamId: turboPair ? undefined :
|
|
272
|
-
|
|
273
|
-
const given = (options.token ?? env['TURBO_TOKEN'] ?? vercel.token) || undefined
|
|
283
|
+
!turboPair && set('VERCEL_ARTIFACTS_TOKEN') && set('VERCEL_ARTIFACTS_OWNER')
|
|
284
|
+
? { token: set('VERCEL_ARTIFACTS_TOKEN'), teamId: set('VERCEL_ARTIFACTS_OWNER') }
|
|
285
|
+
: { token: undefined, teamId: turboPair ? undefined : set('VERCEL_ARTIFACTS_OWNER') }
|
|
286
|
+
const given = (options.token ?? set('TURBO_TOKEN') ?? vercel.token) || undefined
|
|
274
287
|
// OIDC sits under the variables and over the config files.
|
|
275
|
-
const idTokenUrl =
|
|
276
|
-
const idTokenBearer =
|
|
288
|
+
const idTokenUrl = set('ACTIONS_ID_TOKEN_REQUEST_URL')
|
|
289
|
+
const idTokenBearer = set('ACTIONS_ID_TOKEN_REQUEST_TOKEN')
|
|
277
290
|
const exchanges =
|
|
278
291
|
options.oidc !== undefined &&
|
|
279
292
|
given === undefined &&
|
|
@@ -282,41 +295,39 @@ export function resolveTurboCacheConfig(
|
|
|
282
295
|
const token = given ?? (exchanges ? undefined : fromFile(local.token) || undefined)
|
|
283
296
|
const teamId =
|
|
284
297
|
options.teamId ??
|
|
285
|
-
|
|
298
|
+
set('TURBO_TEAMID') ??
|
|
286
299
|
vercel.teamId ??
|
|
287
300
|
fromFile(local.teamId) ??
|
|
288
301
|
fromFile(file.teamId)
|
|
289
302
|
const teamSlug =
|
|
290
|
-
options.teamSlug ??
|
|
291
|
-
// Turbo's own default when a token or a team is set and no
|
|
303
|
+
options.teamSlug ?? set('TURBO_TEAM') ?? fromFile(local.teamSlug) ?? fromFile(file.teamSlug)
|
|
304
|
+
// Turbo's own default when a token or a team is set and no URL is:
|
|
292
305
|
// Vercel's hosted Remote Cache. A token alone is a configured cache, as it
|
|
293
306
|
// is for `turbo`; a URL or a team without one asks the OS keychain at
|
|
294
307
|
// `open`; none of them is the declined, local run.
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
308
|
+
// Each source with its name: TURBO_API was refused as a `url`
|
|
309
|
+
// option the workspace never set.
|
|
310
|
+
const [rawUrl, urlFrom] =
|
|
311
|
+
options.url !== undefined
|
|
312
|
+
? [options.url, 'turboCache() option "url"']
|
|
313
|
+
: set('TURBO_API') !== undefined
|
|
314
|
+
? [set('TURBO_API'), 'TURBO_API']
|
|
315
|
+
: fromFile(local.apiUrl) !== undefined
|
|
316
|
+
? [fromFile(local.apiUrl), '.turbo/config.json apiUrl']
|
|
317
|
+
: fromFile(file.apiUrl) !== undefined
|
|
318
|
+
? [fromFile(file.apiUrl), 'turbo.json remoteCache.apiUrl']
|
|
319
|
+
: [token || teamId || teamSlug ? VERCEL_API : undefined, 'url']
|
|
320
|
+
const url = rawUrl?.replace(/\/+$/, '')
|
|
321
|
+
if (!url) return undefined
|
|
322
|
+
checkRemoteUrl(url, urlFrom)
|
|
303
323
|
const oidc =
|
|
304
324
|
options.oidc === undefined
|
|
305
325
|
? undefined
|
|
306
|
-
: oidcSettings(options.oidc,
|
|
307
|
-
// The URL is printed in every refusal line, so a `user:pass@` in it would
|
|
308
|
-
// leak to the log. Credentials go in the token.
|
|
309
|
-
if (URL.canParse(apiUrl)) {
|
|
310
|
-
const u = new URL(apiUrl)
|
|
311
|
-
if (u.username !== '' || u.password !== '')
|
|
312
|
-
throw new Error(
|
|
313
|
-
'vx/turbo-cache: apiUrl carries credentials (user:pass@); pass them as the token instead',
|
|
314
|
-
)
|
|
315
|
-
}
|
|
326
|
+
: oidcSettings(options.oidc, url === VERCEL_API, teamId ?? teamSlug)
|
|
316
327
|
const fault = token === undefined ? null : headerValueFault(token)
|
|
317
328
|
if (fault !== null)
|
|
318
329
|
throw new Error(
|
|
319
|
-
`
|
|
330
|
+
`turboCache(): the token holds ${fault}, which no HTTP header can carry — check the secret (it is not printed)`,
|
|
320
331
|
)
|
|
321
332
|
const preflight =
|
|
322
333
|
options.preflight ??
|
|
@@ -324,34 +335,34 @@ export function resolveTurboCacheConfig(
|
|
|
324
335
|
file.preflight === true
|
|
325
336
|
// Turbo signs only under `remoteCache.signature: true`; the env key alone
|
|
326
337
|
// signs nothing there, and a short one here refused the whole cache.
|
|
327
|
-
|
|
328
|
-
const
|
|
329
|
-
const signatureKey =
|
|
330
|
-
|
|
338
|
+
const signing = file.signature === true
|
|
339
|
+
const envKey = signing ? env['TURBO_REMOTE_CACHE_SIGNATURE_KEY'] || undefined : undefined
|
|
340
|
+
const signatureKey = options.signatureKey ?? envKey
|
|
341
|
+
// Turbo signs with no key as it fails every request: no usable remote. Read
|
|
342
|
+
// unsigned here, the bytes a signing repo refuses were restored.
|
|
343
|
+
if (signing && signatureKey === undefined) {
|
|
344
|
+
warn(
|
|
345
|
+
'turbo.json sets remoteCache.signature but TURBO_REMOTE_CACHE_SIGNATURE_KEY is unset: remote cache off for this run, as for turbo',
|
|
346
|
+
)
|
|
347
|
+
return undefined
|
|
348
|
+
}
|
|
331
349
|
if (signatureKey !== undefined) {
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
350
|
+
const bytes = Buffer.byteLength(signatureKey)
|
|
351
|
+
// Turbo only warns about a short key unless `futureFlags.longerSignatureKey`
|
|
352
|
+
// is set; refusing it here failed a run Turbo runs.
|
|
353
|
+
if (bytes < MIN_SIGNATURE_KEY_LENGTH) {
|
|
354
|
+
if (options.signatureKey !== undefined || file.longerSignatureKey === true)
|
|
355
|
+
throw new Error(
|
|
356
|
+
`turboCache(): signatureKey must be at least ${MIN_SIGNATURE_KEY_LENGTH} bytes`,
|
|
357
|
+
)
|
|
358
|
+
warn(
|
|
359
|
+
`TURBO_REMOTE_CACHE_SIGNATURE_KEY is ${bytes} bytes; ${MIN_SIGNATURE_KEY_LENGTH} or more is safe (Turbo enforces it under futureFlags.longerSignatureKey)`,
|
|
335
360
|
)
|
|
336
361
|
}
|
|
337
|
-
if (!teamId)
|
|
338
|
-
throw new Error(
|
|
339
|
-
'vx/turbo-cache: signatureKey needs teamId — the team id is part of the signed message',
|
|
340
|
-
)
|
|
341
362
|
}
|
|
342
363
|
const retries = options.retries ?? 1
|
|
343
|
-
// A NaN never reaches the bound, and the request would be resent forever.
|
|
344
|
-
if (!Number.isInteger(retries) || retries < 0)
|
|
345
|
-
throw new Error(`vx/turbo-cache: retries must be a whole number ≥ 0, got ${retries}`)
|
|
346
|
-
// 0 is no deadline, as Turbo reads it; a negative or non-number one made
|
|
347
|
-
// AbortSignal.timeout throw on every request.
|
|
348
|
-
for (const name of ['timeoutMs', 'uploadTimeoutMs'] as const) {
|
|
349
|
-
const ms = options[name]
|
|
350
|
-
if (ms !== undefined && !(typeof ms === 'number' && Number.isFinite(ms) && ms >= 0))
|
|
351
|
-
throw new Error(`vx/turbo-cache: ${name} must be ms ≥ 0 (0: none), got ${JSON.stringify(ms)}`)
|
|
352
|
-
}
|
|
353
364
|
return {
|
|
354
|
-
|
|
365
|
+
url,
|
|
355
366
|
...(token !== undefined ? { token } : {}),
|
|
356
367
|
...(oidc !== undefined && exchanges
|
|
357
368
|
? { oidc: { ...oidc, idTokenUrl: idTokenUrl!, idTokenBearer: idTokenBearer! } }
|
|
@@ -381,7 +392,7 @@ export function resolveTurboCacheConfig(
|
|
|
381
392
|
function truthy(v: string | undefined, name: string): boolean | undefined {
|
|
382
393
|
if (v === undefined || v === '') return undefined
|
|
383
394
|
if (!/^(1|0|true|false)$/.test(v))
|
|
384
|
-
throw new Error(`
|
|
395
|
+
throw new Error(`turboCache(): ${name} should be 1 or 0, got ${JSON.stringify(v)}`)
|
|
385
396
|
return v === '1' || v === 'true'
|
|
386
397
|
}
|
|
387
398
|
|
|
@@ -393,7 +404,7 @@ function seconds(v: unknown, name: string): number | undefined {
|
|
|
393
404
|
if (v === undefined || v === null || v === '') return undefined
|
|
394
405
|
const n = typeof v === 'string' ? Number(v) : v
|
|
395
406
|
if (typeof n !== 'number' || !Number.isInteger(n) || n < 0)
|
|
396
|
-
throw new Error(`
|
|
407
|
+
throw new Error(`turboCache(): ${name} must be whole seconds ≥ 0, got ${JSON.stringify(v)}`)
|
|
397
408
|
return n * 1000
|
|
398
409
|
}
|
|
399
410
|
|
|
@@ -428,6 +439,19 @@ async function boundedText(res: Response, max: number): Promise<string | null> {
|
|
|
428
439
|
*/
|
|
429
440
|
const MAX_SIGNED_BODY = 2 * 1024 ** 3 + ((2 * 1024 ** 3) >> 8) + 64 * 1024
|
|
430
441
|
|
|
442
|
+
/** A lent token ending sooner than this is exchanged again first, so an agent's request never carries a spent one. */
|
|
443
|
+
const LEND_MARGIN_MS = 5 * 60_000
|
|
444
|
+
|
|
445
|
+
/** Statuses of a server that has no batch query. */
|
|
446
|
+
const NO_BATCH = new Set([404, 405, 501])
|
|
447
|
+
|
|
448
|
+
/**
|
|
449
|
+
* The most hashes one batch query may carry: Vercel answers more with 400
|
|
450
|
+
* "Too many hashes. Limit is 100." (main f71b377 asked 107 at once, and
|
|
451
|
+
* the run planned as if none were held).
|
|
452
|
+
*/
|
|
453
|
+
const BATCH_MAX = 100
|
|
454
|
+
|
|
431
455
|
/**
|
|
432
456
|
* The seam implementation: `has` is HEAD, `hasMany` is the batch query,
|
|
433
457
|
* `get`/`put` carry `x-artifact-duration` (and the tag when signing). An
|
|
@@ -458,14 +482,27 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
458
482
|
* entry is immutable, so any other key is asked first.
|
|
459
483
|
*/
|
|
460
484
|
private readonly absent = new Set<string>()
|
|
485
|
+
/**
|
|
486
|
+
* Keys whose GET this process answered 200. Core saves a key after its
|
|
487
|
+
* hit only when that hit did not restore (bad bytes, a tag that does not
|
|
488
|
+
* verify), so `put` sends it over the stored one: its HEAD saw 200 and
|
|
489
|
+
* kept the bad entry for every later run.
|
|
490
|
+
*/
|
|
491
|
+
private readonly served = new Set<string>()
|
|
461
492
|
private readonly key: Uint8Array | undefined
|
|
462
493
|
readonly endpoint: string
|
|
463
494
|
/** Undefined until `open` reads the keychain; a layer never opened (a plan) asks the server nothing. */
|
|
464
495
|
private token: string | undefined
|
|
465
496
|
/** Set by `open` when the token came from the keychain: a refusal deletes it. */
|
|
466
497
|
private forget: (() => Promise<void>) | undefined
|
|
467
|
-
/** The
|
|
468
|
-
private exchanged: string
|
|
498
|
+
/** The tokens OIDC exchanges gave, the last one current, each revoked when the run ends. */
|
|
499
|
+
private exchanged: string[] = []
|
|
500
|
+
/** When the current exchanged token ends (epoch ms), when the server said. */
|
|
501
|
+
private expiresAt: number | undefined
|
|
502
|
+
/** Set by `open` when this machine has a lent token and no token of its own: read per request. */
|
|
503
|
+
private borrowed: (() => string | undefined) | undefined
|
|
504
|
+
/** The token a run running its tasks here lent (`CacheContext.lent`). */
|
|
505
|
+
lent: (() => string | undefined) | undefined
|
|
469
506
|
private readonly keychainRemote: KeychainRemote
|
|
470
507
|
/** The remote's rate limit, which every request of this client waits out. */
|
|
471
508
|
private readonly cooldown = new Cooldown()
|
|
@@ -480,39 +517,72 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
480
517
|
private readonly maxSignedBody: number = MAX_SIGNED_BODY,
|
|
481
518
|
) {
|
|
482
519
|
this.key = config.signatureKey === undefined ? undefined : Buffer.from(config.signatureKey)
|
|
483
|
-
this.endpoint = `${config.
|
|
520
|
+
this.endpoint = `${config.url}/v8/artifacts`
|
|
484
521
|
this.token = config.token
|
|
485
522
|
const team = config.teamId ?? config.teamSlug
|
|
486
523
|
this.keychainRemote = {
|
|
487
524
|
key: team === undefined ? this.endpoint : `${this.endpoint}#${team}`,
|
|
488
|
-
label: remoteLabel(config.
|
|
525
|
+
label: remoteLabel(config.url, team),
|
|
489
526
|
env: 'TURBO_TOKEN',
|
|
490
527
|
}
|
|
491
528
|
}
|
|
492
529
|
|
|
493
530
|
async open(ctx: RemoteOpenContext): Promise<void> {
|
|
494
|
-
|
|
531
|
+
const lent = this.lent?.() === undefined ? undefined : this.lent
|
|
532
|
+
if (this.config.oidcMissing && lent === undefined)
|
|
495
533
|
ctx.warn(
|
|
496
534
|
'oidc: GitHub gave this job no OIDC token: it needs `permissions: id-token: write` (a fork or Dependabot pull request never gets one)',
|
|
497
535
|
)
|
|
498
536
|
if (this.token !== undefined) return
|
|
537
|
+
// A lent token beats this machine's own OIDC: the server files a write
|
|
538
|
+
// by the token, so an agent's own (another workflow, another ref)
|
|
539
|
+
// filed the task where its run could not read it.
|
|
540
|
+
if (lent !== undefined) {
|
|
541
|
+
this.borrowed = lent
|
|
542
|
+
return
|
|
543
|
+
}
|
|
499
544
|
if (this.config.oidc !== undefined) {
|
|
500
545
|
try {
|
|
501
|
-
|
|
546
|
+
await this.exchange()
|
|
502
547
|
} catch (err) {
|
|
503
548
|
throw new Error(
|
|
504
549
|
`oidc: ${err instanceof Error ? err.message : String(err)}; local cache only this run`,
|
|
505
550
|
)
|
|
506
551
|
}
|
|
507
|
-
this.exchanged = this.token
|
|
508
552
|
return
|
|
509
553
|
}
|
|
510
554
|
this.token = await keychainToken(ctx, this.keychainRemote)
|
|
511
555
|
this.forget = () => ctx.forgetSecret({ key: this.keychainRemote.key })
|
|
512
556
|
}
|
|
513
557
|
|
|
558
|
+
private async exchange(): Promise<void> {
|
|
559
|
+
const got = await exchangeOidc(this.config.oidc!, this.fetchImpl, this.wait)
|
|
560
|
+
this.token = got.token
|
|
561
|
+
this.expiresAt = got.expiresAt
|
|
562
|
+
this.exchanged.push(got.token)
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/**
|
|
566
|
+
* The token an OIDC exchange gave this run, for a machine that runs its
|
|
567
|
+
* tasks (`RemoteCacheLayer.lend`); exchanged again when it ends within
|
|
568
|
+
* five minutes. A token of any other kind is never lent.
|
|
569
|
+
*/
|
|
570
|
+
async lend(): Promise<LentAccess | undefined> {
|
|
571
|
+
if (this.exchanged.length === 0) return undefined
|
|
572
|
+
if (this.expiresAt !== undefined && this.expiresAt - Date.now() < LEND_MARGIN_MS)
|
|
573
|
+
await this.exchange()
|
|
574
|
+
return {
|
|
575
|
+
token: this.token!,
|
|
576
|
+
...(this.expiresAt === undefined ? {} : { expiresAt: this.expiresAt }),
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
private bearer(): string | undefined {
|
|
581
|
+
return this.borrowed?.() ?? this.token
|
|
582
|
+
}
|
|
583
|
+
|
|
514
584
|
private url(pathname: string): string {
|
|
515
|
-
const u = new URL(`${this.config.
|
|
585
|
+
const u = new URL(`${this.config.url}/v8/artifacts${pathname}`)
|
|
516
586
|
// Turbo sends a team id only in Vercel's `team_` form (its API client's
|
|
517
587
|
// `add_team_params`); a personal account's owner id is not one, and the
|
|
518
588
|
// signature, which folds the id as given, is unaffected.
|
|
@@ -523,7 +593,7 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
523
593
|
|
|
524
594
|
private headers(extra: Record<string, string> = {}, auth = true): Record<string, string> {
|
|
525
595
|
return {
|
|
526
|
-
...(auth ? { Authorization: `Bearer ${this.
|
|
596
|
+
...(auth ? { Authorization: `Bearer ${this.bearer()}` } : {}),
|
|
527
597
|
'x-artifact-client-interactive': process.stdout.isTTY ? '1' : '0',
|
|
528
598
|
...extra,
|
|
529
599
|
}
|
|
@@ -531,7 +601,7 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
531
601
|
|
|
532
602
|
/**
|
|
533
603
|
* Turbo's preflight: an `OPTIONS` to the request's URL names the method
|
|
534
|
-
* and headers to come; the answer's `Location` (relative to `
|
|
604
|
+
* and headers to come; the answer's `Location` (relative to `url`) is
|
|
535
605
|
* where the request goes, as given — a signed storage URL signs its own
|
|
536
606
|
* query, so nothing is appended — and the token goes along only when
|
|
537
607
|
* `Access-Control-Allow-Headers` admits `Authorization` (or `*`), so it
|
|
@@ -550,7 +620,7 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
550
620
|
const res = await this.fetchImpl(url, {
|
|
551
621
|
method: 'OPTIONS',
|
|
552
622
|
headers: {
|
|
553
|
-
Authorization: `Bearer ${this.
|
|
623
|
+
Authorization: `Bearer ${this.bearer()}`,
|
|
554
624
|
'Access-Control-Request-Method': method === 'HEAD' ? 'GET' : method,
|
|
555
625
|
'Access-Control-Request-Headers': ['Authorization', ...Object.keys(headers)].join(', '),
|
|
556
626
|
},
|
|
@@ -561,7 +631,7 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
561
631
|
const location = res.headers.get('location')
|
|
562
632
|
const allowed = res.headers.get('access-control-allow-headers') ?? ''
|
|
563
633
|
return {
|
|
564
|
-
url: location === null ? url : new URL(location, this.config.
|
|
634
|
+
url: location === null ? url : new URL(location, this.config.url).toString(),
|
|
565
635
|
auth:
|
|
566
636
|
allowed.trim() === '*' ||
|
|
567
637
|
allowed.split(',').some((h) => h.trim().toLowerCase() === 'authorization'),
|
|
@@ -579,7 +649,7 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
579
649
|
pathname: string,
|
|
580
650
|
init: { body?: Blob | string; headers?: Record<string, string>; timeoutMs?: number } = {},
|
|
581
651
|
): Promise<Response | undefined> {
|
|
582
|
-
if (this.
|
|
652
|
+
if (this.bearer() === undefined) return undefined
|
|
583
653
|
const timeoutMs = init.timeoutMs ?? this.config.timeoutMs
|
|
584
654
|
// 0 is Turbo's "no deadline".
|
|
585
655
|
const signal = () => (timeoutMs === 0 ? undefined : deadline(timeoutMs))
|
|
@@ -611,30 +681,32 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
611
681
|
const first = !this.writesRefused
|
|
612
682
|
this.writesRefused = true
|
|
613
683
|
if (!first) return undefined
|
|
684
|
+
// The status and the server's own words (`refusal`): a scope it
|
|
685
|
+
// refuses and a token it no longer takes read alike.
|
|
614
686
|
throw new Error(
|
|
615
|
-
|
|
687
|
+
`${(await refusal(res)).message}: the upload was refused (a read-only token or server); uploads off for this run, reads go on`,
|
|
616
688
|
)
|
|
617
689
|
}
|
|
618
690
|
if (res.status === 401 || res.status === 403) {
|
|
619
691
|
const first = !this.disabled
|
|
620
692
|
this.disabled = true
|
|
621
693
|
if (!first) return undefined
|
|
694
|
+
const said = (await refusal(res)).message
|
|
622
695
|
if (this.forget === undefined)
|
|
623
|
-
throw new Error(
|
|
696
|
+
throw new Error(`${said}: the token was refused; remote cache off for this run`)
|
|
624
697
|
await this.forget()
|
|
625
698
|
throw new Error(
|
|
626
|
-
|
|
699
|
+
`${said}: the token was refused; removed it from the OS keychain, so the next run asks again; remote cache off for this run`,
|
|
627
700
|
)
|
|
628
701
|
}
|
|
629
702
|
return res
|
|
630
703
|
}
|
|
631
704
|
|
|
632
|
-
/** Revoke
|
|
705
|
+
/** Revoke each token this run exchanged, once; a throw is the caller's warning. */
|
|
633
706
|
async revoke(): Promise<void> {
|
|
634
|
-
const
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
await revokeOidc(this.config.oidc!, token, this.fetchImpl, this.wait)
|
|
707
|
+
const tokens = this.exchanged.splice(0)
|
|
708
|
+
for (const token of tokens)
|
|
709
|
+
await revokeOidc(this.config.oidc!, token, this.fetchImpl, this.wait)
|
|
638
710
|
}
|
|
639
711
|
|
|
640
712
|
async has(hash: string): Promise<ArtifactInfo | null> {
|
|
@@ -650,26 +722,66 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
650
722
|
throw new Error(`HTTP ${res.status}`)
|
|
651
723
|
}
|
|
652
724
|
|
|
725
|
+
/**
|
|
726
|
+
* Null when the layer is off: no answer, which core keeps apart from
|
|
727
|
+
* "none held" (an empty map rewrote every key it asked as a miss in
|
|
728
|
+
* `remote_hits`, and the next run fetched none of them).
|
|
729
|
+
*/
|
|
653
730
|
async hasMany(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
|
|
654
|
-
if (this.disabled || hashes.length === 0) return
|
|
731
|
+
if (this.disabled || hashes.length === 0) return null
|
|
732
|
+
if (hashes.length <= BATCH_MAX) return this.batch(hashes)
|
|
733
|
+
const parts = await Promise.all(
|
|
734
|
+
Array.from({ length: Math.ceil(hashes.length / BATCH_MAX) }, (_, i) =>
|
|
735
|
+
this.batch(hashes.slice(i * BATCH_MAX, (i + 1) * BATCH_MAX)),
|
|
736
|
+
),
|
|
737
|
+
)
|
|
738
|
+
if (parts.some((p) => p === null)) return null
|
|
739
|
+
return new Map(parts.flatMap((p) => [...p!]))
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
private async batch(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
|
|
655
743
|
const res = await this.request('POST', '', {
|
|
656
744
|
body: JSON.stringify({ hashes }),
|
|
657
745
|
headers: { 'Content-Type': 'application/json', ...this.reads },
|
|
658
746
|
})
|
|
659
|
-
if (res === undefined) return
|
|
660
|
-
if (res.status !== 200)
|
|
747
|
+
if (res === undefined) return null
|
|
748
|
+
if (res.status !== 200) {
|
|
749
|
+
// A server with no batch query (404, 405, 501) is asked per hash. Any
|
|
750
|
+
// other refusal was silent: main f71b377's probe gave no answer and no
|
|
751
|
+
// one could say why, and DTE sized the run as if every task ran.
|
|
752
|
+
if (NO_BATCH.has(res.status)) {
|
|
753
|
+
await res.body?.cancel()
|
|
754
|
+
return null
|
|
755
|
+
}
|
|
756
|
+
throw await refusal(res, 'batch query answered ')
|
|
757
|
+
}
|
|
661
758
|
// An answer per hash is a few hundred bytes of ArtifactInfo; a reply
|
|
662
759
|
// past this bound is no answer, read no further than the bound (L-9).
|
|
663
760
|
const bound = hashes.length * 4096 + 64 * 1024
|
|
664
761
|
const text = await boundedText(res, bound)
|
|
665
|
-
if (text === null)
|
|
666
|
-
|
|
762
|
+
if (text === null) throw new Error(`batch query answered past ${bound} bytes; read no further`)
|
|
763
|
+
// A 200 that is no JSON object (a proxy's page, `null`, `[]`) threw a
|
|
764
|
+
// parser's line that named no query, or read as every hash absent:
|
|
765
|
+
// DTE sized the run as if every task ran.
|
|
766
|
+
let info: unknown
|
|
767
|
+
try {
|
|
768
|
+
info = JSON.parse(text)
|
|
769
|
+
} catch {
|
|
770
|
+
// quiet: refused below, with what came
|
|
771
|
+
info = undefined
|
|
772
|
+
}
|
|
773
|
+
if (typeof info !== 'object' || info === null || Array.isArray(info)) {
|
|
774
|
+
const said = text.replace(/\s+/g, ' ').trim().slice(0, 200)
|
|
775
|
+
throw new Error(
|
|
776
|
+
`batch query answered HTTP 200 with no JSON object${said === '' ? '' : `: ${said}`}`,
|
|
777
|
+
)
|
|
778
|
+
}
|
|
667
779
|
// Each hash answers ArtifactInfo, null, or `{ error: { message } }`; an
|
|
668
780
|
// error entry is not a stored artifact, and it was counted as one (a GET
|
|
669
781
|
// spent on a 404, item 929).
|
|
670
782
|
const held = new Map<string, ArtifactInfo>()
|
|
671
783
|
for (const h of hashes) {
|
|
672
|
-
const v = info[h]
|
|
784
|
+
const v = (info as Record<string, unknown>)[h]
|
|
673
785
|
if (typeof v !== 'object' || v === null || 'error' in v) this.absent.add(h)
|
|
674
786
|
else {
|
|
675
787
|
const { size, taskDurationMs } = v as { size?: unknown; taskDurationMs?: unknown }
|
|
@@ -688,13 +800,14 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
688
800
|
})
|
|
689
801
|
if (res === undefined) return null
|
|
690
802
|
if (res.status !== 200) {
|
|
691
|
-
await res.body?.cancel()
|
|
692
803
|
if (res.status === 404) {
|
|
804
|
+
await res.body?.cancel()
|
|
693
805
|
this.absent.add(hash)
|
|
694
806
|
return null
|
|
695
807
|
}
|
|
696
|
-
throw
|
|
808
|
+
throw await refusal(res)
|
|
697
809
|
}
|
|
810
|
+
this.served.add(hash)
|
|
698
811
|
const duration = Number(res.headers.get('x-artifact-duration'))
|
|
699
812
|
const durationMs = Number.isFinite(duration) && duration > 0 ? duration : undefined
|
|
700
813
|
if (this.key === undefined) return { body: res, durationMs }
|
|
@@ -742,9 +855,14 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
742
855
|
return new Response(unlinkingStream(temp))
|
|
743
856
|
}
|
|
744
857
|
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
858
|
+
missing(hash: string): void {
|
|
859
|
+
this.absent.add(hash)
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
async put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void | false> {
|
|
863
|
+
// Nothing sent, so no upload is counted.
|
|
864
|
+
if (this.disabled || this.writesRefused) return false
|
|
865
|
+
if (!this.absent.has(hash) && !this.served.has(hash)) {
|
|
748
866
|
// Part of the upload, so under its deadline. Any answer but 200 or 404
|
|
749
867
|
// uploads: a lost store costs more. A server the HEAD cannot reach
|
|
750
868
|
// fails the put, as the PUT would.
|
|
@@ -752,7 +870,8 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
752
870
|
headers: this.reads,
|
|
753
871
|
timeoutMs: this.config.uploadTimeoutMs,
|
|
754
872
|
})
|
|
755
|
-
if (head === undefined
|
|
873
|
+
if (head === undefined) return false
|
|
874
|
+
if (head.status === 200) return
|
|
756
875
|
if (head.status === 404) this.absent.add(hash)
|
|
757
876
|
}
|
|
758
877
|
const headers: Record<string, string> = {
|
|
@@ -768,11 +887,12 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
768
887
|
headers,
|
|
769
888
|
timeoutMs: this.config.uploadTimeoutMs,
|
|
770
889
|
})
|
|
771
|
-
if (res === undefined) return
|
|
890
|
+
if (res === undefined) return false
|
|
772
891
|
// Turbo's client takes any 2xx (`error_for_status`); a server's 201 or
|
|
773
892
|
// 204 stored the artifact and was counted a failed upload.
|
|
774
|
-
if (!res.ok) throw
|
|
893
|
+
if (!res.ok) throw await refusal(res)
|
|
775
894
|
this.absent.delete(hash)
|
|
895
|
+
this.served.delete(hash)
|
|
776
896
|
}
|
|
777
897
|
}
|
|
778
898
|
|
|
@@ -784,6 +904,7 @@ function localConfigOf(root: string): TurboLocalConfig {
|
|
|
784
904
|
let raw: unknown
|
|
785
905
|
try {
|
|
786
906
|
raw = JSON.parse(readFileSync(path.join(root, '.turbo', 'config.json'), 'utf8'))
|
|
907
|
+
// quiet: a local config file: none read is none linked
|
|
787
908
|
} catch {
|
|
788
909
|
return {}
|
|
789
910
|
}
|
|
@@ -803,12 +924,22 @@ function remoteCacheOf(root: string): TurboJsonRemoteCache {
|
|
|
803
924
|
let text: string
|
|
804
925
|
try {
|
|
805
926
|
text = readFileSync(path.join(root, name), 'utf8')
|
|
927
|
+
// quiet: a local config file: none read is none declared
|
|
806
928
|
} catch {
|
|
807
929
|
continue
|
|
808
930
|
}
|
|
809
931
|
try {
|
|
810
|
-
const
|
|
811
|
-
|
|
932
|
+
const json = Bun.JSONC.parse(text) as {
|
|
933
|
+
remoteCache?: unknown
|
|
934
|
+
futureFlags?: { longerSignatureKey?: unknown }
|
|
935
|
+
} | null
|
|
936
|
+
const rc = json?.remoteCache
|
|
937
|
+
const longerSignatureKey = json?.futureFlags?.longerSignatureKey
|
|
938
|
+
return {
|
|
939
|
+
...(typeof rc === 'object' && rc !== null ? (rc as TurboJsonRemoteCache) : {}),
|
|
940
|
+
...(longerSignatureKey === undefined ? {} : { longerSignatureKey }),
|
|
941
|
+
}
|
|
942
|
+
// quiet: a local config file: none read is none declared
|
|
812
943
|
} catch {
|
|
813
944
|
return {}
|
|
814
945
|
}
|
|
@@ -818,7 +949,7 @@ function remoteCacheOf(root: string): TurboJsonRemoteCache {
|
|
|
818
949
|
|
|
819
950
|
/** Each option `TurboCacheOptions` names, with its kind: derived from the type, so the two cannot drift. */
|
|
820
951
|
const TURBO_CACHE_KEYS: PluginOptionKinds<TurboCacheOptions> = {
|
|
821
|
-
|
|
952
|
+
url: 'string',
|
|
822
953
|
token: 'string',
|
|
823
954
|
teamId: 'string',
|
|
824
955
|
teamSlug: 'string',
|
|
@@ -831,18 +962,44 @@ const TURBO_CACHE_KEYS: PluginOptionKinds<TurboCacheOptions> = {
|
|
|
831
962
|
scope: 'string',
|
|
832
963
|
}
|
|
833
964
|
|
|
965
|
+
/**
|
|
966
|
+
* The number options, refused at the call: in the `cache` hook a laptop
|
|
967
|
+
* with no remote named passed what failed the first CI run.
|
|
968
|
+
*/
|
|
969
|
+
function refuseBadNumbers(options: TurboCacheOptions): void {
|
|
970
|
+
// A NaN never reaches the bound, and the request would be resent forever.
|
|
971
|
+
const { retries } = options
|
|
972
|
+
if (retries !== undefined && !(Number.isInteger(retries) && retries >= 0))
|
|
973
|
+
throw new UserError(`turboCache() option "retries" must be a whole number ≥ 0, got ${retries}`)
|
|
974
|
+
// 0 is no deadline, as Turbo reads it; a negative or non-number one made
|
|
975
|
+
// AbortSignal.timeout throw on every request, and one past the longest
|
|
976
|
+
// timer aborted each as it started (Turbo's own env values are held).
|
|
977
|
+
for (const name of ['timeoutMs', 'uploadTimeoutMs'] as const) {
|
|
978
|
+
const ms = options[name]
|
|
979
|
+
if (ms !== undefined && !(ms >= 0 && ms <= MAX_TIMER_MS))
|
|
980
|
+
throw new UserError(
|
|
981
|
+
`turboCache() option "${name}" must be ms ≥ 0 (0: none) up to ${MAX_TIMER_MS}, got ${ms}`,
|
|
982
|
+
)
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
|
|
834
986
|
/**
|
|
835
987
|
* Declare in `.vx/workspace.ts`; the local store stays the floor beneath it:
|
|
836
988
|
*
|
|
837
989
|
* ```ts
|
|
838
|
-
* plugins: [turboCache({
|
|
990
|
+
* plugins: [turboCache({ url: 'https://cache.example.com', token: process.env.CACHE_TOKEN })]
|
|
839
991
|
* ```
|
|
840
992
|
*
|
|
841
993
|
* Declines with no URL, team or token, so it is safe to leave declared.
|
|
842
994
|
*/
|
|
843
995
|
export function turboCache(options: TurboCacheOptions = {}): VxPlugin {
|
|
844
996
|
refuseUnknownOptions('turboCache()', options, TURBO_CACHE_KEYS)
|
|
845
|
-
|
|
997
|
+
refuseBadNumbers(options)
|
|
998
|
+
// At the call too, as vxCache() checks its url: in the hook alone, a verb
|
|
999
|
+
// that opens no cache (`vx show`) took a url every run then failed on.
|
|
1000
|
+
const url = options.url?.replace(/\/+$/, '')
|
|
1001
|
+
if (url) checkRemoteUrl(url, 'turboCache() option "url"')
|
|
1002
|
+
const scope = remoteScope('turboCache', options.scope)
|
|
846
1003
|
// The run's layer and warn, for `teardown` to revoke an exchanged token.
|
|
847
1004
|
let opened: { layer: TurboRemoteCache; warn: (m: string) => void } | undefined
|
|
848
1005
|
return definePlugin(import.meta, {
|
|
@@ -852,6 +1009,7 @@ export function turboCache(options: TurboCacheOptions = {}): VxPlugin {
|
|
|
852
1009
|
Bun.env,
|
|
853
1010
|
remoteCacheOf(ctx.workspaceRoot),
|
|
854
1011
|
localConfigOf(ctx.workspaceRoot),
|
|
1012
|
+
(m) => ctx.warn(`[vx] plugin '@vzn/vx-migrate': ${m}`),
|
|
855
1013
|
)
|
|
856
1014
|
if (config === undefined) return undefined
|
|
857
1015
|
// A signed download waits in a temp until its tag checks: in vx's
|
|
@@ -859,6 +1017,7 @@ export function turboCache(options: TurboCacheOptions = {}): VxPlugin {
|
|
|
859
1017
|
// sandboxed task may read (L-45).
|
|
860
1018
|
const layer = new TurboRemoteCache(config, fetch, ctx.cacheDir)
|
|
861
1019
|
layer.reads = readScopes(ctx.refs)
|
|
1020
|
+
layer.lent = ctx.lent
|
|
862
1021
|
if (config.oidc !== undefined) opened = { layer, warn: ctx.warn }
|
|
863
1022
|
return scope === 'key' ? keyedByRef(layer, ctx.refs) : layer
|
|
864
1023
|
},
|