@vzn/vx-migrate 0.0.646 → 0.0.649
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -61
- package/package.json +3 -3
- package/src/adopt.ts +53 -12
- package/src/bin.ts +21 -1
- package/src/glob-grammar.ts +82 -0
- package/src/migrate-nx.ts +7 -4
- package/src/migrate-scripts.ts +10 -102
- package/src/migrate.ts +53 -25
- package/src/nx/index.ts +19 -5
- package/src/nx/nx-inputs.ts +44 -9
- package/src/nx/nx-map.ts +36 -11
- package/src/nx/nx-outputs.ts +16 -6
- package/src/nx/nx-upstream.ts +10 -2
- package/src/nx-cache/index.ts +60 -50
- package/src/nx-exec.cjs +2 -2
- package/src/package-fs.ts +6 -1
- package/src/packages.ts +9 -7
- package/src/plugin-gaps.ts +32 -6
- package/src/remote-cache.ts +5 -1
- package/src/remote-deadline.ts +7 -1
- package/src/remote-refusal.ts +31 -0
- package/src/remote-retry.ts +9 -4
- package/src/remote-url.ts +28 -0
- package/src/script-command.ts +110 -4
- package/src/shared-outputs.ts +8 -2
- package/src/tracked-outputs.ts +1 -1
- package/src/turbo/index.ts +18 -8
- package/src/turbo/turbo-map.ts +89 -15
- package/src/turbo-cache/index.ts +205 -89
- package/src/turbo-cache/oidc.ts +25 -14
- package/src/turbo-cache/scope.ts +5 -3
- package/src/vite-task/vite-task-map.ts +11 -6
- package/src/workspace-plugins.ts +5 -2
package/src/turbo-cache/index.ts
CHANGED
|
@@ -23,9 +23,12 @@ import {
|
|
|
23
23
|
type RemoteOpenContext,
|
|
24
24
|
type VxPlugin,
|
|
25
25
|
refuseUnknownOptions,
|
|
26
|
+
UserError,
|
|
26
27
|
type PluginOptionKinds,
|
|
27
28
|
} from '@vzn/vx'
|
|
28
|
-
import { deadline } from '../remote-deadline.js'
|
|
29
|
+
import { deadline, MAX_TIMER_MS } from '../remote-deadline.js'
|
|
30
|
+
import { checkRemoteUrl } from '../remote-url.js'
|
|
31
|
+
import { refusal } from '../remote-refusal.js'
|
|
29
32
|
import { keyedByRef, readScopes, remoteScope, type RemoteScope } from './scope.js'
|
|
30
33
|
import { OutageBreaker } from '../remote-breaker.js'
|
|
31
34
|
import { Cooldown, withRetry } from '../remote-retry.js'
|
|
@@ -45,7 +48,7 @@ import {
|
|
|
45
48
|
|
|
46
49
|
export interface TurboCacheOptions {
|
|
47
50
|
/** Base URL of the cache server (`https://cache.example.com`), or `TURBO_API`; with a token and neither, Vercel's hosted cache, as for `turbo`. */
|
|
48
|
-
|
|
51
|
+
url?: string
|
|
49
52
|
/** Bearer token every request carries, or `TURBO_TOKEN`, OIDC, `.turbo/config.json`, then the OS keychain. */
|
|
50
53
|
token?: string
|
|
51
54
|
/**
|
|
@@ -55,7 +58,7 @@ export interface TurboCacheOptions {
|
|
|
55
58
|
* its `tokenUrl`. The job needs `permissions: id-token: write`.
|
|
56
59
|
*/
|
|
57
60
|
oidc?: true | TurboOidcOptions
|
|
58
|
-
/** `teamId` query parameter, or `TURBO_TEAMID
|
|
61
|
+
/** `teamId` query parameter, or `TURBO_TEAMID`; the signature folds it, `''` when unset, as Turbo's does. */
|
|
59
62
|
teamId?: string
|
|
60
63
|
/** `slug` query parameter (a team slug), or `TURBO_TEAM`. */
|
|
61
64
|
teamSlug?: string
|
|
@@ -63,8 +66,10 @@ export interface TurboCacheOptions {
|
|
|
63
66
|
* Sign uploads and verify downloads with Turbo's artifact signature
|
|
64
67
|
* (HMAC-SHA256, `x-artifact-tag`), or `TURBO_REMOTE_CACHE_SIGNATURE_KEY`
|
|
65
68
|
* where turbo.json sets `remoteCache.signature: true`, as Turbo reads it.
|
|
66
|
-
*
|
|
67
|
-
*
|
|
69
|
+
* The option needs at least 32 bytes; the env key warns below that,
|
|
70
|
+
* refused only under turbo.json's `futureFlags.longerSignatureKey`, as
|
|
71
|
+
* in Turbo. Used raw. A download whose tag does not verify is a miss,
|
|
72
|
+
* never a restore.
|
|
68
73
|
*/
|
|
69
74
|
signatureKey?: string
|
|
70
75
|
/**
|
|
@@ -103,6 +108,8 @@ export interface TurboJsonRemoteCache {
|
|
|
103
108
|
timeout?: unknown
|
|
104
109
|
uploadTimeout?: unknown
|
|
105
110
|
preflight?: unknown
|
|
111
|
+
/** turbo.json's `futureFlags.longerSignatureKey`, which `remoteCacheOf` hoists here. */
|
|
112
|
+
longerSignatureKey?: unknown
|
|
106
113
|
}
|
|
107
114
|
|
|
108
115
|
/** `.turbo/config.json`'s fields, under any of the spellings Turbo accepts. */
|
|
@@ -114,7 +121,7 @@ export interface TurboLocalConfig {
|
|
|
114
121
|
}
|
|
115
122
|
|
|
116
123
|
export interface TurboCacheConfig {
|
|
117
|
-
|
|
124
|
+
url: string
|
|
118
125
|
/** Undefined: the layer's `open` exchanges OIDC or asks the OS keychain. */
|
|
119
126
|
token?: string
|
|
120
127
|
/** The job's OIDC exchange, when `oidc` is set and GitHub hands the job a token. */
|
|
@@ -196,6 +203,7 @@ function trackTemp(file: string): void {
|
|
|
196
203
|
for (const f of liveTemps) {
|
|
197
204
|
try {
|
|
198
205
|
unlinkSync(f)
|
|
206
|
+
// quiet: a temp file already gone at exit
|
|
199
207
|
} catch {
|
|
200
208
|
// never written, or gone
|
|
201
209
|
}
|
|
@@ -206,6 +214,7 @@ function trackTemp(file: string): void {
|
|
|
206
214
|
async function removeTemp(file: string): Promise<void> {
|
|
207
215
|
// Listed until the unlink lands: an exit in between would otherwise find
|
|
208
216
|
// nothing to remove and leave the temp (item 868).
|
|
217
|
+
// quiet: a temp file already gone
|
|
209
218
|
await unlink(file).catch(() => undefined)
|
|
210
219
|
liveTemps.delete(file)
|
|
211
220
|
}
|
|
@@ -246,7 +255,7 @@ function tagsEqual(expected: string, actual: string): boolean {
|
|
|
246
255
|
return diff === 0
|
|
247
256
|
}
|
|
248
257
|
|
|
249
|
-
/** Where `turbo` itself sends a token with no `
|
|
258
|
+
/** Where `turbo` itself sends a token with no `url`: Vercel's hosted Remote Cache. */
|
|
250
259
|
export const VERCEL_API = 'https://vercel.com/api'
|
|
251
260
|
|
|
252
261
|
/** Resolve options over Turbo's environment; `undefined` = not configured. */
|
|
@@ -256,25 +265,28 @@ export function resolveTurboCacheConfig(
|
|
|
256
265
|
file: TurboJsonRemoteCache = {},
|
|
257
266
|
/** The repo's `.turbo/config.json` (`turbo link` writes it): below the environment, above turbo.json. */
|
|
258
267
|
local: TurboLocalConfig = {},
|
|
268
|
+
warn: (message: string) => void = (m) => console.error(`[vx] plugin '@vzn/vx-migrate': ${m}`),
|
|
259
269
|
): TurboCacheConfig | undefined {
|
|
260
270
|
// `enabled: false` turns Turbo's remote cache off whatever the env says;
|
|
261
271
|
// options name a cache for vx alone, and win.
|
|
262
|
-
if (file.enabled === false && options.
|
|
272
|
+
if (file.enabled === false && options.url === undefined && options.token === undefined)
|
|
263
273
|
return undefined
|
|
264
274
|
const fromFile = (v: unknown): string | undefined => (typeof v === 'string' ? v : undefined)
|
|
275
|
+
// Empty is unset, as Turbo reads it: a fork's pull request gets `''` for a
|
|
276
|
+
// secret, and `${{ vars.TURBO_API }}` is `''` for an unset variable.
|
|
277
|
+
const set = (name: string): string | undefined => env[name] || undefined
|
|
265
278
|
// A Vercel build hands Turbo its cache as VERCEL_ARTIFACTS_TOKEN and
|
|
266
279
|
// _OWNER (the team id), taken where TURBO_TOKEN with a team is not set
|
|
267
280
|
// (Turbo's `override_env.rs`); unread, turboCache() declined on Vercel.
|
|
268
|
-
const turboPair =
|
|
281
|
+
const turboPair = set('TURBO_TOKEN') && (set('TURBO_TEAMID') || set('TURBO_TEAM'))
|
|
269
282
|
const vercel =
|
|
270
|
-
!turboPair &&
|
|
271
|
-
? { token:
|
|
272
|
-
: { token: undefined, teamId: turboPair ? undefined :
|
|
273
|
-
|
|
274
|
-
const given = (options.token ?? env['TURBO_TOKEN'] ?? vercel.token) || undefined
|
|
283
|
+
!turboPair && set('VERCEL_ARTIFACTS_TOKEN') && set('VERCEL_ARTIFACTS_OWNER')
|
|
284
|
+
? { token: set('VERCEL_ARTIFACTS_TOKEN'), teamId: set('VERCEL_ARTIFACTS_OWNER') }
|
|
285
|
+
: { token: undefined, teamId: turboPair ? undefined : set('VERCEL_ARTIFACTS_OWNER') }
|
|
286
|
+
const given = (options.token ?? set('TURBO_TOKEN') ?? vercel.token) || undefined
|
|
275
287
|
// OIDC sits under the variables and over the config files.
|
|
276
|
-
const idTokenUrl =
|
|
277
|
-
const idTokenBearer =
|
|
288
|
+
const idTokenUrl = set('ACTIONS_ID_TOKEN_REQUEST_URL')
|
|
289
|
+
const idTokenBearer = set('ACTIONS_ID_TOKEN_REQUEST_TOKEN')
|
|
278
290
|
const exchanges =
|
|
279
291
|
options.oidc !== undefined &&
|
|
280
292
|
given === undefined &&
|
|
@@ -283,41 +295,39 @@ export function resolveTurboCacheConfig(
|
|
|
283
295
|
const token = given ?? (exchanges ? undefined : fromFile(local.token) || undefined)
|
|
284
296
|
const teamId =
|
|
285
297
|
options.teamId ??
|
|
286
|
-
|
|
298
|
+
set('TURBO_TEAMID') ??
|
|
287
299
|
vercel.teamId ??
|
|
288
300
|
fromFile(local.teamId) ??
|
|
289
301
|
fromFile(file.teamId)
|
|
290
302
|
const teamSlug =
|
|
291
|
-
options.teamSlug ??
|
|
292
|
-
// Turbo's own default when a token or a team is set and no
|
|
303
|
+
options.teamSlug ?? set('TURBO_TEAM') ?? fromFile(local.teamSlug) ?? fromFile(file.teamSlug)
|
|
304
|
+
// Turbo's own default when a token or a team is set and no URL is:
|
|
293
305
|
// Vercel's hosted Remote Cache. A token alone is a configured cache, as it
|
|
294
306
|
// is for `turbo`; a URL or a team without one asks the OS keychain at
|
|
295
307
|
// `open`; none of them is the declined, local run.
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
308
|
+
// Each source with its name: TURBO_API was refused as a `url`
|
|
309
|
+
// option the workspace never set.
|
|
310
|
+
const [rawUrl, urlFrom] =
|
|
311
|
+
options.url !== undefined
|
|
312
|
+
? [options.url, 'turboCache() option "url"']
|
|
313
|
+
: set('TURBO_API') !== undefined
|
|
314
|
+
? [set('TURBO_API'), 'TURBO_API']
|
|
315
|
+
: fromFile(local.apiUrl) !== undefined
|
|
316
|
+
? [fromFile(local.apiUrl), '.turbo/config.json apiUrl']
|
|
317
|
+
: fromFile(file.apiUrl) !== undefined
|
|
318
|
+
? [fromFile(file.apiUrl), 'turbo.json remoteCache.apiUrl']
|
|
319
|
+
: [token || teamId || teamSlug ? VERCEL_API : undefined, 'url']
|
|
320
|
+
const url = rawUrl?.replace(/\/+$/, '')
|
|
321
|
+
if (!url) return undefined
|
|
322
|
+
checkRemoteUrl(url, urlFrom)
|
|
304
323
|
const oidc =
|
|
305
324
|
options.oidc === undefined
|
|
306
325
|
? undefined
|
|
307
|
-
: oidcSettings(options.oidc,
|
|
308
|
-
// The URL is printed in every refusal line, so a `user:pass@` in it would
|
|
309
|
-
// leak to the log. Credentials go in the token.
|
|
310
|
-
if (URL.canParse(apiUrl)) {
|
|
311
|
-
const u = new URL(apiUrl)
|
|
312
|
-
if (u.username !== '' || u.password !== '')
|
|
313
|
-
throw new Error(
|
|
314
|
-
'vx/turbo-cache: apiUrl carries credentials (user:pass@); pass them as the token instead',
|
|
315
|
-
)
|
|
316
|
-
}
|
|
326
|
+
: oidcSettings(options.oidc, url === VERCEL_API, teamId ?? teamSlug)
|
|
317
327
|
const fault = token === undefined ? null : headerValueFault(token)
|
|
318
328
|
if (fault !== null)
|
|
319
329
|
throw new Error(
|
|
320
|
-
`
|
|
330
|
+
`turboCache(): the token holds ${fault}, which no HTTP header can carry — check the secret (it is not printed)`,
|
|
321
331
|
)
|
|
322
332
|
const preflight =
|
|
323
333
|
options.preflight ??
|
|
@@ -325,34 +335,34 @@ export function resolveTurboCacheConfig(
|
|
|
325
335
|
file.preflight === true
|
|
326
336
|
// Turbo signs only under `remoteCache.signature: true`; the env key alone
|
|
327
337
|
// signs nothing there, and a short one here refused the whole cache.
|
|
328
|
-
|
|
329
|
-
const
|
|
330
|
-
const signatureKey =
|
|
331
|
-
|
|
338
|
+
const signing = file.signature === true
|
|
339
|
+
const envKey = signing ? env['TURBO_REMOTE_CACHE_SIGNATURE_KEY'] || undefined : undefined
|
|
340
|
+
const signatureKey = options.signatureKey ?? envKey
|
|
341
|
+
// Turbo signs with no key as it fails every request: no usable remote. Read
|
|
342
|
+
// unsigned here, the bytes a signing repo refuses were restored.
|
|
343
|
+
if (signing && signatureKey === undefined) {
|
|
344
|
+
warn(
|
|
345
|
+
'turbo.json sets remoteCache.signature but TURBO_REMOTE_CACHE_SIGNATURE_KEY is unset: remote cache off for this run, as for turbo',
|
|
346
|
+
)
|
|
347
|
+
return undefined
|
|
348
|
+
}
|
|
332
349
|
if (signatureKey !== undefined) {
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
350
|
+
const bytes = Buffer.byteLength(signatureKey)
|
|
351
|
+
// Turbo only warns about a short key unless `futureFlags.longerSignatureKey`
|
|
352
|
+
// is set; refusing it here failed a run Turbo runs.
|
|
353
|
+
if (bytes < MIN_SIGNATURE_KEY_LENGTH) {
|
|
354
|
+
if (options.signatureKey !== undefined || file.longerSignatureKey === true)
|
|
355
|
+
throw new Error(
|
|
356
|
+
`turboCache(): signatureKey must be at least ${MIN_SIGNATURE_KEY_LENGTH} bytes`,
|
|
357
|
+
)
|
|
358
|
+
warn(
|
|
359
|
+
`TURBO_REMOTE_CACHE_SIGNATURE_KEY is ${bytes} bytes; ${MIN_SIGNATURE_KEY_LENGTH} or more is safe (Turbo enforces it under futureFlags.longerSignatureKey)`,
|
|
336
360
|
)
|
|
337
361
|
}
|
|
338
|
-
if (!teamId)
|
|
339
|
-
throw new Error(
|
|
340
|
-
'vx/turbo-cache: signatureKey needs teamId — the team id is part of the signed message',
|
|
341
|
-
)
|
|
342
362
|
}
|
|
343
363
|
const retries = options.retries ?? 1
|
|
344
|
-
// A NaN never reaches the bound, and the request would be resent forever.
|
|
345
|
-
if (!Number.isInteger(retries) || retries < 0)
|
|
346
|
-
throw new Error(`vx/turbo-cache: retries must be a whole number ≥ 0, got ${retries}`)
|
|
347
|
-
// 0 is no deadline, as Turbo reads it; a negative or non-number one made
|
|
348
|
-
// AbortSignal.timeout throw on every request.
|
|
349
|
-
for (const name of ['timeoutMs', 'uploadTimeoutMs'] as const) {
|
|
350
|
-
const ms = options[name]
|
|
351
|
-
if (ms !== undefined && !(typeof ms === 'number' && Number.isFinite(ms) && ms >= 0))
|
|
352
|
-
throw new Error(`vx/turbo-cache: ${name} must be ms ≥ 0 (0: none), got ${JSON.stringify(ms)}`)
|
|
353
|
-
}
|
|
354
364
|
return {
|
|
355
|
-
|
|
365
|
+
url,
|
|
356
366
|
...(token !== undefined ? { token } : {}),
|
|
357
367
|
...(oidc !== undefined && exchanges
|
|
358
368
|
? { oidc: { ...oidc, idTokenUrl: idTokenUrl!, idTokenBearer: idTokenBearer! } }
|
|
@@ -382,7 +392,7 @@ export function resolveTurboCacheConfig(
|
|
|
382
392
|
function truthy(v: string | undefined, name: string): boolean | undefined {
|
|
383
393
|
if (v === undefined || v === '') return undefined
|
|
384
394
|
if (!/^(1|0|true|false)$/.test(v))
|
|
385
|
-
throw new Error(`
|
|
395
|
+
throw new Error(`turboCache(): ${name} should be 1 or 0, got ${JSON.stringify(v)}`)
|
|
386
396
|
return v === '1' || v === 'true'
|
|
387
397
|
}
|
|
388
398
|
|
|
@@ -394,7 +404,7 @@ function seconds(v: unknown, name: string): number | undefined {
|
|
|
394
404
|
if (v === undefined || v === null || v === '') return undefined
|
|
395
405
|
const n = typeof v === 'string' ? Number(v) : v
|
|
396
406
|
if (typeof n !== 'number' || !Number.isInteger(n) || n < 0)
|
|
397
|
-
throw new Error(`
|
|
407
|
+
throw new Error(`turboCache(): ${name} must be whole seconds ≥ 0, got ${JSON.stringify(v)}`)
|
|
398
408
|
return n * 1000
|
|
399
409
|
}
|
|
400
410
|
|
|
@@ -432,6 +442,16 @@ const MAX_SIGNED_BODY = 2 * 1024 ** 3 + ((2 * 1024 ** 3) >> 8) + 64 * 1024
|
|
|
432
442
|
/** A lent token ending sooner than this is exchanged again first, so an agent's request never carries a spent one. */
|
|
433
443
|
const LEND_MARGIN_MS = 5 * 60_000
|
|
434
444
|
|
|
445
|
+
/** Statuses of a server that has no batch query. */
|
|
446
|
+
const NO_BATCH = new Set([404, 405, 501])
|
|
447
|
+
|
|
448
|
+
/**
|
|
449
|
+
* The most hashes one batch query may carry: Vercel answers more with 400
|
|
450
|
+
* "Too many hashes. Limit is 100." (main f71b377 asked 107 at once, and
|
|
451
|
+
* the run planned as if none were held).
|
|
452
|
+
*/
|
|
453
|
+
const BATCH_MAX = 100
|
|
454
|
+
|
|
435
455
|
/**
|
|
436
456
|
* The seam implementation: `has` is HEAD, `hasMany` is the batch query,
|
|
437
457
|
* `get`/`put` carry `x-artifact-duration` (and the tag when signing). An
|
|
@@ -462,6 +482,13 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
462
482
|
* entry is immutable, so any other key is asked first.
|
|
463
483
|
*/
|
|
464
484
|
private readonly absent = new Set<string>()
|
|
485
|
+
/**
|
|
486
|
+
* Keys whose GET this process answered 200. Core saves a key after its
|
|
487
|
+
* hit only when that hit did not restore (bad bytes, a tag that does not
|
|
488
|
+
* verify), so `put` sends it over the stored one: its HEAD saw 200 and
|
|
489
|
+
* kept the bad entry for every later run.
|
|
490
|
+
*/
|
|
491
|
+
private readonly served = new Set<string>()
|
|
465
492
|
private readonly key: Uint8Array | undefined
|
|
466
493
|
readonly endpoint: string
|
|
467
494
|
/** Undefined until `open` reads the keychain; a layer never opened (a plan) asks the server nothing. */
|
|
@@ -490,12 +517,12 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
490
517
|
private readonly maxSignedBody: number = MAX_SIGNED_BODY,
|
|
491
518
|
) {
|
|
492
519
|
this.key = config.signatureKey === undefined ? undefined : Buffer.from(config.signatureKey)
|
|
493
|
-
this.endpoint = `${config.
|
|
520
|
+
this.endpoint = `${config.url}/v8/artifacts`
|
|
494
521
|
this.token = config.token
|
|
495
522
|
const team = config.teamId ?? config.teamSlug
|
|
496
523
|
this.keychainRemote = {
|
|
497
524
|
key: team === undefined ? this.endpoint : `${this.endpoint}#${team}`,
|
|
498
|
-
label: remoteLabel(config.
|
|
525
|
+
label: remoteLabel(config.url, team),
|
|
499
526
|
env: 'TURBO_TOKEN',
|
|
500
527
|
}
|
|
501
528
|
}
|
|
@@ -555,7 +582,7 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
555
582
|
}
|
|
556
583
|
|
|
557
584
|
private url(pathname: string): string {
|
|
558
|
-
const u = new URL(`${this.config.
|
|
585
|
+
const u = new URL(`${this.config.url}/v8/artifacts${pathname}`)
|
|
559
586
|
// Turbo sends a team id only in Vercel's `team_` form (its API client's
|
|
560
587
|
// `add_team_params`); a personal account's owner id is not one, and the
|
|
561
588
|
// signature, which folds the id as given, is unaffected.
|
|
@@ -574,7 +601,7 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
574
601
|
|
|
575
602
|
/**
|
|
576
603
|
* Turbo's preflight: an `OPTIONS` to the request's URL names the method
|
|
577
|
-
* and headers to come; the answer's `Location` (relative to `
|
|
604
|
+
* and headers to come; the answer's `Location` (relative to `url`) is
|
|
578
605
|
* where the request goes, as given — a signed storage URL signs its own
|
|
579
606
|
* query, so nothing is appended — and the token goes along only when
|
|
580
607
|
* `Access-Control-Allow-Headers` admits `Authorization` (or `*`), so it
|
|
@@ -604,7 +631,7 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
604
631
|
const location = res.headers.get('location')
|
|
605
632
|
const allowed = res.headers.get('access-control-allow-headers') ?? ''
|
|
606
633
|
return {
|
|
607
|
-
url: location === null ? url : new URL(location, this.config.
|
|
634
|
+
url: location === null ? url : new URL(location, this.config.url).toString(),
|
|
608
635
|
auth:
|
|
609
636
|
allowed.trim() === '*' ||
|
|
610
637
|
allowed.split(',').some((h) => h.trim().toLowerCase() === 'authorization'),
|
|
@@ -654,19 +681,22 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
654
681
|
const first = !this.writesRefused
|
|
655
682
|
this.writesRefused = true
|
|
656
683
|
if (!first) return undefined
|
|
684
|
+
// The status and the server's own words (`refusal`): a scope it
|
|
685
|
+
// refuses and a token it no longer takes read alike.
|
|
657
686
|
throw new Error(
|
|
658
|
-
|
|
687
|
+
`${(await refusal(res)).message}: the upload was refused (a read-only token or server); uploads off for this run, reads go on`,
|
|
659
688
|
)
|
|
660
689
|
}
|
|
661
690
|
if (res.status === 401 || res.status === 403) {
|
|
662
691
|
const first = !this.disabled
|
|
663
692
|
this.disabled = true
|
|
664
693
|
if (!first) return undefined
|
|
694
|
+
const said = (await refusal(res)).message
|
|
665
695
|
if (this.forget === undefined)
|
|
666
|
-
throw new Error(
|
|
696
|
+
throw new Error(`${said}: the token was refused; remote cache off for this run`)
|
|
667
697
|
await this.forget()
|
|
668
698
|
throw new Error(
|
|
669
|
-
|
|
699
|
+
`${said}: the token was refused; removed it from the OS keychain, so the next run asks again; remote cache off for this run`,
|
|
670
700
|
)
|
|
671
701
|
}
|
|
672
702
|
return res
|
|
@@ -692,26 +722,66 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
692
722
|
throw new Error(`HTTP ${res.status}`)
|
|
693
723
|
}
|
|
694
724
|
|
|
725
|
+
/**
|
|
726
|
+
* Null when the layer is off: no answer, which core keeps apart from
|
|
727
|
+
* "none held" (an empty map rewrote every key it asked as a miss in
|
|
728
|
+
* `remote_hits`, and the next run fetched none of them).
|
|
729
|
+
*/
|
|
695
730
|
async hasMany(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
|
|
696
|
-
if (this.disabled || hashes.length === 0) return
|
|
731
|
+
if (this.disabled || hashes.length === 0) return null
|
|
732
|
+
if (hashes.length <= BATCH_MAX) return this.batch(hashes)
|
|
733
|
+
const parts = await Promise.all(
|
|
734
|
+
Array.from({ length: Math.ceil(hashes.length / BATCH_MAX) }, (_, i) =>
|
|
735
|
+
this.batch(hashes.slice(i * BATCH_MAX, (i + 1) * BATCH_MAX)),
|
|
736
|
+
),
|
|
737
|
+
)
|
|
738
|
+
if (parts.some((p) => p === null)) return null
|
|
739
|
+
return new Map(parts.flatMap((p) => [...p!]))
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
private async batch(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
|
|
697
743
|
const res = await this.request('POST', '', {
|
|
698
744
|
body: JSON.stringify({ hashes }),
|
|
699
745
|
headers: { 'Content-Type': 'application/json', ...this.reads },
|
|
700
746
|
})
|
|
701
|
-
if (res === undefined) return
|
|
702
|
-
if (res.status !== 200)
|
|
747
|
+
if (res === undefined) return null
|
|
748
|
+
if (res.status !== 200) {
|
|
749
|
+
// A server with no batch query (404, 405, 501) is asked per hash. Any
|
|
750
|
+
// other refusal was silent: main f71b377's probe gave no answer and no
|
|
751
|
+
// one could say why, and DTE sized the run as if every task ran.
|
|
752
|
+
if (NO_BATCH.has(res.status)) {
|
|
753
|
+
await res.body?.cancel()
|
|
754
|
+
return null
|
|
755
|
+
}
|
|
756
|
+
throw await refusal(res, 'batch query answered ')
|
|
757
|
+
}
|
|
703
758
|
// An answer per hash is a few hundred bytes of ArtifactInfo; a reply
|
|
704
759
|
// past this bound is no answer, read no further than the bound (L-9).
|
|
705
760
|
const bound = hashes.length * 4096 + 64 * 1024
|
|
706
761
|
const text = await boundedText(res, bound)
|
|
707
|
-
if (text === null)
|
|
708
|
-
|
|
762
|
+
if (text === null) throw new Error(`batch query answered past ${bound} bytes; read no further`)
|
|
763
|
+
// A 200 that is no JSON object (a proxy's page, `null`, `[]`) threw a
|
|
764
|
+
// parser's line that named no query, or read as every hash absent:
|
|
765
|
+
// DTE sized the run as if every task ran.
|
|
766
|
+
let info: unknown
|
|
767
|
+
try {
|
|
768
|
+
info = JSON.parse(text)
|
|
769
|
+
} catch {
|
|
770
|
+
// quiet: refused below, with what came
|
|
771
|
+
info = undefined
|
|
772
|
+
}
|
|
773
|
+
if (typeof info !== 'object' || info === null || Array.isArray(info)) {
|
|
774
|
+
const said = text.replace(/\s+/g, ' ').trim().slice(0, 200)
|
|
775
|
+
throw new Error(
|
|
776
|
+
`batch query answered HTTP 200 with no JSON object${said === '' ? '' : `: ${said}`}`,
|
|
777
|
+
)
|
|
778
|
+
}
|
|
709
779
|
// Each hash answers ArtifactInfo, null, or `{ error: { message } }`; an
|
|
710
780
|
// error entry is not a stored artifact, and it was counted as one (a GET
|
|
711
781
|
// spent on a 404, item 929).
|
|
712
782
|
const held = new Map<string, ArtifactInfo>()
|
|
713
783
|
for (const h of hashes) {
|
|
714
|
-
const v = info[h]
|
|
784
|
+
const v = (info as Record<string, unknown>)[h]
|
|
715
785
|
if (typeof v !== 'object' || v === null || 'error' in v) this.absent.add(h)
|
|
716
786
|
else {
|
|
717
787
|
const { size, taskDurationMs } = v as { size?: unknown; taskDurationMs?: unknown }
|
|
@@ -730,13 +800,14 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
730
800
|
})
|
|
731
801
|
if (res === undefined) return null
|
|
732
802
|
if (res.status !== 200) {
|
|
733
|
-
await res.body?.cancel()
|
|
734
803
|
if (res.status === 404) {
|
|
804
|
+
await res.body?.cancel()
|
|
735
805
|
this.absent.add(hash)
|
|
736
806
|
return null
|
|
737
807
|
}
|
|
738
|
-
throw
|
|
808
|
+
throw await refusal(res)
|
|
739
809
|
}
|
|
810
|
+
this.served.add(hash)
|
|
740
811
|
const duration = Number(res.headers.get('x-artifact-duration'))
|
|
741
812
|
const durationMs = Number.isFinite(duration) && duration > 0 ? duration : undefined
|
|
742
813
|
if (this.key === undefined) return { body: res, durationMs }
|
|
@@ -784,9 +855,14 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
784
855
|
return new Response(unlinkingStream(temp))
|
|
785
856
|
}
|
|
786
857
|
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
858
|
+
missing(hash: string): void {
|
|
859
|
+
this.absent.add(hash)
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
async put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void | false> {
|
|
863
|
+
// Nothing sent, so no upload is counted.
|
|
864
|
+
if (this.disabled || this.writesRefused) return false
|
|
865
|
+
if (!this.absent.has(hash) && !this.served.has(hash)) {
|
|
790
866
|
// Part of the upload, so under its deadline. Any answer but 200 or 404
|
|
791
867
|
// uploads: a lost store costs more. A server the HEAD cannot reach
|
|
792
868
|
// fails the put, as the PUT would.
|
|
@@ -794,7 +870,8 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
794
870
|
headers: this.reads,
|
|
795
871
|
timeoutMs: this.config.uploadTimeoutMs,
|
|
796
872
|
})
|
|
797
|
-
if (head === undefined
|
|
873
|
+
if (head === undefined) return false
|
|
874
|
+
if (head.status === 200) return
|
|
798
875
|
if (head.status === 404) this.absent.add(hash)
|
|
799
876
|
}
|
|
800
877
|
const headers: Record<string, string> = {
|
|
@@ -810,11 +887,12 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
810
887
|
headers,
|
|
811
888
|
timeoutMs: this.config.uploadTimeoutMs,
|
|
812
889
|
})
|
|
813
|
-
if (res === undefined) return
|
|
890
|
+
if (res === undefined) return false
|
|
814
891
|
// Turbo's client takes any 2xx (`error_for_status`); a server's 201 or
|
|
815
892
|
// 204 stored the artifact and was counted a failed upload.
|
|
816
|
-
if (!res.ok) throw
|
|
893
|
+
if (!res.ok) throw await refusal(res)
|
|
817
894
|
this.absent.delete(hash)
|
|
895
|
+
this.served.delete(hash)
|
|
818
896
|
}
|
|
819
897
|
}
|
|
820
898
|
|
|
@@ -826,6 +904,7 @@ function localConfigOf(root: string): TurboLocalConfig {
|
|
|
826
904
|
let raw: unknown
|
|
827
905
|
try {
|
|
828
906
|
raw = JSON.parse(readFileSync(path.join(root, '.turbo', 'config.json'), 'utf8'))
|
|
907
|
+
// quiet: a local config file: none read is none linked
|
|
829
908
|
} catch {
|
|
830
909
|
return {}
|
|
831
910
|
}
|
|
@@ -845,12 +924,22 @@ function remoteCacheOf(root: string): TurboJsonRemoteCache {
|
|
|
845
924
|
let text: string
|
|
846
925
|
try {
|
|
847
926
|
text = readFileSync(path.join(root, name), 'utf8')
|
|
927
|
+
// quiet: a local config file: none read is none declared
|
|
848
928
|
} catch {
|
|
849
929
|
continue
|
|
850
930
|
}
|
|
851
931
|
try {
|
|
852
|
-
const
|
|
853
|
-
|
|
932
|
+
const json = Bun.JSONC.parse(text) as {
|
|
933
|
+
remoteCache?: unknown
|
|
934
|
+
futureFlags?: { longerSignatureKey?: unknown }
|
|
935
|
+
} | null
|
|
936
|
+
const rc = json?.remoteCache
|
|
937
|
+
const longerSignatureKey = json?.futureFlags?.longerSignatureKey
|
|
938
|
+
return {
|
|
939
|
+
...(typeof rc === 'object' && rc !== null ? (rc as TurboJsonRemoteCache) : {}),
|
|
940
|
+
...(longerSignatureKey === undefined ? {} : { longerSignatureKey }),
|
|
941
|
+
}
|
|
942
|
+
// quiet: a local config file: none read is none declared
|
|
854
943
|
} catch {
|
|
855
944
|
return {}
|
|
856
945
|
}
|
|
@@ -860,7 +949,7 @@ function remoteCacheOf(root: string): TurboJsonRemoteCache {
|
|
|
860
949
|
|
|
861
950
|
/** Each option `TurboCacheOptions` names, with its kind: derived from the type, so the two cannot drift. */
|
|
862
951
|
const TURBO_CACHE_KEYS: PluginOptionKinds<TurboCacheOptions> = {
|
|
863
|
-
|
|
952
|
+
url: 'string',
|
|
864
953
|
token: 'string',
|
|
865
954
|
teamId: 'string',
|
|
866
955
|
teamSlug: 'string',
|
|
@@ -873,18 +962,44 @@ const TURBO_CACHE_KEYS: PluginOptionKinds<TurboCacheOptions> = {
|
|
|
873
962
|
scope: 'string',
|
|
874
963
|
}
|
|
875
964
|
|
|
965
|
+
/**
|
|
966
|
+
* The number options, refused at the call: in the `cache` hook a laptop
|
|
967
|
+
* with no remote named passed what failed the first CI run.
|
|
968
|
+
*/
|
|
969
|
+
function refuseBadNumbers(options: TurboCacheOptions): void {
|
|
970
|
+
// A NaN never reaches the bound, and the request would be resent forever.
|
|
971
|
+
const { retries } = options
|
|
972
|
+
if (retries !== undefined && !(Number.isInteger(retries) && retries >= 0))
|
|
973
|
+
throw new UserError(`turboCache() option "retries" must be a whole number ≥ 0, got ${retries}`)
|
|
974
|
+
// 0 is no deadline, as Turbo reads it; a negative or non-number one made
|
|
975
|
+
// AbortSignal.timeout throw on every request, and one past the longest
|
|
976
|
+
// timer aborted each as it started (Turbo's own env values are held).
|
|
977
|
+
for (const name of ['timeoutMs', 'uploadTimeoutMs'] as const) {
|
|
978
|
+
const ms = options[name]
|
|
979
|
+
if (ms !== undefined && !(ms >= 0 && ms <= MAX_TIMER_MS))
|
|
980
|
+
throw new UserError(
|
|
981
|
+
`turboCache() option "${name}" must be ms ≥ 0 (0: none) up to ${MAX_TIMER_MS}, got ${ms}`,
|
|
982
|
+
)
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
|
|
876
986
|
/**
|
|
877
987
|
* Declare in `.vx/workspace.ts`; the local store stays the floor beneath it:
|
|
878
988
|
*
|
|
879
989
|
* ```ts
|
|
880
|
-
* plugins: [turboCache({
|
|
990
|
+
* plugins: [turboCache({ url: 'https://cache.example.com', token: process.env.CACHE_TOKEN })]
|
|
881
991
|
* ```
|
|
882
992
|
*
|
|
883
993
|
* Declines with no URL, team or token, so it is safe to leave declared.
|
|
884
994
|
*/
|
|
885
995
|
export function turboCache(options: TurboCacheOptions = {}): VxPlugin {
|
|
886
996
|
refuseUnknownOptions('turboCache()', options, TURBO_CACHE_KEYS)
|
|
887
|
-
|
|
997
|
+
refuseBadNumbers(options)
|
|
998
|
+
// At the call too, as vxCache() checks its url: in the hook alone, a verb
|
|
999
|
+
// that opens no cache (`vx show`) took a url every run then failed on.
|
|
1000
|
+
const url = options.url?.replace(/\/+$/, '')
|
|
1001
|
+
if (url) checkRemoteUrl(url, 'turboCache() option "url"')
|
|
1002
|
+
const scope = remoteScope('turboCache', options.scope)
|
|
888
1003
|
// The run's layer and warn, for `teardown` to revoke an exchanged token.
|
|
889
1004
|
let opened: { layer: TurboRemoteCache; warn: (m: string) => void } | undefined
|
|
890
1005
|
return definePlugin(import.meta, {
|
|
@@ -894,6 +1009,7 @@ export function turboCache(options: TurboCacheOptions = {}): VxPlugin {
|
|
|
894
1009
|
Bun.env,
|
|
895
1010
|
remoteCacheOf(ctx.workspaceRoot),
|
|
896
1011
|
localConfigOf(ctx.workspaceRoot),
|
|
1012
|
+
(m) => ctx.warn(`[vx] plugin '@vzn/vx-migrate': ${m}`),
|
|
897
1013
|
)
|
|
898
1014
|
if (config === undefined) return undefined
|
|
899
1015
|
// A signed download waits in a temp until its tag checks: in vx's
|
package/src/turbo-cache/oidc.ts
CHANGED
|
@@ -48,13 +48,11 @@ export function oidcSettings(
|
|
|
48
48
|
): Omit<OidcConfig, 'idTokenUrl' | 'idTokenBearer'> {
|
|
49
49
|
const o = oidc === true ? {} : oidc
|
|
50
50
|
if (typeof o !== 'object' || o === null)
|
|
51
|
-
throw new Error(
|
|
52
|
-
'vx/turbo-cache: oidc must be true or { tokenUrl, revokeUrl, audience, params }',
|
|
53
|
-
)
|
|
51
|
+
throw new Error('turboCache(): oidc must be true or { tokenUrl, revokeUrl, audience, params }')
|
|
54
52
|
if (!vercel && o.tokenUrl === undefined)
|
|
55
|
-
throw new Error("
|
|
53
|
+
throw new Error("turboCache(): oidc needs tokenUrl for a cache that is not Vercel's")
|
|
56
54
|
if (vercel && team === undefined)
|
|
57
|
-
throw new Error('
|
|
55
|
+
throw new Error('turboCache(): oidc on Vercel needs teamId or teamSlug (TURBO_TEAM)')
|
|
58
56
|
const revokeUrl = o.revokeUrl ?? (vercel ? VERCEL_REVOKE_URL : undefined)
|
|
59
57
|
return {
|
|
60
58
|
tokenUrl: o.tokenUrl ?? VERCEL_TOKEN_URL,
|
|
@@ -101,11 +99,27 @@ const form = (fields: Record<string, string>) => ({
|
|
|
101
99
|
/** Where a URL points, never its query: GitHub's carries a runtime id. */
|
|
102
100
|
const where = (url: string) => (URL.canParse(url) ? new URL(url).host : 'the endpoint')
|
|
103
101
|
|
|
102
|
+
/**
|
|
103
|
+
* A refusal's own words: RFC 6749's `error` and `error_description`, or
|
|
104
|
+
* GitHub's `message`. Only those fields, which name a fault and never
|
|
105
|
+
* carry a token; any other body is left unread. "answered HTTP 400" alone
|
|
106
|
+
* did not say the audience or the grant was wrong.
|
|
107
|
+
*/
|
|
108
|
+
async function refused(res: Response, what: string): Promise<Error> {
|
|
109
|
+
// quiet: a body that is not JSON says nothing; the status still names the refusal
|
|
110
|
+
const body = (await res.json().catch(() => null)) as Record<string, unknown> | null
|
|
111
|
+
const text = (v: unknown): string | undefined =>
|
|
112
|
+
typeof v === 'string' && v.trim() !== ''
|
|
113
|
+
? v.replace(/\s+/g, ' ').trim().slice(0, 200)
|
|
114
|
+
: undefined
|
|
115
|
+
const said = [text(body?.['error']), text(body?.['error_description']) ?? text(body?.['message'])]
|
|
116
|
+
.filter((s) => s !== undefined)
|
|
117
|
+
.join(': ')
|
|
118
|
+
return new Error(`${what} answered HTTP ${res.status}${said === '' ? '' : `: ${said}`}`)
|
|
119
|
+
}
|
|
120
|
+
|
|
104
121
|
async function field(res: Response, name: string, what: string): Promise<string> {
|
|
105
|
-
if (!res.ok)
|
|
106
|
-
await res.body?.cancel()
|
|
107
|
-
throw new Error(`${what} answered HTTP ${res.status}`)
|
|
108
|
-
}
|
|
122
|
+
if (!res.ok) throw await refused(res, what)
|
|
109
123
|
const value = ((await res.json().catch(() => null)) as Record<string, unknown> | null)?.[name]
|
|
110
124
|
if (typeof value !== 'string' || value === '') throw new Error(`${what} sent no ${name}`)
|
|
111
125
|
return value
|
|
@@ -148,10 +162,7 @@ export async function exchangeOidc(
|
|
|
148
162
|
wait,
|
|
149
163
|
)
|
|
150
164
|
const what = `the token exchange (${where(cfg.tokenUrl)})`
|
|
151
|
-
if (!res.ok)
|
|
152
|
-
await res.body?.cancel()
|
|
153
|
-
throw new Error(`${what} answered HTTP ${res.status}`)
|
|
154
|
-
}
|
|
165
|
+
if (!res.ok) throw await refused(res, what)
|
|
155
166
|
const body = (await res.json().catch(() => null)) as Record<string, unknown> | null
|
|
156
167
|
const token = body?.['access_token']
|
|
157
168
|
if (typeof token !== 'string' || token === '') throw new Error(`${what} sent no access_token`)
|
|
@@ -181,6 +192,6 @@ export async function revokeOidc(
|
|
|
181
192
|
}),
|
|
182
193
|
wait,
|
|
183
194
|
)
|
|
195
|
+
if (!res.ok) throw await refused(res, `revoking at ${where(cfg.revokeUrl)}`)
|
|
184
196
|
await res.body?.cancel()
|
|
185
|
-
if (!res.ok) throw new Error(`revoking at ${where(cfg.revokeUrl)} answered HTTP ${res.status}`)
|
|
186
197
|
}
|