@vzn/vx-migrate 0.0.652 → 0.0.653
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -33
- package/package.json +2 -2
- package/src/migrate-scripts.ts +11 -5
- package/src/migrate-turbo.ts +8 -1
- package/src/migrate.ts +102 -18
- package/src/nx/index.ts +61 -11
- package/src/nx/nx-inputs.ts +48 -24
- package/src/nx/nx-map.ts +199 -14
- package/src/nx/nx-outputs.ts +6 -1
- package/src/nx/nx-upstream.ts +11 -11
- package/src/nx-cache/index.ts +42 -28
- package/src/packages.ts +51 -1
- package/src/remote-refusal.ts +49 -4
- package/src/shared-outputs.ts +64 -9
- package/src/tracked-outputs.ts +58 -19
- package/src/turbo/index.ts +2 -1
- package/src/turbo/turbo-map.ts +157 -22
- package/src/turbo-cache/index.ts +71 -40
- package/src/turbo-cache/oidc.ts +15 -7
- package/src/workspace-notes.ts +19 -8
- package/src/workspace-plugins.ts +11 -3
package/src/turbo-cache/index.ts
CHANGED
|
@@ -29,7 +29,7 @@ import {
|
|
|
29
29
|
} from '@vzn/vx'
|
|
30
30
|
import { deadline, MAX_TIMER_MS } from '../remote-deadline.js'
|
|
31
31
|
import { checkRemoteUrl } from '../remote-url.js'
|
|
32
|
-
import { refusal } from '../remote-refusal.js'
|
|
32
|
+
import { hideTokens, hidingErrors, refusal } from '../remote-refusal.js'
|
|
33
33
|
import { keyedByRef, readScopes, remoteScope, type RemoteScope } from './scope.js'
|
|
34
34
|
import { OutageBreaker } from '../remote-breaker.js'
|
|
35
35
|
import { Cooldown, withRetry } from '../remote-retry.js'
|
|
@@ -588,8 +588,14 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
588
588
|
}
|
|
589
589
|
}
|
|
590
590
|
|
|
591
|
+
/** Every bearer a request carried: a refusal that quotes one says `***`. */
|
|
592
|
+
private readonly sent = new Set<string>()
|
|
593
|
+
private readonly hidden = (text: string): string => hideTokens(text, this.sent)
|
|
594
|
+
|
|
591
595
|
private bearer(): string | undefined {
|
|
592
|
-
|
|
596
|
+
const token = this.borrowed?.() ?? this.token
|
|
597
|
+
if (token !== undefined) this.sent.add(token)
|
|
598
|
+
return token
|
|
593
599
|
}
|
|
594
600
|
|
|
595
601
|
private url(pathname: string): string {
|
|
@@ -619,7 +625,9 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
619
625
|
* never reaches a third-party store. The answer's status is not read, as
|
|
620
626
|
* in Turbo. A redirect is not followed: its `Location` is the answer, and
|
|
621
627
|
* the token's verdict stays the API's. Following one let the redirected
|
|
622
|
-
* host's `Access-Control-Allow-Headers` admit the token to itself.
|
|
628
|
+
* host's `Access-Control-Allow-Headers` admit the token to itself. A
|
|
629
|
+
* `Location` that is not http(s), or plain http the token would go to
|
|
630
|
+
* from an https API, is a request error.
|
|
623
631
|
*/
|
|
624
632
|
private async preflight(
|
|
625
633
|
url: string,
|
|
@@ -641,12 +649,21 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
641
649
|
await res.body?.cancel()
|
|
642
650
|
const location = res.headers.get('location')
|
|
643
651
|
const allowed = res.headers.get('access-control-allow-headers') ?? ''
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
652
|
+
const auth =
|
|
653
|
+
allowed.trim() === '*' ||
|
|
654
|
+
allowed.split(',').some((h) => h.trim().toLowerCase() === 'authorization')
|
|
655
|
+
if (location === null) return { url, auth }
|
|
656
|
+
const to = new URL(location, this.config.url)
|
|
657
|
+
// `checkRemoteUrl` saw only the configured URL. A `file:` Location read
|
|
658
|
+
// a local file as the artifact, and a PUT to it "succeeded" storing
|
|
659
|
+
// nothing.
|
|
660
|
+
if (to.protocol !== 'http:' && to.protocol !== 'https:')
|
|
661
|
+
throw new Error(`the preflight pointed at ${to.protocol}, not an http(s) URL`)
|
|
662
|
+
if (auth && to.protocol === 'http:' && new URL(this.config.url).protocol === 'https:')
|
|
663
|
+
throw new Error(
|
|
664
|
+
`the preflight pointed an https API at ${to.origin}, which would send the token in the clear`,
|
|
665
|
+
)
|
|
666
|
+
return { url: to.toString(), auth }
|
|
650
667
|
}
|
|
651
668
|
|
|
652
669
|
/**
|
|
@@ -666,27 +683,30 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
666
683
|
const signal = () => (timeoutMs === 0 ? undefined : deadline(timeoutMs))
|
|
667
684
|
// The batch query is never preflighted, as in Turbo.
|
|
668
685
|
const preflight = this.config.preflight && method !== 'POST'
|
|
669
|
-
const res = await
|
|
670
|
-
(
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
686
|
+
const res = await hidingErrors(
|
|
687
|
+
this.breaker.send(
|
|
688
|
+
() =>
|
|
689
|
+
withRetry(
|
|
690
|
+
async () => {
|
|
691
|
+
const extra = init.headers ?? {}
|
|
692
|
+
const to = preflight
|
|
693
|
+
? await this.preflight(this.url(pathname), method, extra, signal)
|
|
694
|
+
: { url: this.url(pathname), auth: true }
|
|
695
|
+
const s = signal()
|
|
696
|
+
return this.fetchImpl(to.url, {
|
|
697
|
+
method,
|
|
698
|
+
headers: this.headers(extra, to.auth),
|
|
699
|
+
...(init.body === undefined ? {} : { body: init.body }),
|
|
700
|
+
...(s === undefined ? {} : { signal: s }),
|
|
701
|
+
})
|
|
702
|
+
},
|
|
703
|
+
this.config.retries,
|
|
704
|
+
this.wait,
|
|
705
|
+
this.cooldown,
|
|
706
|
+
),
|
|
707
|
+
method === 'GET',
|
|
708
|
+
),
|
|
709
|
+
this.hidden,
|
|
690
710
|
)
|
|
691
711
|
if (method === 'PUT' && res.status === 403) {
|
|
692
712
|
const first = !this.writesRefused
|
|
@@ -695,14 +715,14 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
695
715
|
// The status and the server's own words (`refusal`): a scope it
|
|
696
716
|
// refuses and a token it no longer takes read alike.
|
|
697
717
|
throw new Error(
|
|
698
|
-
`${(await refusal(res)).message}: the upload was refused (a read-only token or server); uploads off for this run, reads go on`,
|
|
718
|
+
`${(await refusal(res, this.hidden)).message}: the upload was refused (a read-only token or server); uploads off for this run, reads go on`,
|
|
699
719
|
)
|
|
700
720
|
}
|
|
701
721
|
if (res.status === 401 || res.status === 403) {
|
|
702
722
|
const first = !this.disabled
|
|
703
723
|
this.disabled = true
|
|
704
724
|
if (!first) return undefined
|
|
705
|
-
const said = (await refusal(res)).message
|
|
725
|
+
const said = (await refusal(res, this.hidden)).message
|
|
706
726
|
if (this.forget === undefined)
|
|
707
727
|
throw new Error(`${said}: the token was refused; remote cache off for this run`)
|
|
708
728
|
await this.forget()
|
|
@@ -764,7 +784,7 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
764
784
|
await res.body?.cancel()
|
|
765
785
|
return null
|
|
766
786
|
}
|
|
767
|
-
throw await refusal(res, 'batch query answered ')
|
|
787
|
+
throw await refusal(res, this.hidden, 'batch query answered ')
|
|
768
788
|
}
|
|
769
789
|
// An answer per hash is a few hundred bytes of ArtifactInfo; a reply
|
|
770
790
|
// past this bound is no answer, read no further than the bound (L-9).
|
|
@@ -802,21 +822,25 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
802
822
|
return held
|
|
803
823
|
}
|
|
804
824
|
|
|
805
|
-
async get(
|
|
806
|
-
|
|
825
|
+
async get(
|
|
826
|
+
hash: string,
|
|
827
|
+
): Promise<{ body: Response; durationMs: number | undefined } | null | undefined> {
|
|
828
|
+
// Off, or refused: no answer, which core keeps apart from a miss (a
|
|
829
|
+
// null forgot each hit the index kept, the remote never asked).
|
|
830
|
+
if (this.disabled) return undefined
|
|
807
831
|
// A gateway that keys binary media on Accept base64-encodes a `*/*`
|
|
808
832
|
// response, and every hit read as a corrupt artifact (nx#33092's class).
|
|
809
833
|
const res = await this.request('GET', `/${hash}`, {
|
|
810
834
|
headers: { Accept: 'application/octet-stream', ...this.reads },
|
|
811
835
|
})
|
|
812
|
-
if (res === undefined) return
|
|
836
|
+
if (res === undefined) return undefined
|
|
813
837
|
if (res.status !== 200) {
|
|
814
838
|
if (res.status === 404) {
|
|
815
839
|
await res.body?.cancel()
|
|
816
840
|
this.absent.add(hash)
|
|
817
841
|
return null
|
|
818
842
|
}
|
|
819
|
-
throw await refusal(res)
|
|
843
|
+
throw await refusal(res, this.hidden)
|
|
820
844
|
}
|
|
821
845
|
this.served.add(hash)
|
|
822
846
|
const duration = Number(res.headers.get('x-artifact-duration'))
|
|
@@ -870,10 +894,17 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
870
894
|
this.absent.add(hash)
|
|
871
895
|
}
|
|
872
896
|
|
|
873
|
-
async put(
|
|
897
|
+
async put(
|
|
898
|
+
hash: string,
|
|
899
|
+
body: Blob,
|
|
900
|
+
meta: { durationMs: number; replace?: true },
|
|
901
|
+
): Promise<void | false> {
|
|
874
902
|
// Nothing sent, so no upload is counted.
|
|
875
903
|
if (this.disabled || this.writesRefused) return false
|
|
876
|
-
|
|
904
|
+
// `replace`: the copy there did not read back, maybe on another machine
|
|
905
|
+
// of the run (a vx-dte worker), so a HEAD's 200 is the bad copy. Turbo's
|
|
906
|
+
// PUT writes over a held key.
|
|
907
|
+
if (meta.replace !== true && !this.absent.has(hash) && !this.served.has(hash)) {
|
|
877
908
|
// Part of the upload, so under its deadline. Any answer but 200 or 404
|
|
878
909
|
// uploads: a lost store costs more. A server the HEAD cannot reach
|
|
879
910
|
// fails the put, as the PUT would.
|
|
@@ -901,7 +932,7 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
901
932
|
if (res === undefined) return false
|
|
902
933
|
// Turbo's client takes any 2xx (`error_for_status`); a server's 201 or
|
|
903
934
|
// 204 stored the artifact and was counted a failed upload.
|
|
904
|
-
if (!res.ok) throw await refusal(res)
|
|
935
|
+
if (!res.ok) throw await refusal(res, this.hidden)
|
|
905
936
|
this.absent.delete(hash)
|
|
906
937
|
this.served.delete(hash)
|
|
907
938
|
}
|
package/src/turbo-cache/oidc.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
// 2026-10-17); any other server names its token URL.
|
|
6
6
|
|
|
7
7
|
import { deadline } from '../remote-deadline.js'
|
|
8
|
+
import { hideTokens } from '../remote-refusal.js'
|
|
8
9
|
|
|
9
10
|
/** `turboCache({ oidc })`: `true` is Vercel's hosted cache with its defaults. */
|
|
10
11
|
export interface TurboOidcOptions {
|
|
@@ -103,14 +104,15 @@ const where = (url: string) => (URL.canParse(url) ? new URL(url).host : 'the end
|
|
|
103
104
|
* A refusal's own words: RFC 6749's `error` and `error_description`, or
|
|
104
105
|
* GitHub's `message`. Only those fields, which name a fault and never
|
|
105
106
|
* carry a token; any other body is left unread. "answered HTTP 400" alone
|
|
106
|
-
* did not say the audience or the grant was wrong.
|
|
107
|
+
* did not say the audience or the grant was wrong. A server may still
|
|
108
|
+
* quote what it was `sent`: each token is written `***` before the cut.
|
|
107
109
|
*/
|
|
108
|
-
async function refused(res: Response, what: string): Promise<Error> {
|
|
110
|
+
async function refused(res: Response, what: string, sent: readonly string[]): Promise<Error> {
|
|
109
111
|
// quiet: a body that is not JSON says nothing; the status still names the refusal
|
|
110
112
|
const body = (await res.json().catch(() => null)) as Record<string, unknown> | null
|
|
111
113
|
const text = (v: unknown): string | undefined =>
|
|
112
114
|
typeof v === 'string' && v.trim() !== ''
|
|
113
|
-
? v.replace(/\s+/g, ' ').trim().slice(0, 200)
|
|
115
|
+
? hideTokens(v, sent).replace(/\s+/g, ' ').trim().slice(0, 200)
|
|
114
116
|
: undefined
|
|
115
117
|
const said = [text(body?.['error']), text(body?.['error_description']) ?? text(body?.['message'])]
|
|
116
118
|
.filter((s) => s !== undefined)
|
|
@@ -118,8 +120,13 @@ async function refused(res: Response, what: string): Promise<Error> {
|
|
|
118
120
|
return new Error(`${what} answered HTTP ${res.status}${said === '' ? '' : `: ${said}`}`)
|
|
119
121
|
}
|
|
120
122
|
|
|
121
|
-
async function field(
|
|
122
|
-
|
|
123
|
+
async function field(
|
|
124
|
+
res: Response,
|
|
125
|
+
name: string,
|
|
126
|
+
what: string,
|
|
127
|
+
sent: readonly string[],
|
|
128
|
+
): Promise<string> {
|
|
129
|
+
if (!res.ok) throw await refused(res, what, sent)
|
|
123
130
|
const value = ((await res.json().catch(() => null)) as Record<string, unknown> | null)?.[name]
|
|
124
131
|
if (typeof value !== 'string' || value === '') throw new Error(`${what} sent no ${name}`)
|
|
125
132
|
return value
|
|
@@ -148,6 +155,7 @@ export async function exchangeOidc(
|
|
|
148
155
|
),
|
|
149
156
|
'value',
|
|
150
157
|
`GitHub's OIDC endpoint (${where(cfg.idTokenUrl)})`,
|
|
158
|
+
[cfg.idTokenBearer],
|
|
151
159
|
)
|
|
152
160
|
const res = await send(
|
|
153
161
|
fetchImpl,
|
|
@@ -162,7 +170,7 @@ export async function exchangeOidc(
|
|
|
162
170
|
wait,
|
|
163
171
|
)
|
|
164
172
|
const what = `the token exchange (${where(cfg.tokenUrl)})`
|
|
165
|
-
if (!res.ok) throw await refused(res, what)
|
|
173
|
+
if (!res.ok) throw await refused(res, what, [idToken])
|
|
166
174
|
const body = (await res.json().catch(() => null)) as Record<string, unknown> | null
|
|
167
175
|
const token = body?.['access_token']
|
|
168
176
|
if (typeof token !== 'string' || token === '') throw new Error(`${what} sent no access_token`)
|
|
@@ -192,6 +200,6 @@ export async function revokeOidc(
|
|
|
192
200
|
}),
|
|
193
201
|
wait,
|
|
194
202
|
)
|
|
195
|
-
if (!res.ok) throw await refused(res, `revoking at ${where(cfg.revokeUrl)}
|
|
203
|
+
if (!res.ok) throw await refused(res, `revoking at ${where(cfg.revokeUrl)}`, [token])
|
|
196
204
|
await res.body?.cancel()
|
|
197
205
|
}
|
package/src/workspace-notes.ts
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
// What the written configs leave to .vx/workspace.ts, said once by both
|
|
2
2
|
// migrators: the adoption plugin `vx init` declared still reads the old
|
|
3
3
|
// tool's file every run, and the old tool keyed each package on its own
|
|
4
|
-
// lockfile entries where core keys
|
|
5
|
-
//
|
|
6
|
-
//
|
|
4
|
+
// lockfile entries where core keys no lockfile (#3562): without a
|
|
5
|
+
// lockfile plugin a dependency bump re-runs nothing, and a cached task
|
|
6
|
+
// replays output built on the old versions.
|
|
7
7
|
|
|
8
8
|
import { existsSync } from 'node:fs'
|
|
9
9
|
import path from 'node:path'
|
|
10
|
+
import type { PackageManager } from './adopt.js'
|
|
10
11
|
|
|
11
12
|
/** Each lockfile and the `@vzn/vx-lockfile` plugin that claims it. */
|
|
12
13
|
const LOCKFILES: ReadonlyArray<readonly [file: string, plugin: string]> = [
|
|
@@ -18,9 +19,18 @@ const LOCKFILES: ReadonlyArray<readonly [file: string, plugin: string]> = [
|
|
|
18
19
|
['yarn.lock', 'yarn'],
|
|
19
20
|
]
|
|
20
21
|
|
|
21
|
-
/**
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
/**
|
|
23
|
+
* The lockfile at `root` and the `@vzn/vx-lockfile` factory that claims it.
|
|
24
|
+
* With none there, `installer`'s: the manager this run installs with
|
|
25
|
+
* writes it, and the lock that follows needs its plugin for `^` edges.
|
|
26
|
+
*/
|
|
27
|
+
export function lockfilePlugin(
|
|
28
|
+
root: string,
|
|
29
|
+
installer?: PackageManager,
|
|
30
|
+
): { file: string; factory: string } | undefined {
|
|
31
|
+
const lock =
|
|
32
|
+
LOCKFILES.find(([file]) => existsSync(path.join(root, file))) ??
|
|
33
|
+
LOCKFILES.find(([, factory]) => factory === installer)
|
|
24
34
|
return lock === undefined ? undefined : { file: lock[0], factory: lock[1] }
|
|
25
35
|
}
|
|
26
36
|
|
|
@@ -58,8 +68,9 @@ export async function adoptedToolNotes(root: string, tool: AdoptedTool): Promise
|
|
|
58
68
|
const lock = lockfilePlugin(root)
|
|
59
69
|
if (lock !== undefined && !new RegExp(`\\b${lock.factory}\\s*\\(`).test(declared))
|
|
60
70
|
notes.push(
|
|
61
|
-
`${tool.keys(lock.file)}; vx keys
|
|
62
|
-
|
|
71
|
+
`${tool.keys(lock.file)}; vx keys no lockfile by itself, so a dependency bump re-runs ` +
|
|
72
|
+
'nothing and a cached task replays output built on the old versions. ' +
|
|
73
|
+
`Declare ${lock.factory}() from @vzn/vx-lockfile ` +
|
|
63
74
|
"in .vx/workspace.ts to key each task on its package's dependency closure",
|
|
64
75
|
)
|
|
65
76
|
return notes
|
package/src/workspace-plugins.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
import { existsSync } from 'node:fs'
|
|
8
8
|
import path from 'node:path'
|
|
9
|
+
import type { PackageManager } from './adopt.js'
|
|
9
10
|
import { REMOTE_CACHE, remoteCacheSignal } from './remote-cache.js'
|
|
10
11
|
import { lockfilePlugin } from './workspace-notes.js'
|
|
11
12
|
|
|
@@ -16,12 +17,19 @@ export interface WorkspacePlugin {
|
|
|
16
17
|
readonly factory: string
|
|
17
18
|
}
|
|
18
19
|
|
|
19
|
-
/**
|
|
20
|
-
|
|
20
|
+
/**
|
|
21
|
+
* `runner`: the Turbo or Nx repo's own remote cache, where the repo shows
|
|
22
|
+
* one. `installer`: the manager this run installs with (`lockfilePlugin`).
|
|
23
|
+
*/
|
|
24
|
+
export function workspacePlugins(
|
|
25
|
+
root: string,
|
|
26
|
+
runner?: 'turbo' | 'nx',
|
|
27
|
+
installer?: PackageManager,
|
|
28
|
+
): WorkspacePlugin[] {
|
|
21
29
|
const out: WorkspacePlugin[] = []
|
|
22
30
|
if (runner !== undefined && remoteCacheSignal(root, runner) !== undefined)
|
|
23
31
|
out.push({ pkg: '@vzn/vx-migrate', factory: REMOTE_CACHE[runner] })
|
|
24
|
-
const lock = lockfilePlugin(root)
|
|
32
|
+
const lock = lockfilePlugin(root, installer)
|
|
25
33
|
if (lock !== undefined) out.push({ pkg: '@vzn/vx-lockfile', factory: lock.factory })
|
|
26
34
|
out.push({ pkg: '@vzn/vx-history', factory: 'vxHistory' })
|
|
27
35
|
if (existsSync(path.join(root, '.github', 'workflows')))
|