@vzn/vx-cache 0.0.0 → 0.0.648
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +122 -0
- package/index.ts +4 -0
- package/package.json +44 -1
- package/src/index.ts +938 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 VZN
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# @vzn/vx-cache
|
|
2
|
+
|
|
3
|
+
Website: <https://vznjs.github.io/vx/> · source: <https://github.com/vznjs/vx>
|
|
4
|
+
|
|
5
|
+
`vxCache()`: a remote cache for [`@vzn/vx`](https://vznjs.github.io/vx/)
|
|
6
|
+
runs, on a server you host that speaks the vx-cache protocol, such as
|
|
7
|
+
[`@vzn/vx-cache-server`](https://github.com/vznjs/vx/tree/main/packages/vx-cache-server).
|
|
8
|
+
The local cache stays first; the server is asked on a local miss, and
|
|
9
|
+
each miss's artifact is uploaded to it.
|
|
10
|
+
|
|
11
|
+
```sh
|
|
12
|
+
bun add -d @vzn/vx-cache
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
// .vx/workspace.ts
|
|
17
|
+
import { vxCache } from '@vzn/vx-cache'
|
|
18
|
+
|
|
19
|
+
export default {
|
|
20
|
+
plugins: [
|
|
21
|
+
vxCache({
|
|
22
|
+
url: 'https://cache.example.com',
|
|
23
|
+
issuer: 'https://token.actions.githubusercontent.com',
|
|
24
|
+
repository: 'acme/app',
|
|
25
|
+
}),
|
|
26
|
+
],
|
|
27
|
+
}
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
`vxCache({ url?, issuer, repository, timeoutMs?, uploadTimeoutMs?, headers? })`:
|
|
31
|
+
|
|
32
|
+
- `url` is the server's base URL (`https://cache.example.com`, without `/v1`), or `VX_CACHE_URL`.
|
|
33
|
+
It must be `https://`: every request carries the CI token, so `http://`
|
|
34
|
+
is taken only to this machine (`127.0.0.1`, `::1`, `localhost`). The
|
|
35
|
+
option is checked when the config loads, `VX_CACHE_URL` when a run
|
|
36
|
+
opens the cache.
|
|
37
|
+
- `issuer` (the CI issuer's URL, as its tokens name it) and `repository`
|
|
38
|
+
(`org/name` there) are required: nothing guesses them. Every request
|
|
39
|
+
names both by their hashes (`vx-issuer`, `vx-repository`), and the
|
|
40
|
+
server refuses a token of another repository.
|
|
41
|
+
- `timeoutMs` (default 30 000) bounds each read and query, and
|
|
42
|
+
`uploadTimeoutMs` (default 60 000) each upload. A download is bounded
|
|
43
|
+
the same way until its answer, then must bring at least 1 MiB a second
|
|
44
|
+
over each `timeoutMs`; an upload must send at least 1 MiB a second over
|
|
45
|
+
each `uploadTimeoutMs`, then be answered within one. A big artifact
|
|
46
|
+
still moving is not cut off, and one trickling or stalled is: it fails
|
|
47
|
+
the transfer, said as any cache error. Each is ms above 0 and up to 2³¹−1 (the
|
|
48
|
+
longest a timer holds), or `vxCache()` refuses it.
|
|
49
|
+
- `headers` are sent on every request to the server, for a proxy in
|
|
50
|
+
front of it, such as a Cloudflare Access service token:
|
|
51
|
+
`headers: { 'CF-Access-Client-Id': process.env.CF_ACCESS_CLIENT_ID,
|
|
52
|
+
'CF-Access-Client-Secret': process.env.CF_ACCESS_CLIENT_SECRET }`. An
|
|
53
|
+
unset value is left out; `authorization`, `content-type`, `content-length`
|
|
54
|
+
and the `vx-*` names stay the request's own (setting one is refused). A
|
|
55
|
+
redirect is not followed (it would carry them on): it fails the
|
|
56
|
+
request, naming where it pointed. A DTE
|
|
57
|
+
worker sends them too, so give its job the same secrets.
|
|
58
|
+
|
|
59
|
+
## Tokens
|
|
60
|
+
|
|
61
|
+
A CI job's requests carry its own OIDC token, and the server's `ALLOW`
|
|
62
|
+
decides whether the job gets in; it writes its own branch.
|
|
63
|
+
|
|
64
|
+
- On GitHub Actions the job's token is asked for the server's origin,
|
|
65
|
+
and asked again before it ends; after a 5xx, a 429 or no answer
|
|
66
|
+
from GitHub it is asked again, three tries in all, before the remote
|
|
67
|
+
turns off. The job needs
|
|
68
|
+
`permissions: id-token: write`, which sets
|
|
69
|
+
`ACTIONS_ID_TOKEN_REQUEST_URL` and `ACTIONS_ID_TOKEN_REQUEST_TOKEN`;
|
|
70
|
+
without it (`GITHUB_ACTIONS` is set but no token comes), the plugin
|
|
71
|
+
warns and the job reads only.
|
|
72
|
+
- On GitLab, name an `id_tokens` entry `VX_CACHE_TOKEN` with the
|
|
73
|
+
server's origin as `aud`; without it (`GITLAB_CI` is set but no
|
|
74
|
+
token comes), the plugin warns and the job reads only.
|
|
75
|
+
|
|
76
|
+
With no token (off CI) a run reads only and uploads nothing, from a
|
|
77
|
+
server whose `READ_ACCESS` is `public`; any other server refuses the
|
|
78
|
+
first read, said once, and the run stays local.
|
|
79
|
+
|
|
80
|
+
A run that hands tasks to other jobs lends them its token (core's
|
|
81
|
+
`lend()`), and they read and write with it: a job started from another
|
|
82
|
+
workflow would carry a token that writes nowhere the run reads. A job
|
|
83
|
+
uses a lent token over its own and never lends one on, nor any once
|
|
84
|
+
its remote is off for the run.
|
|
85
|
+
|
|
86
|
+
A refused token is said once and turns the remote off for the run; a
|
|
87
|
+
refused write (a job that reads only) turns off writes alone. Any other
|
|
88
|
+
refusal is a warning naming the server's own words, and the task reads
|
|
89
|
+
as a miss.
|
|
90
|
+
|
|
91
|
+
Every read names scopes, each a ref's hash (no ref goes on the wire
|
|
92
|
+
plain). A check names the run's refs in order (its branch, then the
|
|
93
|
+
branch it merges into, then the default branch; on CI `github()` or
|
|
94
|
+
`gitlab()` from `@vzn/vx-ci` names them, so declare it too: without it
|
|
95
|
+
a run reads git's branch and the default branch only), and the first ref
|
|
96
|
+
holding a hash answers it; its download names that ref. A run that reads
|
|
97
|
+
no ref says so and stays local. The run asks the server's limits once
|
|
98
|
+
(`/v1/capabilities`) and checks in chunks of its `maxCheckHashes`
|
|
99
|
+
lookups, all at once; a run reading more refs than one check takes
|
|
100
|
+
(Workers' default is 6) spreads each hash's scopes over several. It
|
|
101
|
+
keeps to the limits each `/v1/artifacts` answer names, a refusal's too,
|
|
102
|
+
so a server restarted with lower ones refuses one check, not the run's. Each
|
|
103
|
+
upload is one artifact by `PUT`, up to four at once: its bytes reach the
|
|
104
|
+
bucket untouched by the server's code (a batch, split by that code, ran
|
|
105
|
+
out of Workers' CPU). While the server names no limits (said once),
|
|
106
|
+
checks go one hash at a time: no limit is guessed. An artifact over the server's upload limit is not sent: it
|
|
107
|
+
is said once. An answer is read no further than its use: 64 KiB for the
|
|
108
|
+
capabilities or a token, 512 bytes a hash for a check, a refusal's first
|
|
109
|
+
2 KiB; a longer one is refused, so a server's endless answer cannot fill
|
|
110
|
+
memory.
|
|
111
|
+
|
|
112
|
+
Task timings are not this plugin's: `@vzn/vx-history` keeps them.
|
|
113
|
+
|
|
114
|
+
## API
|
|
115
|
+
|
|
116
|
+
- `vxCache(options: VxCacheOptions)` is the plugin; `VxCacheOptions` is the options above.
|
|
117
|
+
- `VxCacheLayer` is its remote cache layer (core's `RemoteCacheLayer`),
|
|
118
|
+
reading the refs in its `refs`.
|
|
119
|
+
- `resolveVxCacheConfig(options, env)` is the settings it runs with
|
|
120
|
+
(`ResolvedVxCacheConfig`: every option set, `headers` checked), or
|
|
121
|
+
`undefined` with no URL, and `hasCiToken(env, lent)` whether a job has
|
|
122
|
+
a token to send.
|
package/index.ts
ADDED
package/package.json
CHANGED
|
@@ -1 +1,44 @@
|
|
|
1
|
-
{
|
|
1
|
+
{
|
|
2
|
+
"name": "@vzn/vx-cache",
|
|
3
|
+
"version": "0.0.648",
|
|
4
|
+
"license": "MIT",
|
|
5
|
+
"description": "vxCache(): a vx cache plugin for a server that speaks the vx-cache protocol, such as @vzn/vx-cache-server.",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"vx",
|
|
8
|
+
"monorepo",
|
|
9
|
+
"vx-plugin",
|
|
10
|
+
"remote-cache"
|
|
11
|
+
],
|
|
12
|
+
"type": "module",
|
|
13
|
+
"main": "./src/index.ts",
|
|
14
|
+
"types": "./src/index.ts",
|
|
15
|
+
"exports": {
|
|
16
|
+
".": {
|
|
17
|
+
"types": "./src/index.ts",
|
|
18
|
+
"import": "./src/index.ts"
|
|
19
|
+
}
|
|
20
|
+
},
|
|
21
|
+
"files": [
|
|
22
|
+
"index.ts",
|
|
23
|
+
"src",
|
|
24
|
+
"README.md",
|
|
25
|
+
"LICENSE"
|
|
26
|
+
],
|
|
27
|
+
"engines": {
|
|
28
|
+
"bun": ">=1.4.4"
|
|
29
|
+
},
|
|
30
|
+
"//": [
|
|
31
|
+
"@vzn/vx-cache-server, @vzn/vx-devtools: the tests run the plugin against the real server, in process.",
|
|
32
|
+
"jose: the tests sign the CI tokens the server checks."
|
|
33
|
+
],
|
|
34
|
+
"peerDependencies": {
|
|
35
|
+
"@vzn/vx": "^0.0.648"
|
|
36
|
+
},
|
|
37
|
+
"repository": {
|
|
38
|
+
"type": "git",
|
|
39
|
+
"url": "git+https://github.com/vznjs/vx.git",
|
|
40
|
+
"directory": "packages/vx-cache"
|
|
41
|
+
},
|
|
42
|
+
"homepage": "https://github.com/vznjs/vx/tree/main/packages/vx-cache#readme",
|
|
43
|
+
"bugs": "https://github.com/vznjs/vx/issues"
|
|
44
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,938 @@
|
|
|
1
|
+
// @vzn/vx-cache — `vxCache()`, a vx `cache` plugin for a server that speaks
|
|
2
|
+
// the vx-cache protocol (`@vzn/vx-cache-server`, its PROTOCOL.md, /v1).
|
|
3
|
+
// Nothing is on by default: declare it in `.vx/workspace.ts` with the
|
|
4
|
+
// server's URL, the CI issuer and the repository, which every request
|
|
5
|
+
// names by their hashes (2026-10-25: the user names them, nothing
|
|
6
|
+
// guesses). A CI job's requests carry its OIDC token and write; with no
|
|
7
|
+
// token it reads only, from a server with `READ_ACCESS=public`. Every read
|
|
8
|
+
// names exact scopes: the run's branch, then the branches it reads after it.
|
|
9
|
+
// Imports core only through `@vzn/vx`.
|
|
10
|
+
|
|
11
|
+
import {
|
|
12
|
+
type ArtifactInfo,
|
|
13
|
+
definePlugin,
|
|
14
|
+
type LentAccess,
|
|
15
|
+
type PluginOptionKinds,
|
|
16
|
+
refuseUnknownOptions,
|
|
17
|
+
type RemoteCacheLayer,
|
|
18
|
+
UserError,
|
|
19
|
+
type VxPlugin,
|
|
20
|
+
} from '@vzn/vx'
|
|
21
|
+
|
|
22
|
+
export interface VxCacheOptions {
|
|
23
|
+
/** The server's base URL (`https://cache.example.com`; http only to this machine), or `VX_CACHE_URL`. */
|
|
24
|
+
url?: string
|
|
25
|
+
/** The CI issuer whose tokens the server takes, as they name it (`https://token.actions.githubusercontent.com`). */
|
|
26
|
+
issuer: string
|
|
27
|
+
/** The repository at that issuer, `org/name`. */
|
|
28
|
+
repository: string
|
|
29
|
+
/** Per-request deadline for reads and queries (default 30 s); a download's bounds its answer, then each gap between its bytes. */
|
|
30
|
+
timeoutMs?: number
|
|
31
|
+
/** … and for an upload (default 60 s): its bytes must leave at 1 MiB a second over each, then its answer come within one. */
|
|
32
|
+
uploadTimeoutMs?: number
|
|
33
|
+
/**
|
|
34
|
+
* Headers every request to the server carries, for a proxy in front of
|
|
35
|
+
* it (`{ 'CF-Access-Client-Id': process.env.CF_ACCESS_CLIENT_ID }`, an
|
|
36
|
+
* Access service token, 2026-10-25). An unset value is left out;
|
|
37
|
+
* `authorization`, `content-type`, `content-length` and the `vx-*` names
|
|
38
|
+
* the request sets are refused.
|
|
39
|
+
*/
|
|
40
|
+
headers?: Record<string, string | undefined>
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const KEYS: PluginOptionKinds<VxCacheOptions> = {
|
|
44
|
+
url: 'string',
|
|
45
|
+
issuer: 'string',
|
|
46
|
+
repository: 'string',
|
|
47
|
+
timeoutMs: 'number',
|
|
48
|
+
uploadTimeoutMs: 'number',
|
|
49
|
+
headers: 'object',
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** A JWT's `exp`, in ms, or 0 when it has none. */
|
|
53
|
+
function expiryOf(jwt: string): number {
|
|
54
|
+
try {
|
|
55
|
+
const exp = (
|
|
56
|
+
JSON.parse(atob(jwt.split('.')[1]!.replace(/-/g, '+').replace(/_/g, '/'))) as {
|
|
57
|
+
exp?: unknown
|
|
58
|
+
}
|
|
59
|
+
).exp
|
|
60
|
+
return typeof exp === 'number' ? exp * 1000 : 0
|
|
61
|
+
} catch {
|
|
62
|
+
// quiet: an unreadable exp is no known expiry; the server refuses the token
|
|
63
|
+
return 0
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** A lent token ending sooner than this is asked again first, so a request never carries a spent one. */
|
|
68
|
+
const LEND_MARGIN_MS = 5 * 60_000
|
|
69
|
+
/** Tries of GitHub's OIDC endpoint before the remote turns off (`askOnce`). */
|
|
70
|
+
const ASK_ATTEMPTS = 3
|
|
71
|
+
/** Uploads open at once; more wait. */
|
|
72
|
+
const IN_FLIGHT = 4
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* The names a request sets itself: one of them in `headers`, in another
|
|
76
|
+
* case, was joined to the request's own value, and the server refused it.
|
|
77
|
+
*/
|
|
78
|
+
const WIRE_HEADERS = new Set([
|
|
79
|
+
'authorization',
|
|
80
|
+
'content-type',
|
|
81
|
+
'content-length',
|
|
82
|
+
'vx-issuer',
|
|
83
|
+
'vx-repository',
|
|
84
|
+
'vx-scope',
|
|
85
|
+
])
|
|
86
|
+
|
|
87
|
+
/** `headers` with unset values left out; a name or value no header can carry, or one the wire sets, refused. */
|
|
88
|
+
function proxyHeaders(
|
|
89
|
+
factory: string,
|
|
90
|
+
headers: Record<string, string | undefined> | undefined,
|
|
91
|
+
): Record<string, string> {
|
|
92
|
+
const out: Record<string, string> = {}
|
|
93
|
+
for (const [name, value] of Object.entries(headers ?? {})) {
|
|
94
|
+
if (value === undefined) continue
|
|
95
|
+
if (!/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(name) || WIRE_HEADERS.has(name.toLowerCase()))
|
|
96
|
+
throw new UserError(`${factory} option "headers" cannot set "${name}"`)
|
|
97
|
+
if (typeof value !== 'string' || /[\r\n\0]/.test(value))
|
|
98
|
+
throw new UserError(
|
|
99
|
+
`${factory} option "headers.${name}" must be one line of text (its value is not printed)`,
|
|
100
|
+
)
|
|
101
|
+
out[name] = value
|
|
102
|
+
}
|
|
103
|
+
return out
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** A ref's scope, an issuer or a repository on the wire: its SHA-256's first 128 bits in hex, never the value (2026-10-25). */
|
|
107
|
+
const scopeOf = (ref: string): string =>
|
|
108
|
+
new Bun.CryptoHasher('sha256').update(ref).digest('hex').slice(0, 32)
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* A body's first `max` bytes as text, the rest cancelled unread, and
|
|
112
|
+
* whether there was more: an answer is read no further than vx needs, so
|
|
113
|
+
* a server's endless one cannot fill this machine's memory.
|
|
114
|
+
*/
|
|
115
|
+
async function upTo(res: Response, max: number): Promise<{ text: string; over: boolean }> {
|
|
116
|
+
const reader = res.body?.getReader()
|
|
117
|
+
if (reader === undefined) return { text: '', over: false }
|
|
118
|
+
const parts: Uint8Array[] = []
|
|
119
|
+
let n = 0
|
|
120
|
+
for (;;) {
|
|
121
|
+
const { done, value } = await reader.read()
|
|
122
|
+
if (done) break
|
|
123
|
+
parts.push(value)
|
|
124
|
+
n += value.byteLength
|
|
125
|
+
if (n > max) {
|
|
126
|
+
// quiet: the answer is refused for its size; its cancel adds nothing
|
|
127
|
+
await reader.cancel().catch(() => {})
|
|
128
|
+
return { text: Buffer.concat(parts).subarray(0, max).toString('utf8'), over: true }
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return { text: Buffer.concat(parts).toString('utf8'), over: false }
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** A JSON answer of at most `max` bytes; past it, a refusal that says so. */
|
|
135
|
+
async function jsonUpTo(res: Response, max: number, what: string): Promise<unknown> {
|
|
136
|
+
const { text, over } = await upTo(res, max)
|
|
137
|
+
if (over) throw new Error(`${what} answered over ${max} bytes`)
|
|
138
|
+
return JSON.parse(text)
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** Capabilities and a CI token are a few hundred bytes: room to spare. */
|
|
142
|
+
const SMALL_ANSWER_BYTES = 64 * 1024
|
|
143
|
+
/** A check's answer per hash asked: a hash, a size and a scope in JSON are under 150 bytes. */
|
|
144
|
+
const CHECK_BYTES_PER_HASH = 512
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* `s` with each C0/C1 control and DEL written as `\xNN`: a server's words
|
|
148
|
+
* reach the terminal, and an `ESC ]` or `\r` in them retitled it or
|
|
149
|
+
* overwrote the line before (core's `printable`, L-31).
|
|
150
|
+
*/
|
|
151
|
+
const printable = (s: string): string =>
|
|
152
|
+
// oxlint-disable-next-line no-control-regex
|
|
153
|
+
s.replace(/[\x00-\x1f\x7f-\x9f]/g, (c) => `\\x${c.charCodeAt(0).toString(16).padStart(2, '0')}`)
|
|
154
|
+
|
|
155
|
+
/** A refusal, named by its status and the server's own `message`. */
|
|
156
|
+
async function refusal(res: Response): Promise<Error> {
|
|
157
|
+
let said = ''
|
|
158
|
+
try {
|
|
159
|
+
const { text } = await upTo(res, 2048)
|
|
160
|
+
try {
|
|
161
|
+
const message = (JSON.parse(text) as { message?: unknown }).message
|
|
162
|
+
said = typeof message === 'string' ? message : text
|
|
163
|
+
} catch {
|
|
164
|
+
// quiet: not JSON, so the text is what it said
|
|
165
|
+
said = text
|
|
166
|
+
}
|
|
167
|
+
} catch {
|
|
168
|
+
// quiet: the status alone still names the refusal
|
|
169
|
+
}
|
|
170
|
+
said = printable(said.replace(/\s+/g, ' ').trim().slice(0, 200))
|
|
171
|
+
return new Error(`HTTP ${res.status}${said === '' ? '' : `: ${said}`}`)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** A 200's JSON; one that is none (a website's page) names the likely fault, the url. */
|
|
175
|
+
async function jsonOf(res: Response, path: string, max: number): Promise<unknown> {
|
|
176
|
+
const { text, over } = await upTo(res, max)
|
|
177
|
+
if (over) throw new Error(`${path} answered over ${max} bytes`)
|
|
178
|
+
try {
|
|
179
|
+
return JSON.parse(text)
|
|
180
|
+
} catch {
|
|
181
|
+
const type = res.headers.get('content-type') ?? 'no content type'
|
|
182
|
+
throw new Error(
|
|
183
|
+
`${path} answered ${res.status} with ${type}, not JSON: is the url a vx-cache server?`,
|
|
184
|
+
)
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* A redirect, never followed: fetch drops only `Authorization` on one to
|
|
190
|
+
* another origin, so `headers` (a proxy's secret) went where a 307 named.
|
|
191
|
+
*/
|
|
192
|
+
async function redirected(res: Response, from: string): Promise<Error> {
|
|
193
|
+
await res.body?.cancel()
|
|
194
|
+
const to = res.headers.get('location')
|
|
195
|
+
return new Error(
|
|
196
|
+
`HTTP ${res.status}${to === null ? '' : ` to ${new URL(to, from).href}`}: not followed, as it would carry the server's headers; name the server's own URL`,
|
|
197
|
+
)
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** A deadline that names itself: the runtime's own abort says only "The operation timed out." */
|
|
201
|
+
function deadline(ms: number): AbortSignal {
|
|
202
|
+
const controller = new AbortController()
|
|
203
|
+
setTimeout(() => controller.abort(new Error(`no answer within ${ms} ms`)), ms).unref()
|
|
204
|
+
return controller.signal
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** A transfer's least pace: at 1 MiB a second, a 60 MB artifact is in within a minute. */
|
|
208
|
+
const FLOOR_BYTES_PER_S = 1024 * 1024
|
|
209
|
+
|
|
210
|
+
/** An upload's body is handed to the fetch in slices this long, so its pace can be read. */
|
|
211
|
+
const SLICE_BYTES = 64 * 1024
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* At least `FLOOR_BYTES_PER_S` over each `ms`, or `controller` aborts
|
|
215
|
+
* naming the pace. A whole-body deadline failed a big artifact still
|
|
216
|
+
* arriving at a steady pace (a 30 s cut, CI 2026-10-25); a gap-only one
|
|
217
|
+
* let a trickle hold a run 19 minutes (a byte now and then reset it, CI
|
|
218
|
+
* 2026-10-28).
|
|
219
|
+
*/
|
|
220
|
+
function pace(
|
|
221
|
+
controller: AbortController,
|
|
222
|
+
ms: number,
|
|
223
|
+
): { moved(bytes: number): void; stop(): void } {
|
|
224
|
+
const floor = (FLOOR_BYTES_PER_S * ms) / 1000
|
|
225
|
+
let got = 0
|
|
226
|
+
const window = setInterval(() => {
|
|
227
|
+
if (got < floor)
|
|
228
|
+
controller.abort(
|
|
229
|
+
new Error(
|
|
230
|
+
got === 0 ? `no bytes in ${ms} ms` : `${got} bytes in ${ms} ms, under 1 MiB a second`,
|
|
231
|
+
),
|
|
232
|
+
)
|
|
233
|
+
got = 0
|
|
234
|
+
}, ms)
|
|
235
|
+
window.unref()
|
|
236
|
+
const stop = () => clearInterval(window)
|
|
237
|
+
controller.signal.addEventListener('abort', stop, { once: true })
|
|
238
|
+
return {
|
|
239
|
+
moved: (bytes) => {
|
|
240
|
+
got += bytes
|
|
241
|
+
},
|
|
242
|
+
stop,
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* A download's deadline: `ms` for the answer, then `pace` over its body.
|
|
248
|
+
* Core judges whether running the task beats the rest of the wait.
|
|
249
|
+
*/
|
|
250
|
+
function streamed(res: Response, controller: AbortController, ms: number): Response {
|
|
251
|
+
if (res.body === null) return res
|
|
252
|
+
const window = pace(controller, ms)
|
|
253
|
+
const body = res.body.pipeThrough(
|
|
254
|
+
new TransformStream<Uint8Array, Uint8Array>({
|
|
255
|
+
transform(chunk, out) {
|
|
256
|
+
window.moved(chunk.byteLength)
|
|
257
|
+
out.enqueue(chunk)
|
|
258
|
+
},
|
|
259
|
+
flush: window.stop,
|
|
260
|
+
}),
|
|
261
|
+
)
|
|
262
|
+
return new Response(body, {
|
|
263
|
+
status: res.status,
|
|
264
|
+
statusText: res.statusText,
|
|
265
|
+
headers: res.headers,
|
|
266
|
+
})
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* An upload's body, held to `pace` while the fetch takes it, then `ms`
|
|
271
|
+
* for the answer (`sent`). A deadline over the whole upload failed every
|
|
272
|
+
* artifact slower than its size over it: 100 MB under 1.7 MB a second at
|
|
273
|
+
* the default 60 s. The pace is what the fetch takes, so the bytes the
|
|
274
|
+
* socket buffers ahead of the wire (MBs) count early once; a stalled
|
|
275
|
+
* link still ends at the first window they do not fill.
|
|
276
|
+
*/
|
|
277
|
+
function paced(
|
|
278
|
+
blob: Blob,
|
|
279
|
+
controller: AbortController,
|
|
280
|
+
ms: number,
|
|
281
|
+
sent: () => void,
|
|
282
|
+
): { body: ReadableStream<Uint8Array>; stop(): void } {
|
|
283
|
+
const window = pace(controller, ms)
|
|
284
|
+
let at = 0
|
|
285
|
+
const body = new ReadableStream<Uint8Array>(
|
|
286
|
+
{
|
|
287
|
+
async pull(out) {
|
|
288
|
+
if (at >= blob.size) {
|
|
289
|
+
window.stop()
|
|
290
|
+
out.close()
|
|
291
|
+
sent()
|
|
292
|
+
return
|
|
293
|
+
}
|
|
294
|
+
const slice = await blob.slice(at, at + SLICE_BYTES).bytes()
|
|
295
|
+
at += slice.byteLength
|
|
296
|
+
window.moved(slice.byteLength)
|
|
297
|
+
out.enqueue(slice)
|
|
298
|
+
},
|
|
299
|
+
cancel: window.stop,
|
|
300
|
+
},
|
|
301
|
+
{ highWaterMark: 0 },
|
|
302
|
+
)
|
|
303
|
+
return { body, stop: window.stop }
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Uploads: up to `IN_FLIGHT` at once, each one artifact by `PUT`, whose
|
|
308
|
+
* bytes reach the bucket untouched (2026-10-25: no batch upload; a
|
|
309
|
+
* batch passes through the server's code, which on Workers costs CPU by the
|
|
310
|
+
* byte, and a 40 MB one ran out of it).
|
|
311
|
+
*/
|
|
312
|
+
class Lane<T, R> {
|
|
313
|
+
private readonly waiting: { item: T; resolve(r: R): void; reject(err: unknown): void }[] = []
|
|
314
|
+
private open = 0
|
|
315
|
+
|
|
316
|
+
constructor(private readonly send: (item: T) => Promise<R>) {}
|
|
317
|
+
|
|
318
|
+
push(item: T): Promise<R> {
|
|
319
|
+
const { promise, resolve, reject } = Promise.withResolvers<R>()
|
|
320
|
+
this.waiting.push({ item, resolve, reject })
|
|
321
|
+
this.pump()
|
|
322
|
+
return promise
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
private pump(): void {
|
|
326
|
+
while (this.open < IN_FLIGHT && this.waiting.length > 0) {
|
|
327
|
+
const w = this.waiting.shift()!
|
|
328
|
+
this.open++
|
|
329
|
+
void this.send(w.item)
|
|
330
|
+
.then(w.resolve, w.reject)
|
|
331
|
+
.finally(() => {
|
|
332
|
+
this.open--
|
|
333
|
+
this.pump()
|
|
334
|
+
})
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/** The limits a server names at `/v1/capabilities` and on each check (`vx-max-*`). */
|
|
340
|
+
interface Limits {
|
|
341
|
+
readonly maxCheckHashes: number
|
|
342
|
+
readonly maxUploadBytes: number
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
const LIMIT_HEADERS: Record<keyof Limits, string> = {
|
|
346
|
+
maxCheckHashes: 'vx-max-check-hashes',
|
|
347
|
+
maxUploadBytes: 'vx-max-upload-bytes',
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
const count = (n: unknown): number | undefined =>
|
|
351
|
+
typeof n === 'number' && Number.isSafeInteger(n) && n > 0 ? n : undefined
|
|
352
|
+
|
|
353
|
+
const chunks = <T>(all: readonly T[], size: number): T[][] =>
|
|
354
|
+
Array.from({ length: Math.ceil(all.length / size) }, (_, i) =>
|
|
355
|
+
all.slice(i * size, (i + 1) * size),
|
|
356
|
+
)
|
|
357
|
+
|
|
358
|
+
/** The limits an answer's `vx-max-*` headers name. */
|
|
359
|
+
const limitsNamed = (res: Response): Limits | undefined =>
|
|
360
|
+
limitsOf((k) => Number(res.headers.get(LIMIT_HEADERS[k]) ?? NaN))
|
|
361
|
+
|
|
362
|
+
/** Each limit, or undefined when one is missing or bad. */
|
|
363
|
+
function limitsOf(read: (k: keyof Limits) => unknown): Limits | undefined {
|
|
364
|
+
const maxCheckHashes = count(read('maxCheckHashes'))
|
|
365
|
+
const maxUploadBytes = count(read('maxUploadBytes'))
|
|
366
|
+
return maxCheckHashes === undefined || maxUploadBytes === undefined
|
|
367
|
+
? undefined
|
|
368
|
+
: { maxCheckHashes, maxUploadBytes }
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* The seam over vx-cache v1. `has` and `hasMany` are `check`, `get` a
|
|
373
|
+
* download and `put` an upload by `PUT` (`Lane`). A check names the
|
|
374
|
+
* scope of each ref the run reads, in order, and the first holding a hash
|
|
375
|
+
* answers it; a download names the scope its check found. A refused token (401/403) throws
|
|
376
|
+
* once, for core to report, and turns the layer off for the run; an
|
|
377
|
+
* upload's 403 turns off writes alone (a job that reads only).
|
|
378
|
+
*/
|
|
379
|
+
export class VxCacheLayer implements RemoteCacheLayer {
|
|
380
|
+
readonly endpoint: string
|
|
381
|
+
private disabled = false
|
|
382
|
+
private writesRefused = false
|
|
383
|
+
/** The refs a read names, in order: the first holding a hash answers it. */
|
|
384
|
+
refs: readonly string[] = []
|
|
385
|
+
/** The ref a check found each hash under, which its download names. */
|
|
386
|
+
private readonly found = new Map<string, string>()
|
|
387
|
+
/** The limits the server named, at `/v1/capabilities` and on its last check; nothing is guessed. */
|
|
388
|
+
private limits: Limits | undefined
|
|
389
|
+
private asked: Promise<Limits | undefined> | undefined
|
|
390
|
+
/** Where the layer says what is no failure of a request: core's `ctx.warn`. */
|
|
391
|
+
warn: (message: string) => void = () => {}
|
|
392
|
+
private limitsMissing = false
|
|
393
|
+
|
|
394
|
+
private readonly uploads = new Lane<{ hash: string; body: Blob }, void | false>((item) =>
|
|
395
|
+
this.upload(item),
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
/** The issuer and repository, named on every request by their hashes. */
|
|
399
|
+
private readonly named: Readonly<Record<string, string>>
|
|
400
|
+
|
|
401
|
+
constructor(
|
|
402
|
+
private readonly config: ResolvedVxCacheConfig,
|
|
403
|
+
private readonly fetchImpl: typeof fetch = fetch,
|
|
404
|
+
private readonly env: Record<string, string | undefined> = Bun.env,
|
|
405
|
+
private readonly wait: (ms: number) => Promise<void> = Bun.sleep,
|
|
406
|
+
) {
|
|
407
|
+
this.endpoint = config.url
|
|
408
|
+
this.named = {
|
|
409
|
+
'vx-issuer': scopeOf(config.issuer),
|
|
410
|
+
'vx-repository': scopeOf(config.repository),
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
private idToken: { jwt: string; exp: number } | undefined
|
|
415
|
+
/** The token a run running its tasks here lent (`CacheContext.lent`). */
|
|
416
|
+
lent: (() => string | undefined) | undefined
|
|
417
|
+
|
|
418
|
+
/** On GitHub Actions with `id-token: write`: the job's token is asked for. */
|
|
419
|
+
private get github(): boolean {
|
|
420
|
+
return !!this.env.ACTIONS_ID_TOKEN_REQUEST_URL && !!this.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* The bearer: a token the run lent, else the job's GitHub OIDC token for
|
|
425
|
+
* the server's origin, asked again a minute before it ends, else
|
|
426
|
+
* `VX_CACHE_TOKEN` (GitLab's `id_tokens`). A lent token beats the job's
|
|
427
|
+
* own: a job of another workflow writes nowhere the run reads. A token
|
|
428
|
+
* GitHub would not give throws once, and the layer turns off.
|
|
429
|
+
*/
|
|
430
|
+
private async bearer(marginMs = 60_000): Promise<string> {
|
|
431
|
+
const lent = this.lent?.()
|
|
432
|
+
if (lent !== undefined) return lent
|
|
433
|
+
if (!this.github) return this.env.VX_CACHE_TOKEN ?? ''
|
|
434
|
+
if (this.idToken !== undefined && Date.now() < this.idToken.exp - marginMs) {
|
|
435
|
+
return this.idToken.jwt
|
|
436
|
+
}
|
|
437
|
+
// One ask for the requests that need it at once: each asked its own,
|
|
438
|
+
// a check and two uploads three (2026-10-26).
|
|
439
|
+
this.asking ??= this.askGithub().finally(() => (this.asking = undefined))
|
|
440
|
+
return this.asking
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
private asking: Promise<string> | undefined
|
|
444
|
+
/** A failed ask was said: the requests that shared it say nothing. */
|
|
445
|
+
private tokenFailed = false
|
|
446
|
+
|
|
447
|
+
/**
|
|
448
|
+
* One answer from GitHub's OIDC endpoint, asked again after a 5xx, a 429
|
|
449
|
+
* or none, up to three tries: a DTE run asks every few minutes for the
|
|
450
|
+
* token it lends, and one 503 turned the remote off for the rest of the
|
|
451
|
+
* run on every worker. A refusal (a 4xx) is final.
|
|
452
|
+
*/
|
|
453
|
+
private async askOnce(ask: URL): Promise<Response> {
|
|
454
|
+
for (let attempt = 1; ; attempt++) {
|
|
455
|
+
let res: Response | undefined
|
|
456
|
+
let failure: unknown
|
|
457
|
+
try {
|
|
458
|
+
res = await this.fetchImpl(ask, {
|
|
459
|
+
headers: { authorization: `bearer ${this.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN!}` },
|
|
460
|
+
signal: deadline(this.config.timeoutMs),
|
|
461
|
+
})
|
|
462
|
+
} catch (err) {
|
|
463
|
+
failure = err
|
|
464
|
+
}
|
|
465
|
+
if (res !== undefined && res.status !== 429 && res.status < 500) return res
|
|
466
|
+
if (attempt === ASK_ATTEMPTS) {
|
|
467
|
+
if (res === undefined) throw failure
|
|
468
|
+
return res
|
|
469
|
+
}
|
|
470
|
+
await res?.body?.cancel()
|
|
471
|
+
await this.wait(1000 * attempt)
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
private async askGithub(): Promise<string> {
|
|
476
|
+
const ask = new URL(this.env.ACTIONS_ID_TOKEN_REQUEST_URL!)
|
|
477
|
+
ask.searchParams.set('audience', new URL(this.config.url).origin)
|
|
478
|
+
let jwt: unknown
|
|
479
|
+
try {
|
|
480
|
+
const res = await this.askOnce(ask)
|
|
481
|
+
if (!res.ok) throw await refusal(res)
|
|
482
|
+
jwt = ((await jsonUpTo(res, SMALL_ANSWER_BYTES, 'it')) as { value?: unknown } | null)?.value
|
|
483
|
+
if (typeof jwt !== 'string' || jwt === '') throw new Error('no token in its answer')
|
|
484
|
+
} catch (err) {
|
|
485
|
+
this.disabled = true
|
|
486
|
+
throw new Error(
|
|
487
|
+
`GitHub's OIDC endpoint: ${err instanceof Error ? err.message : String(err)}; remote cache off for this run`,
|
|
488
|
+
)
|
|
489
|
+
}
|
|
490
|
+
this.idToken = { jwt, exp: expiryOf(jwt) }
|
|
491
|
+
return jwt
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
/**
|
|
495
|
+
* This job's token, for a job that runs this run's tasks
|
|
496
|
+
* (`RemoteCacheLayer.lend`): its writes land where this run reads them.
|
|
497
|
+
* GitHub's is asked again when it ends within five minutes. A token lent
|
|
498
|
+
* to this job is never lent on, nor one once the remote is off: core
|
|
499
|
+
* lends while its policy reads, and a refused token lent was refused on
|
|
500
|
+
* every worker.
|
|
501
|
+
*/
|
|
502
|
+
async lend(): Promise<LentAccess | undefined> {
|
|
503
|
+
if (this.disabled || this.lent?.() !== undefined) return undefined
|
|
504
|
+
const jwt = await this.bearer(LEND_MARGIN_MS)
|
|
505
|
+
if (jwt === '') return undefined
|
|
506
|
+
const exp = this.github ? this.idToken!.exp : expiryOf(jwt)
|
|
507
|
+
// Its real end: the run lends again a few minutes before it (vx-dte's
|
|
508
|
+
// RELEND_MS). Less a margin, GitHub's five-minute token ended as it was
|
|
509
|
+
// lent, and the run lent again at once, in a loop.
|
|
510
|
+
return { token: jwt, ...(exp === 0 ? {} : { expiresAt: exp }) }
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
private async request(
|
|
514
|
+
path: string,
|
|
515
|
+
init: {
|
|
516
|
+
method?: string
|
|
517
|
+
body?: Blob | string
|
|
518
|
+
headers?: Record<string, string>
|
|
519
|
+
timeoutMs?: number
|
|
520
|
+
write?: boolean
|
|
521
|
+
/** A body read at its own pace: the deadline then bounds its pace (`streamed`). */
|
|
522
|
+
streamed?: boolean
|
|
523
|
+
/** A body sent at its own pace: the deadline bounds its pace, then the answer (`paced`). */
|
|
524
|
+
paced?: boolean
|
|
525
|
+
} = {},
|
|
526
|
+
): Promise<Response | undefined> {
|
|
527
|
+
let bearer: string
|
|
528
|
+
try {
|
|
529
|
+
bearer = await this.bearer()
|
|
530
|
+
} catch (err) {
|
|
531
|
+
// The requests waiting on one ask share its failure: the first says it.
|
|
532
|
+
if (this.tokenFailed) return undefined
|
|
533
|
+
this.tokenFailed = true
|
|
534
|
+
throw err
|
|
535
|
+
}
|
|
536
|
+
const ms = init.timeoutMs ?? this.config.timeoutMs
|
|
537
|
+
const controller = new AbortController()
|
|
538
|
+
let answer: ReturnType<typeof setTimeout> | undefined
|
|
539
|
+
const arm = () => {
|
|
540
|
+
if (answer !== undefined) return
|
|
541
|
+
answer = setTimeout(() => controller.abort(new Error(`no answer within ${ms} ms`)), ms)
|
|
542
|
+
answer.unref()
|
|
543
|
+
}
|
|
544
|
+
const sending =
|
|
545
|
+
init.paced === true && init.body instanceof Blob
|
|
546
|
+
? paced(init.body, controller, ms, arm)
|
|
547
|
+
: undefined
|
|
548
|
+
if (sending === undefined) arm()
|
|
549
|
+
const url = `${this.config.url}${path}`
|
|
550
|
+
let res = await this.fetchImpl(url, {
|
|
551
|
+
method: init.method ?? 'POST',
|
|
552
|
+
headers: {
|
|
553
|
+
...this.config.headers,
|
|
554
|
+
...this.named,
|
|
555
|
+
...(bearer === '' ? {} : { authorization: `Bearer ${bearer}` }),
|
|
556
|
+
...init.headers,
|
|
557
|
+
// A stream names no length, and the server refuses an upload without one.
|
|
558
|
+
...(sending === undefined ? {} : { 'content-length': String((init.body as Blob).size) }),
|
|
559
|
+
},
|
|
560
|
+
...(init.body === undefined ? {} : { body: sending?.body ?? init.body }),
|
|
561
|
+
signal: controller.signal,
|
|
562
|
+
redirect: 'manual',
|
|
563
|
+
})
|
|
564
|
+
// An answer before the last byte (one the scope holds): its body is bounded as any answer's.
|
|
565
|
+
sending?.stop()
|
|
566
|
+
arm()
|
|
567
|
+
if (res.status >= 300 && res.status < 400) {
|
|
568
|
+
clearTimeout(answer)
|
|
569
|
+
throw await redirected(res, url)
|
|
570
|
+
}
|
|
571
|
+
// Any answer's, a refusal's too: one refused for its size by a server
|
|
572
|
+
// whose limits fell mid-run names them for the next check and upload.
|
|
573
|
+
this.limits = limitsNamed(res) ?? this.limits
|
|
574
|
+
if (init.streamed === true) {
|
|
575
|
+
clearTimeout(answer)
|
|
576
|
+
res = streamed(res, controller, ms)
|
|
577
|
+
}
|
|
578
|
+
if (init.write === true && res.status === 403) {
|
|
579
|
+
const first = !this.writesRefused
|
|
580
|
+
this.writesRefused = true
|
|
581
|
+
// The first says it; an unread body would hold its connection.
|
|
582
|
+
if (!first) {
|
|
583
|
+
await res.body?.cancel()
|
|
584
|
+
return undefined
|
|
585
|
+
}
|
|
586
|
+
throw new Error(
|
|
587
|
+
`${(await refusal(res)).message}: the write was refused (this job reads only); writes off for this run, reads go on`,
|
|
588
|
+
)
|
|
589
|
+
}
|
|
590
|
+
if (res.status === 401 || res.status === 403) {
|
|
591
|
+
const first = !this.disabled
|
|
592
|
+
this.disabled = true
|
|
593
|
+
if (!first) {
|
|
594
|
+
await res.body?.cancel()
|
|
595
|
+
return undefined
|
|
596
|
+
}
|
|
597
|
+
throw new Error(
|
|
598
|
+
`${(await refusal(res)).message}: ${bearer === '' ? 'a read with no token was refused' : `the token was refused (this workspace names ${this.config.repository} at ${this.config.issuer})`}; remote cache off for this run`,
|
|
599
|
+
)
|
|
600
|
+
}
|
|
601
|
+
return res
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
private json(path: string, body: unknown): Promise<Response | undefined> {
|
|
605
|
+
return this.request(path, {
|
|
606
|
+
body: JSON.stringify(body),
|
|
607
|
+
headers: { 'content-type': 'application/json' },
|
|
608
|
+
})
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
/**
|
|
612
|
+
* The server's limits: `/v1/capabilities`, asked once and kept, then
|
|
613
|
+
* whatever each check names. Unnamed, said once: checks go one hash at
|
|
614
|
+
* a time.
|
|
615
|
+
*/
|
|
616
|
+
private async limitsNow(): Promise<Limits | undefined> {
|
|
617
|
+
if (this.limits !== undefined) return this.limits
|
|
618
|
+
const asked = (this.asked ??= (async () => {
|
|
619
|
+
const res = await this.request('/v1/capabilities', { method: 'GET' })
|
|
620
|
+
if (res === undefined) return undefined
|
|
621
|
+
if (res.status !== 200) throw await refusal(res)
|
|
622
|
+
const body = (await jsonOf(res, '/v1/capabilities', SMALL_ANSWER_BYTES)) as Record<
|
|
623
|
+
string,
|
|
624
|
+
unknown
|
|
625
|
+
> | null
|
|
626
|
+
const named = limitsOf((k) => body?.[k])
|
|
627
|
+
if (named === undefined) this.limitsUnnamed(`/v1/capabilities (HTTP ${res.status})`)
|
|
628
|
+
return named
|
|
629
|
+
})())
|
|
630
|
+
// A failed ask is asked again by the next check: kept, two failures
|
|
631
|
+
// while a server restarted failed every read of the run (2026-10-26).
|
|
632
|
+
// quiet: the awaited ask rethrows it; this only forgets it
|
|
633
|
+
asked.catch(() => {
|
|
634
|
+
if (this.asked === asked) this.asked = undefined
|
|
635
|
+
})
|
|
636
|
+
this.limits ??= await asked
|
|
637
|
+
return this.limits
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
private limitsUnnamed(where: string): void {
|
|
641
|
+
if (this.limitsMissing) return
|
|
642
|
+
this.limitsMissing = true
|
|
643
|
+
this.warn(
|
|
644
|
+
`[vx] vx-cache: ${where} named no limits (maxCheckHashes, maxUploadBytes); checks go one hash at a time`,
|
|
645
|
+
)
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
/** A refusal met while starting, which no call could throw: the next check or download does. */
|
|
649
|
+
private unsaid: Error | undefined
|
|
650
|
+
/** Refusals a call threw: a check that waited on the start's ask threw it too. */
|
|
651
|
+
private readonly said = new WeakSet<object>()
|
|
652
|
+
|
|
653
|
+
/** Throws the refusal met while starting, once. */
|
|
654
|
+
private sayUnsaid(): void {
|
|
655
|
+
const err = this.unsaid
|
|
656
|
+
this.unsaid = undefined
|
|
657
|
+
if (err !== undefined && !this.said.has(err)) {
|
|
658
|
+
this.said.add(err)
|
|
659
|
+
throw err
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
/**
|
|
664
|
+
* Core's `open`: called only when the run's policy reads or writes this
|
|
665
|
+
* remote, so a `--no-cache` run or a plan asks the server (and GitHub)
|
|
666
|
+
* nothing.
|
|
667
|
+
*/
|
|
668
|
+
async open(): Promise<void> {
|
|
669
|
+
this.start()
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
/** Starts asking for the limits, so the first check need not wait for them. */
|
|
673
|
+
start(): void {
|
|
674
|
+
this.limitsNow().catch((err: unknown) => {
|
|
675
|
+
// A refusal turned the layer off, and is said only once: the first
|
|
676
|
+
// check says it. Anything else, the first check asks again and says.
|
|
677
|
+
if (this.disabled) this.unsaid = err instanceof Error ? err : new Error(String(err))
|
|
678
|
+
})
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
async has(hash: string): Promise<ArtifactInfo | null> {
|
|
682
|
+
return (await this.hasMany([hash]))?.get(hash) ?? null
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
/**
|
|
686
|
+
* One check of the run's refs' scopes, in order: each hash answers the
|
|
687
|
+
* first that holds it. Null once the layer is off: no answer, which core
|
|
688
|
+
* keeps apart from "none held" (an empty map rewrote every key it asked
|
|
689
|
+
* as a miss in `remote_hits`, and the next run fetched none of them).
|
|
690
|
+
*/
|
|
691
|
+
async hasMany(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
|
|
692
|
+
const held = new Map<string, ArtifactInfo>()
|
|
693
|
+
this.sayUnsaid()
|
|
694
|
+
if (this.disabled || this.refs.length === 0) return null
|
|
695
|
+
const scopes = this.refs.map(scopeOf)
|
|
696
|
+
const ask = [...new Set(hashes)]
|
|
697
|
+
let limits: Limits | undefined
|
|
698
|
+
try {
|
|
699
|
+
limits = await this.limitsNow()
|
|
700
|
+
} catch (err) {
|
|
701
|
+
// A refusal turned the layer off: each waiter shares it, the first says it.
|
|
702
|
+
if (this.disabled && err instanceof Error) {
|
|
703
|
+
if (this.said.has(err)) return null
|
|
704
|
+
this.said.add(err)
|
|
705
|
+
}
|
|
706
|
+
throw err
|
|
707
|
+
}
|
|
708
|
+
const max = limits?.maxCheckHashes ?? scopes.length
|
|
709
|
+
// More refs than one check takes: each hash's scopes span several
|
|
710
|
+
// checks, the first in order holding it answering (a check of every
|
|
711
|
+
// scope at once was refused, every time).
|
|
712
|
+
const span = Math.min(scopes.length, max)
|
|
713
|
+
const per = Math.max(1, Math.floor(max / span))
|
|
714
|
+
const spans = chunks(scopes, span)
|
|
715
|
+
const parts = await Promise.all(
|
|
716
|
+
chunks(ask, per).flatMap((some) => spans.map((s) => this.check(s, some))),
|
|
717
|
+
)
|
|
718
|
+
for (const part of parts)
|
|
719
|
+
for (const [hash, { size, scope }] of part) {
|
|
720
|
+
if (held.has(hash)) continue
|
|
721
|
+
held.set(hash, { size })
|
|
722
|
+
this.found.set(hash, scope)
|
|
723
|
+
}
|
|
724
|
+
return held
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
/** `POST /v1/artifacts` (Turbo's batch query): the size and scope of each hash held. */
|
|
728
|
+
private async check(
|
|
729
|
+
scopes: readonly string[],
|
|
730
|
+
hashes: readonly string[],
|
|
731
|
+
): Promise<Map<string, { size: number; scope: string }>> {
|
|
732
|
+
const res = await this.json('/v1/artifacts', { hashes, scopes })
|
|
733
|
+
if (res === undefined) return new Map()
|
|
734
|
+
if (res.status !== 200) throw await refusal(res)
|
|
735
|
+
if (limitsNamed(res) === undefined) this.limitsUnnamed(`the check (HTTP ${res.status})`)
|
|
736
|
+
const body = (await jsonOf(
|
|
737
|
+
res,
|
|
738
|
+
'/v1/artifacts',
|
|
739
|
+
SMALL_ANSWER_BYTES + hashes.length * CHECK_BYTES_PER_HASH,
|
|
740
|
+
)) as Record<string, unknown> | null
|
|
741
|
+
if (typeof body !== 'object' || body === null || Array.isArray(body)) {
|
|
742
|
+
throw new Error('the check answered no object of hashes')
|
|
743
|
+
}
|
|
744
|
+
const out = new Map<string, { size: number; scope: string }>()
|
|
745
|
+
for (const hash of hashes) {
|
|
746
|
+
const info = body[hash]
|
|
747
|
+
if (info === null) continue
|
|
748
|
+
const { size, scope } = (info ?? {}) as { size?: unknown; scope?: unknown }
|
|
749
|
+
if (
|
|
750
|
+
typeof size !== 'number' ||
|
|
751
|
+
!(size >= 0) ||
|
|
752
|
+
typeof scope !== 'string' ||
|
|
753
|
+
!scopes.includes(scope)
|
|
754
|
+
) {
|
|
755
|
+
throw new Error(`the check answered no size and scope asked, or null, for ${hash}`)
|
|
756
|
+
}
|
|
757
|
+
out.set(hash, { size, scope })
|
|
758
|
+
}
|
|
759
|
+
return out
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
/** The scope a check found it under, else each ref's in order until one holds it. */
|
|
763
|
+
async get(hash: string): Promise<{ body: Response; durationMs: undefined } | null> {
|
|
764
|
+
this.sayUnsaid()
|
|
765
|
+
const at = this.found.get(hash)
|
|
766
|
+
for (const scope of at === undefined ? this.refs.map(scopeOf) : [at]) {
|
|
767
|
+
if (this.disabled) return null
|
|
768
|
+
const res = await this.request(`/v1/artifacts/${hash}`, {
|
|
769
|
+
method: 'GET',
|
|
770
|
+
headers: { 'vx-scope': scope },
|
|
771
|
+
streamed: true,
|
|
772
|
+
})
|
|
773
|
+
if (res === undefined) return null
|
|
774
|
+
if (res.status === 404) {
|
|
775
|
+
await res.body?.cancel()
|
|
776
|
+
continue
|
|
777
|
+
}
|
|
778
|
+
if (res.status !== 200) throw await refusal(res)
|
|
779
|
+
return { body: res, durationMs: undefined }
|
|
780
|
+
}
|
|
781
|
+
return null
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
/** An artifact over the server's upload limit is not sent: it is said. */
|
|
785
|
+
async put(hash: string, body: Blob, _meta: { durationMs: number }): Promise<void | false> {
|
|
786
|
+
// With no token a job reads only: nothing is uploaded, and none counted.
|
|
787
|
+
if (this.disabled || this.writesRefused || !hasCiToken(this.env, this.lent?.() !== undefined))
|
|
788
|
+
return false
|
|
789
|
+
const max = this.limits?.maxUploadBytes
|
|
790
|
+
if (max !== undefined && body.size > max) {
|
|
791
|
+
this.warn(
|
|
792
|
+
`[vx] vx-cache: ${hash} is ${body.size} bytes, over the server's upload limit of ${max}: not uploaded`,
|
|
793
|
+
)
|
|
794
|
+
return false
|
|
795
|
+
}
|
|
796
|
+
return this.uploads.push({ hash, body })
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
/** One artifact by `PUT`: the server answers one its scope holds before reading the bytes. */
|
|
800
|
+
private async upload(item: { hash: string; body: Blob }): Promise<void | false> {
|
|
801
|
+
// Queued before the remote or its writes were turned off.
|
|
802
|
+
if (this.disabled || this.writesRefused) return false
|
|
803
|
+
const res = await this.request(`/v1/artifacts/${item.hash}`, {
|
|
804
|
+
method: 'PUT',
|
|
805
|
+
body: item.body,
|
|
806
|
+
headers: { 'content-type': 'application/octet-stream' },
|
|
807
|
+
timeoutMs: this.config.uploadTimeoutMs,
|
|
808
|
+
write: true,
|
|
809
|
+
paced: true,
|
|
810
|
+
})
|
|
811
|
+
if (res === undefined) return false
|
|
812
|
+
if (!res.ok) throw await refusal(res)
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
/** The issuer and repository as a key names them: the issuer's URL with no trailing slash, `org/name` lowercased. */
|
|
817
|
+
function namedOf(options: VxCacheOptions): { issuer: string; repository: string } {
|
|
818
|
+
const { issuer, repository } = options as Partial<VxCacheOptions>
|
|
819
|
+
if (typeof issuer !== 'string' || !/^https?:\/\/\S+$/.test(issuer))
|
|
820
|
+
throw new UserError(
|
|
821
|
+
`vxCache(): issuer must be the CI issuer's URL (https://token.actions.githubusercontent.com), got ${String(issuer)}`,
|
|
822
|
+
)
|
|
823
|
+
if (typeof repository !== 'string' || !/^[^/\s]+(\/[^/\s]+)+$/.test(repository))
|
|
824
|
+
throw new UserError(`vxCache(): repository must be org/name, got ${String(repository)}`)
|
|
825
|
+
return { issuer: issuer.replace(/\/$/, ''), repository: repository.toLowerCase() }
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
/** The longest delay a timer holds. */
|
|
829
|
+
const MAX_TIMER_MS = 2 ** 31 - 1
|
|
830
|
+
|
|
831
|
+
/**
|
|
832
|
+
* https, or http to this machine: every request carries the CI token, and
|
|
833
|
+
* http:// to another host sent it in clear.
|
|
834
|
+
*/
|
|
835
|
+
function secure(url: string): boolean {
|
|
836
|
+
if (/^https:\/\/\S/.test(url)) return true
|
|
837
|
+
if (!/^http:\/\/\S/.test(url)) return false
|
|
838
|
+
const host = URL.parse(url)?.hostname
|
|
839
|
+
return host === '127.0.0.1' || host === '[::1]' || host === 'localhost'
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
/** Options over `VX_CACHE_URL`; `undefined`: no server named. */
|
|
843
|
+
export function resolveVxCacheConfig(
|
|
844
|
+
options: VxCacheOptions,
|
|
845
|
+
env: Record<string, string | undefined> = Bun.env,
|
|
846
|
+
): ResolvedVxCacheConfig | undefined {
|
|
847
|
+
const named = namedOf(options)
|
|
848
|
+
const url = (options.url ?? env.VX_CACHE_URL)?.replace(/\/+$/, '')
|
|
849
|
+
if (url === undefined || url === '') return undefined
|
|
850
|
+
// Named as the workspace set it: a bad VX_CACHE_URL was refused as `url`.
|
|
851
|
+
const from = options.url === undefined ? 'VX_CACHE_URL' : 'url'
|
|
852
|
+
if (!secure(url))
|
|
853
|
+
throw new UserError(
|
|
854
|
+
`vxCache(): ${from} must be https (http only to 127.0.0.1, ::1 or localhost), got ${url}`,
|
|
855
|
+
)
|
|
856
|
+
// Each route is the url's /v1/…: one that names /v1 asked /v1/v1/…, and every answer was a 404.
|
|
857
|
+
if (url.endsWith('/v1'))
|
|
858
|
+
throw new UserError(`vxCache(): ${from} is the server's base, without /v1: ${url.slice(0, -3)}`)
|
|
859
|
+
return {
|
|
860
|
+
url,
|
|
861
|
+
...named,
|
|
862
|
+
timeoutMs: options.timeoutMs ?? 30_000,
|
|
863
|
+
uploadTimeoutMs: options.uploadTimeoutMs ?? 60_000,
|
|
864
|
+
headers: proxyHeaders('vxCache()', options.headers),
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
export interface ResolvedVxCacheConfig extends Required<Omit<VxCacheOptions, 'headers'>> {
|
|
869
|
+
readonly headers: Readonly<Record<string, string>>
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
/** Whether this job has a CI token to send: GitHub's, GitLab's `VX_CACHE_TOKEN`, or one a run lent. */
|
|
873
|
+
export function hasCiToken(env: Record<string, string | undefined>, lent: boolean): boolean {
|
|
874
|
+
return (
|
|
875
|
+
lent ||
|
|
876
|
+
(!!env.ACTIONS_ID_TOKEN_REQUEST_URL && !!env.ACTIONS_ID_TOKEN_REQUEST_TOKEN) ||
|
|
877
|
+
!!env.VX_CACHE_TOKEN
|
|
878
|
+
)
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
/**
|
|
882
|
+
* Declare in `.vx/workspace.ts`; the local store stays the floor beneath it:
|
|
883
|
+
*
|
|
884
|
+
* ```ts
|
|
885
|
+
* plugins: [
|
|
886
|
+
* vxCache({
|
|
887
|
+
* url: 'https://cache.example.com',
|
|
888
|
+
* issuer: 'https://token.actions.githubusercontent.com',
|
|
889
|
+
* repository: 'acme/app',
|
|
890
|
+
* }),
|
|
891
|
+
* ]
|
|
892
|
+
* ```
|
|
893
|
+
*/
|
|
894
|
+
export function vxCache(options: VxCacheOptions): VxPlugin {
|
|
895
|
+
refuseUnknownOptions('vxCache()', options, KEYS)
|
|
896
|
+
// A NaN, zero or negative deadline aborted every request as it began,
|
|
897
|
+
// and so did one over 2³¹−1: Bun runs a timer it cannot hold after 1 ms.
|
|
898
|
+
for (const name of ['timeoutMs', 'uploadTimeoutMs'] as const) {
|
|
899
|
+
const ms = options[name]
|
|
900
|
+
if (ms !== undefined && !(ms > 0 && ms <= MAX_TIMER_MS))
|
|
901
|
+
throw new UserError(
|
|
902
|
+
`vxCache() option "${name}" must be a positive number of ms up to ${MAX_TIMER_MS}, got ${ms}`,
|
|
903
|
+
)
|
|
904
|
+
}
|
|
905
|
+
proxyHeaders('vxCache()', options.headers)
|
|
906
|
+
namedOf(options)
|
|
907
|
+
// At the call, as vxHistory() checks its url: in the hook alone, a verb
|
|
908
|
+
// that opens no cache (`vx show`) took a url every run then failed on.
|
|
909
|
+
if (options.url !== undefined) resolveVxCacheConfig(options, {})
|
|
910
|
+
return definePlugin(import.meta, {
|
|
911
|
+
cache(ctx): RemoteCacheLayer | undefined {
|
|
912
|
+
const config = resolveVxCacheConfig(options)
|
|
913
|
+
if (config === undefined) return undefined
|
|
914
|
+
const lent = ctx.lent?.() !== undefined
|
|
915
|
+
// A GitLab job with no `id_tokens` entry read only, unsaid (2026-10-26).
|
|
916
|
+
if (
|
|
917
|
+
!hasCiToken(Bun.env, lent) &&
|
|
918
|
+
(Bun.env.GITHUB_ACTIONS === 'true' || Bun.env.GITLAB_CI === 'true')
|
|
919
|
+
)
|
|
920
|
+
ctx.warn(
|
|
921
|
+
Bun.env.GITHUB_ACTIONS === 'true'
|
|
922
|
+
? '[vx] vx-cache: this job gets no OIDC token (give it `permissions: id-token: write`): it reads only'
|
|
923
|
+
: '[vx] vx-cache: no CI token for the cache server (name an id_tokens entry VX_CACHE_TOKEN): it reads only',
|
|
924
|
+
)
|
|
925
|
+
if (ctx.refs.length === 0) {
|
|
926
|
+
ctx.warn(
|
|
927
|
+
'[vx] vx-cache: the run reads no ref, so it names no key; remote cache off for this run',
|
|
928
|
+
)
|
|
929
|
+
return undefined
|
|
930
|
+
}
|
|
931
|
+
const layer = new VxCacheLayer(config)
|
|
932
|
+
layer.refs = [...new Set(ctx.refs)]
|
|
933
|
+
layer.lent = ctx.lent
|
|
934
|
+
layer.warn = ctx.warn
|
|
935
|
+
return layer
|
|
936
|
+
},
|
|
937
|
+
})
|
|
938
|
+
}
|