@smoothbricks/nx-cache-server 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -2
- package/dist/server/nx-cache-server.js +710 -0
- package/dist/ts/config.d.ts +58 -0
- package/dist/ts/config.d.ts.map +1 -0
- package/dist/ts/config.js +116 -0
- package/dist/ts/gc.d.ts +36 -0
- package/dist/ts/gc.d.ts.map +1 -0
- package/dist/ts/gc.js +135 -0
- package/dist/ts/index.d.ts +6 -0
- package/dist/ts/index.d.ts.map +1 -0
- package/dist/ts/index.js +5 -0
- package/dist/ts/log.d.ts +16 -0
- package/dist/ts/log.d.ts.map +1 -0
- package/dist/ts/log.js +22 -0
- package/dist/ts/main.d.ts +3 -0
- package/dist/ts/main.d.ts.map +1 -0
- package/dist/ts/main.js +86 -0
- package/dist/ts/root-lease.d.ts +7 -0
- package/dist/ts/root-lease.d.ts.map +1 -0
- package/dist/ts/root-lease.js +72 -0
- package/dist/ts/server.d.ts +71 -0
- package/dist/ts/server.d.ts.map +1 -0
- package/dist/ts/server.js +282 -0
- package/dist/ts/store.d.ts +74 -0
- package/dist/ts/store.d.ts.map +1 -0
- package/dist/ts/store.js +166 -0
- package/package.json +103 -4
- package/src/__tests__/supervisor.fixture.ts +15 -0
- package/src/config.ts +201 -0
- package/src/gc.ts +164 -0
- package/src/index.ts +5 -0
- package/src/log.ts +26 -0
- package/src/main.ts +97 -0
- package/src/root-lease.ts +71 -0
- package/src/server.ts +416 -0
- package/src/store.ts +215 -0
|
@@ -0,0 +1,710 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
// @bun
|
|
3
|
+
|
|
4
|
+
// src/config.ts
|
|
5
|
+
var DEFAULT_HOST = "127.0.0.1";
|
|
6
|
+
var DEFAULT_PORT = 8765;
|
|
7
|
+
var DEFAULT_MAX_BYTES = 1024 * 1024 * 1024;
|
|
8
|
+
var DEFAULT_CAP_BYTES = 10 * 1024 * 1024 * 1024;
|
|
9
|
+
var MAX_PORT = 65535;
|
|
10
|
+
var LABEL = /^[a-z0-9][a-z0-9-]{0,31}$/;
|
|
11
|
+
var TOKEN_SOURCES = [
|
|
12
|
+
["NX_CACHE_SERVER_WRITE_TOKENS", "write"],
|
|
13
|
+
["NX_CACHE_SERVER_READ_TOKENS", "read"]
|
|
14
|
+
];
|
|
15
|
+
function cacheServerConfigFromEnv(env) {
|
|
16
|
+
const problems = [];
|
|
17
|
+
const directory = readDirectory(env, problems);
|
|
18
|
+
const port = readInteger(env, "NX_CACHE_SERVER_PORT", DEFAULT_PORT, 0, MAX_PORT, problems);
|
|
19
|
+
const maxBytes = readInteger(env, "NX_CACHE_SERVER_MAX_BYTES", DEFAULT_MAX_BYTES, 1, Number.MAX_SAFE_INTEGER, problems);
|
|
20
|
+
const capBytes = readInteger(env, "NX_CACHE_SERVER_CAP_BYTES", DEFAULT_CAP_BYTES, 1, Number.MAX_SAFE_INTEGER, problems);
|
|
21
|
+
if (maxBytes !== undefined && capBytes !== undefined && maxBytes > capBytes) {
|
|
22
|
+
problems.push("NX_CACHE_SERVER_MAX_BYTES must not exceed NX_CACHE_SERVER_CAP_BYTES.");
|
|
23
|
+
}
|
|
24
|
+
const principals = readPrincipals(env, problems);
|
|
25
|
+
const configuredHost = env.NX_CACHE_SERVER_HOST?.trim();
|
|
26
|
+
const host = configuredHost === undefined || configuredHost === "" ? DEFAULT_HOST : configuredHost;
|
|
27
|
+
if (problems.length > 0 || directory === undefined || port === undefined || maxBytes === undefined || capBytes === undefined || principals === undefined) {
|
|
28
|
+
return { ok: false, problems };
|
|
29
|
+
}
|
|
30
|
+
return { ok: true, config: { directory, host, port, maxBytes, capBytes, principals } };
|
|
31
|
+
}
|
|
32
|
+
function readDirectory(env, problems) {
|
|
33
|
+
const raw = env.NX_CACHE_SERVER_DIR?.trim();
|
|
34
|
+
if (raw === undefined || raw === "") {
|
|
35
|
+
problems.push("NX_CACHE_SERVER_DIR is not set. Name the directory that holds cache entries; there is no default because a wrong guess would start an empty cache that looks healthy.");
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
if (!raw.startsWith("/")) {
|
|
39
|
+
problems.push(`NX_CACHE_SERVER_DIR=${JSON.stringify(raw)} must be an absolute path: a supervisor starts this process, and its working directory is not the operator's.`);
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
return raw;
|
|
43
|
+
}
|
|
44
|
+
function readInteger(env, name, fallback, min, max, problems) {
|
|
45
|
+
const raw = env[name]?.trim();
|
|
46
|
+
if (raw === undefined || raw === "")
|
|
47
|
+
return fallback;
|
|
48
|
+
const value = Number(raw);
|
|
49
|
+
if (!Number.isInteger(value) || value < min || value > max) {
|
|
50
|
+
problems.push(`${name}=${JSON.stringify(raw)} must be an integer between ${min} and ${max}.`);
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
return value;
|
|
54
|
+
}
|
|
55
|
+
function readPrincipals(env, problems) {
|
|
56
|
+
const principals = [];
|
|
57
|
+
const labels = new Set;
|
|
58
|
+
const secrets = new Set;
|
|
59
|
+
for (const [envName, capability] of TOKEN_SOURCES) {
|
|
60
|
+
const raw = env[envName];
|
|
61
|
+
if (raw === undefined)
|
|
62
|
+
continue;
|
|
63
|
+
const entries = raw.split(",");
|
|
64
|
+
entries.forEach((rawEntry, index) => {
|
|
65
|
+
const entry = rawEntry.trim();
|
|
66
|
+
if (entry === "")
|
|
67
|
+
return;
|
|
68
|
+
const at = `${envName} entry ${index + 1}`;
|
|
69
|
+
const colon = entry.indexOf(":");
|
|
70
|
+
if (colon < 1) {
|
|
71
|
+
problems.push(`${at} is not "label:secret". Every token carries a label so an audit line can name the principal that published an artifact without printing the token.`);
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
const label = entry.slice(0, colon);
|
|
75
|
+
const secret = entry.slice(colon + 1);
|
|
76
|
+
if (!LABEL.test(label)) {
|
|
77
|
+
problems.push(`${at} label ${JSON.stringify(label)} must match ${LABEL.source} \u2014 lowercase, digits and dashes.`);
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
if (secret === "") {
|
|
81
|
+
problems.push(`${at} (label ${label}) has an empty secret.`);
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
if (labels.has(label)) {
|
|
85
|
+
problems.push(`${at} reuses label ${label}. One label names one principal, or the audit log lies.`);
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
if (secrets.has(secret)) {
|
|
89
|
+
problems.push(`${at} (label ${label}) repeats a secret already configured for another principal. One secret maps to exactly one label and one capability.`);
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
labels.add(label);
|
|
93
|
+
secrets.add(secret);
|
|
94
|
+
principals.push({ label, bearer: `Bearer ${secret}`, capability });
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
if (principals.length === 0) {
|
|
98
|
+
problems.push('No tokens configured. Set NX_CACHE_SERVER_WRITE_TOKENS and/or NX_CACHE_SERVER_READ_TOKENS to comma-separated "label:secret" entries; the write list may publish artifacts, the read list may only fetch them.');
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
return principals;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// src/log.ts
|
|
105
|
+
var PROGRAM = "nx-cache-server";
|
|
106
|
+
var BARE_VALUE = /^[\w.:@/+-]+$/;
|
|
107
|
+
function logLine(fields) {
|
|
108
|
+
console.error(`${PROGRAM} ${fields}`);
|
|
109
|
+
}
|
|
110
|
+
function logValue(value) {
|
|
111
|
+
return BARE_VALUE.test(value) ? value : JSON.stringify(value);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// src/gc.ts
|
|
115
|
+
import { lstat, readdir as readdir2, readFile, rename, rm, utimes } from "fs/promises";
|
|
116
|
+
import { join as join2 } from "path";
|
|
117
|
+
|
|
118
|
+
// src/store.ts
|
|
119
|
+
import { link, mkdir, open as open2, readdir, unlink } from "fs/promises";
|
|
120
|
+
import { join } from "path";
|
|
121
|
+
import { Readable } from "stream";
|
|
122
|
+
|
|
123
|
+
// src/root-lease.ts
|
|
124
|
+
import { dlopen, read } from "bun:ffi";
|
|
125
|
+
import { open } from "fs/promises";
|
|
126
|
+
import { getSystemErrorName } from "util";
|
|
127
|
+
var LOCK_EX = 2;
|
|
128
|
+
var LOCK_NB = 4;
|
|
129
|
+
function flockErrno(fd, flock, errnoLocation) {
|
|
130
|
+
if (flock(fd, LOCK_EX | LOCK_NB) === 0)
|
|
131
|
+
return 0;
|
|
132
|
+
const location = errnoLocation();
|
|
133
|
+
if (!location)
|
|
134
|
+
throw new Error("root lease: libc returned a null errno location");
|
|
135
|
+
const errno = read.i32(location, 0);
|
|
136
|
+
if (errno === 0)
|
|
137
|
+
throw new Error("root lease: flock failed but errno is 0; refusing to treat the lock as held");
|
|
138
|
+
return errno;
|
|
139
|
+
}
|
|
140
|
+
function lockErrno(fd) {
|
|
141
|
+
if (process.platform === "darwin") {
|
|
142
|
+
const libc = dlopen("libc.dylib", {
|
|
143
|
+
flock: { args: ["i32", "i32"], returns: "i32" },
|
|
144
|
+
__error: { args: [], returns: "ptr" }
|
|
145
|
+
});
|
|
146
|
+
try {
|
|
147
|
+
return flockErrno(fd, libc.symbols.flock, libc.symbols.__error);
|
|
148
|
+
} finally {
|
|
149
|
+
libc.close();
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
if (process.platform === "linux") {
|
|
153
|
+
const libc = dlopen("libc.so.6", {
|
|
154
|
+
flock: { args: ["i32", "i32"], returns: "i32" },
|
|
155
|
+
__errno_location: { args: [], returns: "ptr" }
|
|
156
|
+
});
|
|
157
|
+
try {
|
|
158
|
+
return flockErrno(fd, libc.symbols.flock, libc.symbols.__errno_location);
|
|
159
|
+
} finally {
|
|
160
|
+
libc.close();
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
throw new Error(`root lease is unsupported on platform ${process.platform}`);
|
|
164
|
+
}
|
|
165
|
+
async function tryAcquireRootLease(path) {
|
|
166
|
+
const handle = await open(path, "a", 384);
|
|
167
|
+
let owned = false;
|
|
168
|
+
try {
|
|
169
|
+
const errno = lockErrno(handle.fd);
|
|
170
|
+
if (errno === 0) {
|
|
171
|
+
owned = true;
|
|
172
|
+
return handle;
|
|
173
|
+
}
|
|
174
|
+
if (errno === (process.platform === "darwin" ? 35 : 11))
|
|
175
|
+
return;
|
|
176
|
+
throw new Error(`flock ${path} failed: ${getSystemErrorName(-errno)}`);
|
|
177
|
+
} finally {
|
|
178
|
+
if (!owned)
|
|
179
|
+
await handle.close();
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// src/store.ts
|
|
184
|
+
var STAGING = ".staging";
|
|
185
|
+
var CACHE_KEY = /^[A-Za-z0-9_-]{1,128}$/;
|
|
186
|
+
var ALREADY_STORED = { kind: "exists", stagingRetained: false };
|
|
187
|
+
function isCacheKey(candidate) {
|
|
188
|
+
return CACHE_KEY.test(candidate);
|
|
189
|
+
}
|
|
190
|
+
function entryPath(store, key) {
|
|
191
|
+
return join(store.directory, key);
|
|
192
|
+
}
|
|
193
|
+
async function openCacheStore(directory) {
|
|
194
|
+
await mkdir(directory, { recursive: true });
|
|
195
|
+
const lockPath = join(directory, ".owner.lock");
|
|
196
|
+
const lease = await tryAcquireRootLease(lockPath);
|
|
197
|
+
if (!lease)
|
|
198
|
+
return { ok: false, kind: "store-locked", path: lockPath };
|
|
199
|
+
try {
|
|
200
|
+
const staging = join(directory, STAGING);
|
|
201
|
+
await mkdir(staging, { recursive: true });
|
|
202
|
+
const abandoned = await readdir(staging);
|
|
203
|
+
for (const name of abandoned) {
|
|
204
|
+
await unlink(join(staging, name));
|
|
205
|
+
}
|
|
206
|
+
return {
|
|
207
|
+
ok: true,
|
|
208
|
+
store: { directory, staging, reclaimedStagingFiles: abandoned.length, close: () => lease.close() }
|
|
209
|
+
};
|
|
210
|
+
} catch (failure) {
|
|
211
|
+
await lease.close();
|
|
212
|
+
throw failure;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
var MISSING = { kind: "missing" };
|
|
216
|
+
async function readEntry(store, key) {
|
|
217
|
+
try {
|
|
218
|
+
const file = await open2(entryPath(store, key), "r");
|
|
219
|
+
try {
|
|
220
|
+
const stats = await file.stat();
|
|
221
|
+
if (!stats.isFile() || stats.size === 0) {
|
|
222
|
+
await file.close();
|
|
223
|
+
return MISSING;
|
|
224
|
+
}
|
|
225
|
+
if (stats.atimeMs <= stats.mtimeMs) {
|
|
226
|
+
await file.utimes((stats.mtimeMs + 1) / 1000, stats.mtimeMs / 1000);
|
|
227
|
+
}
|
|
228
|
+
return {
|
|
229
|
+
kind: "found",
|
|
230
|
+
bytes: stats.size,
|
|
231
|
+
body: Readable.toWeb(file.createReadStream({ autoClose: true }))
|
|
232
|
+
};
|
|
233
|
+
} catch (cause) {
|
|
234
|
+
await file.close();
|
|
235
|
+
return { kind: "failed", reason: describe(cause) };
|
|
236
|
+
}
|
|
237
|
+
} catch (cause) {
|
|
238
|
+
return errnoOf(cause) === "ENOENT" ? MISSING : { kind: "failed", reason: describe(cause) };
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
async function stageEntry(store, key, body, declaredBytes) {
|
|
242
|
+
const staged = join(store.staging, `${key}.${crypto.randomUUID()}`);
|
|
243
|
+
let bytes;
|
|
244
|
+
try {
|
|
245
|
+
bytes = await Bun.write(staged, new Response(body));
|
|
246
|
+
} catch (cause) {
|
|
247
|
+
const stagingRetained = !await discard(staged);
|
|
248
|
+
return { kind: "failed", reason: describe(cause), stagingRetained };
|
|
249
|
+
}
|
|
250
|
+
if (bytes !== declaredBytes) {
|
|
251
|
+
const stagingRetained = !await discard(staged);
|
|
252
|
+
return { kind: "incomplete", bytes, stagingRetained };
|
|
253
|
+
}
|
|
254
|
+
return { kind: "ready", key, stagedPath: staged, bytes };
|
|
255
|
+
}
|
|
256
|
+
async function publishEntry(store, upload) {
|
|
257
|
+
try {
|
|
258
|
+
await link(upload.stagedPath, entryPath(store, upload.key));
|
|
259
|
+
} catch (cause) {
|
|
260
|
+
const stagingRetained = !await discard(upload.stagedPath);
|
|
261
|
+
if (errnoOf(cause) === "EEXIST") {
|
|
262
|
+
return stagingRetained ? { kind: "exists", stagingRetained } : ALREADY_STORED;
|
|
263
|
+
}
|
|
264
|
+
return { kind: "failed", reason: describe(cause), stagingRetained };
|
|
265
|
+
}
|
|
266
|
+
const stagingRetained = !await discard(upload.stagedPath);
|
|
267
|
+
return { kind: "stored", bytes: upload.bytes, stagingRetained };
|
|
268
|
+
}
|
|
269
|
+
async function discard(staged) {
|
|
270
|
+
try {
|
|
271
|
+
await unlink(staged);
|
|
272
|
+
return true;
|
|
273
|
+
} catch (cause) {
|
|
274
|
+
if (errnoOf(cause) === "ENOENT")
|
|
275
|
+
return true;
|
|
276
|
+
logLine(`ev=staging.leaked path=${logValue(staged)} reason=${logValue(describe(cause))}`);
|
|
277
|
+
return false;
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
function errnoOf(cause) {
|
|
281
|
+
if (typeof cause !== "object" || cause === null || !("code" in cause))
|
|
282
|
+
return;
|
|
283
|
+
const { code } = cause;
|
|
284
|
+
return typeof code === "string" ? code : undefined;
|
|
285
|
+
}
|
|
286
|
+
function describe(cause) {
|
|
287
|
+
if (cause instanceof Error) {
|
|
288
|
+
const errno = errnoOf(cause);
|
|
289
|
+
return errno === undefined ? cause.message : `${errno}: ${cause.message}`;
|
|
290
|
+
}
|
|
291
|
+
return String(cause);
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
// src/gc.ts
|
|
295
|
+
function insertionOrder(entries) {
|
|
296
|
+
return [...entries].sort((a, b) => a.mtimeMs - b.mtimeMs || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
|
|
297
|
+
}
|
|
298
|
+
async function sweep(store, capBytes, hand, protectedPath) {
|
|
299
|
+
const ring = insertionOrder(await store.list());
|
|
300
|
+
const usedBefore = ring.reduce((sum, e) => sum + e.size, 0);
|
|
301
|
+
let used = usedBefore;
|
|
302
|
+
let deleted = 0;
|
|
303
|
+
let cleared = 0;
|
|
304
|
+
let cursor = hand;
|
|
305
|
+
if (used > capBytes && ring.length > 0) {
|
|
306
|
+
const start = ring.findIndex((e) => e.mtimeMs >= hand);
|
|
307
|
+
const order = start < 0 ? ring : [...ring.slice(start), ...ring.slice(0, start)];
|
|
308
|
+
for (let round = 0;round < 2 && used > capBytes; round++) {
|
|
309
|
+
for (const entry of order) {
|
|
310
|
+
if (used <= capBytes)
|
|
311
|
+
break;
|
|
312
|
+
if (entry.path === protectedPath)
|
|
313
|
+
continue;
|
|
314
|
+
const visited = entry.atimeMs > entry.mtimeMs;
|
|
315
|
+
if (round === 1 && !visited)
|
|
316
|
+
continue;
|
|
317
|
+
if (round === 0 && visited) {
|
|
318
|
+
await store.clearVisited(entry);
|
|
319
|
+
cleared++;
|
|
320
|
+
} else {
|
|
321
|
+
await store.evict(entry);
|
|
322
|
+
used -= entry.size;
|
|
323
|
+
deleted++;
|
|
324
|
+
cursor = entry.mtimeMs;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
return { usedBefore, usedAfter: used, deleted, cleared, hand: cursor };
|
|
330
|
+
}
|
|
331
|
+
function directoryStore(entries) {
|
|
332
|
+
return {
|
|
333
|
+
async list() {
|
|
334
|
+
const names = await readdir2(entries);
|
|
335
|
+
const out = [];
|
|
336
|
+
for (const name of names) {
|
|
337
|
+
if (!isCacheKey(name))
|
|
338
|
+
continue;
|
|
339
|
+
const path = join2(entries, name);
|
|
340
|
+
const s = await lstat(path);
|
|
341
|
+
if (s.isFile())
|
|
342
|
+
out.push({
|
|
343
|
+
path,
|
|
344
|
+
mtimeMs: s.mtimeMs,
|
|
345
|
+
atimeMs: s.atimeMs,
|
|
346
|
+
size: s.size
|
|
347
|
+
});
|
|
348
|
+
}
|
|
349
|
+
return out;
|
|
350
|
+
},
|
|
351
|
+
async clearVisited(entry) {
|
|
352
|
+
await utimes(entry.path, entry.mtimeMs / 1000, entry.mtimeMs / 1000);
|
|
353
|
+
},
|
|
354
|
+
async evict(entry) {
|
|
355
|
+
await rm(entry.path, { force: true });
|
|
356
|
+
}
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
async function sweepDirectory(store, capBytes, protectedKey) {
|
|
360
|
+
const entries = store.directory;
|
|
361
|
+
const handFile = join2(entries, "..", "sieve.hand");
|
|
362
|
+
const temporary = `${handFile}.tmp`;
|
|
363
|
+
try {
|
|
364
|
+
let hand = 0;
|
|
365
|
+
try {
|
|
366
|
+
const text = (await readFile(handFile, "utf8")).trim();
|
|
367
|
+
hand = Number(text);
|
|
368
|
+
if (text === "" || !Number.isFinite(hand) || hand < 0) {
|
|
369
|
+
return { ok: false, reason: `Invalid SIEVE cursor in ${handFile}.` };
|
|
370
|
+
}
|
|
371
|
+
} catch (cause) {
|
|
372
|
+
if (errnoOf(cause) !== "ENOENT")
|
|
373
|
+
return { ok: false, reason: describe(cause) };
|
|
374
|
+
}
|
|
375
|
+
const protectedPath = protectedKey === undefined ? undefined : entryPath(store, protectedKey);
|
|
376
|
+
const outcome = await sweep(directoryStore(entries), capBytes, hand, protectedPath);
|
|
377
|
+
await Bun.write(temporary, `${outcome.hand}
|
|
378
|
+
`);
|
|
379
|
+
await rename(temporary, handFile);
|
|
380
|
+
logLine(`ev=gc used_before=${outcome.usedBefore} used_after=${outcome.usedAfter} cap=${capBytes} deleted=${outcome.deleted} cleared=${outcome.cleared} hand=${outcome.hand}`);
|
|
381
|
+
return { ok: true, outcome };
|
|
382
|
+
} catch (cause) {
|
|
383
|
+
const reason = describe(cause);
|
|
384
|
+
logLine(`ev=gc.failed reason=${logValue(reason)}`);
|
|
385
|
+
return { ok: false, reason };
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
if (false) {}
|
|
389
|
+
|
|
390
|
+
// src/server.ts
|
|
391
|
+
var CACHE_ROUTE = "/v1/cache/:hash";
|
|
392
|
+
var OCTET_STREAM = {
|
|
393
|
+
"content-type": "application/octet-stream"
|
|
394
|
+
};
|
|
395
|
+
var PLAIN = {
|
|
396
|
+
"content-type": "text/plain"
|
|
397
|
+
};
|
|
398
|
+
var PLAIN_UNAUTHENTICATED = {
|
|
399
|
+
"content-type": "text/plain",
|
|
400
|
+
"www-authenticate": "Bearer"
|
|
401
|
+
};
|
|
402
|
+
var MISS_BODY = `No cache entry for that key.
|
|
403
|
+
`;
|
|
404
|
+
var FRAMING_CEILING = Number.MAX_SAFE_INTEGER;
|
|
405
|
+
async function startCacheServer(config) {
|
|
406
|
+
let opened;
|
|
407
|
+
try {
|
|
408
|
+
opened = await openCacheStore(config.directory);
|
|
409
|
+
} catch (cause) {
|
|
410
|
+
return { ok: false, stage: "store", directory: config.directory, reason: describe(cause) };
|
|
411
|
+
}
|
|
412
|
+
if (!opened.ok) {
|
|
413
|
+
return {
|
|
414
|
+
ok: false,
|
|
415
|
+
stage: "store-locked",
|
|
416
|
+
directory: config.directory,
|
|
417
|
+
path: opened.path,
|
|
418
|
+
reason: `EWOULDBLOCK: another process holds the store lock at ${opened.path}`
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
const store = opened.store;
|
|
422
|
+
const initial = await sweepDirectory(store, config.capBytes);
|
|
423
|
+
if (!initial.ok) {
|
|
424
|
+
await store.close();
|
|
425
|
+
return { ok: false, stage: "gc", directory: config.directory, reason: initial.reason };
|
|
426
|
+
}
|
|
427
|
+
const admission = { reservedBytes: 0 };
|
|
428
|
+
let publishing = Promise.resolve();
|
|
429
|
+
function publish(upload) {
|
|
430
|
+
const next = publishing.then(async () => {
|
|
431
|
+
const outcome = await publishEntry(store, upload);
|
|
432
|
+
if (outcome.kind !== "stored")
|
|
433
|
+
return outcome;
|
|
434
|
+
const retained = await sweepDirectory(store, config.capBytes, upload.key);
|
|
435
|
+
return retained.ok ? outcome : {
|
|
436
|
+
kind: "failed",
|
|
437
|
+
reason: retained.reason,
|
|
438
|
+
stagingRetained: outcome.stagingRetained
|
|
439
|
+
};
|
|
440
|
+
});
|
|
441
|
+
publishing = next.then(() => {});
|
|
442
|
+
return next;
|
|
443
|
+
}
|
|
444
|
+
try {
|
|
445
|
+
const server = Bun.serve({
|
|
446
|
+
hostname: config.host,
|
|
447
|
+
port: config.port,
|
|
448
|
+
maxRequestBodySize: FRAMING_CEILING,
|
|
449
|
+
routes: {
|
|
450
|
+
[CACHE_ROUTE]: {
|
|
451
|
+
GET: (request) => handleGet(store, config, request),
|
|
452
|
+
PUT: (request) => handlePut(store, config, request, admission, publish)
|
|
453
|
+
}
|
|
454
|
+
},
|
|
455
|
+
fetch(request) {
|
|
456
|
+
const { pathname } = new URL(request.url);
|
|
457
|
+
logLine(`ev=unrouted method=${logValue(request.method)} path=${logValue(pathname)} status=404`);
|
|
458
|
+
return new Response(`This server serves /v1/cache/<hash> only (GET, PUT). A path with a doubled slash means NX_SELF_HOSTED_REMOTE_CACHE_SERVER carries a trailing slash.
|
|
459
|
+
`, { status: 404, headers: PLAIN });
|
|
460
|
+
},
|
|
461
|
+
error(failure) {
|
|
462
|
+
logLine(`ev=server.error status=500 reason=${logValue(String(failure.message ?? failure))}`);
|
|
463
|
+
return new Response(`The cache server failed to handle that request; see its log.
|
|
464
|
+
`, {
|
|
465
|
+
status: 500,
|
|
466
|
+
headers: PLAIN
|
|
467
|
+
});
|
|
468
|
+
}
|
|
469
|
+
});
|
|
470
|
+
const principals = config.principals.map((principal) => `${principal.label}:${principal.capability}`).join(",");
|
|
471
|
+
logLine(`ev=listening origin=${server.url.origin} dir=${logValue(config.directory)} principals=${principals} max_bytes=${config.maxBytes} cap_bytes=${config.capBytes} reclaimed_staging=${store.reclaimedStagingFiles}`);
|
|
472
|
+
return {
|
|
473
|
+
ok: true,
|
|
474
|
+
server: {
|
|
475
|
+
origin: server.url.origin,
|
|
476
|
+
port: server.port ?? config.port,
|
|
477
|
+
reclaimedStagingFiles: store.reclaimedStagingFiles,
|
|
478
|
+
async stop() {
|
|
479
|
+
await server.stop(false);
|
|
480
|
+
await publishing;
|
|
481
|
+
await store.close();
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
};
|
|
485
|
+
} catch (cause) {
|
|
486
|
+
await store.close();
|
|
487
|
+
return { ok: false, stage: "listen", directory: config.directory, reason: describe(cause) };
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
async function handleGet(store, config, request) {
|
|
491
|
+
const startedNs = Bun.nanoseconds();
|
|
492
|
+
const key = request.params.hash;
|
|
493
|
+
const principal = authorize(config.principals, request.headers.get("authorization"));
|
|
494
|
+
if (principal === undefined) {
|
|
495
|
+
record("get", key, undefined, 401, 0, startedNs, "unauthenticated");
|
|
496
|
+
return unauthenticated();
|
|
497
|
+
}
|
|
498
|
+
if (!isCacheKey(key)) {
|
|
499
|
+
record("get", key, principal, 404, 0, startedNs, "invalid-key");
|
|
500
|
+
return new Response(MISS_BODY, { status: 404, headers: PLAIN });
|
|
501
|
+
}
|
|
502
|
+
const entry = await readEntry(store, key);
|
|
503
|
+
switch (entry.kind) {
|
|
504
|
+
case "missing":
|
|
505
|
+
record("get", key, principal, 404, 0, startedNs, "miss");
|
|
506
|
+
return new Response(MISS_BODY, { status: 404, headers: PLAIN });
|
|
507
|
+
case "failed":
|
|
508
|
+
record("get", key, principal, 500, 0, startedNs, entry.reason);
|
|
509
|
+
return new Response(`The cache server could not read that artifact; see its log.
|
|
510
|
+
`, {
|
|
511
|
+
status: 500,
|
|
512
|
+
headers: PLAIN
|
|
513
|
+
});
|
|
514
|
+
case "found":
|
|
515
|
+
record("get", key, principal, 200, entry.bytes, startedNs, undefined);
|
|
516
|
+
return new Response(entry.body, { status: 200, headers: OCTET_STREAM });
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
async function handlePut(store, config, request, admission, publish) {
|
|
520
|
+
const startedNs = Bun.nanoseconds();
|
|
521
|
+
const key = request.params.hash;
|
|
522
|
+
const principal = authorize(config.principals, request.headers.get("authorization"));
|
|
523
|
+
if (principal === undefined) {
|
|
524
|
+
record("put", key, undefined, 401, 0, startedNs, "unauthenticated");
|
|
525
|
+
return unauthenticated();
|
|
526
|
+
}
|
|
527
|
+
if (principal.capability !== "write") {
|
|
528
|
+
record("put", key, principal, 403, 0, startedNs, "read-only-token");
|
|
529
|
+
return new Response(`This token may read the cache but not publish into it.
|
|
530
|
+
`, { status: 403, headers: PLAIN });
|
|
531
|
+
}
|
|
532
|
+
if (!isCacheKey(key)) {
|
|
533
|
+
record("put", key, principal, 400, 0, startedNs, "invalid-key");
|
|
534
|
+
return new Response(`A cache key is 1-128 characters of [A-Za-z0-9_-].
|
|
535
|
+
`, {
|
|
536
|
+
status: 400,
|
|
537
|
+
headers: PLAIN
|
|
538
|
+
});
|
|
539
|
+
}
|
|
540
|
+
const header = request.headers.get("content-length");
|
|
541
|
+
if (header === null) {
|
|
542
|
+
record("put", key, principal, 411, 0, startedNs, "no-declared-length");
|
|
543
|
+
return new Response(`A cache artifact must arrive with a content-length.
|
|
544
|
+
`, { status: 411, headers: PLAIN });
|
|
545
|
+
}
|
|
546
|
+
const declared = Number(header);
|
|
547
|
+
if (!Number.isInteger(declared) || declared < 1) {
|
|
548
|
+
record("put", key, principal, 400, 0, startedNs, "bad-declared-length");
|
|
549
|
+
return new Response(`content-length must be a positive integer.
|
|
550
|
+
`, {
|
|
551
|
+
status: 400,
|
|
552
|
+
headers: PLAIN
|
|
553
|
+
});
|
|
554
|
+
}
|
|
555
|
+
if (declared > config.maxBytes) {
|
|
556
|
+
record("put", key, principal, 403, declared, startedNs, "over-max-bytes");
|
|
557
|
+
return new Response(`This artifact exceeds the server's ${config.maxBytes}-byte limit.
|
|
558
|
+
`, {
|
|
559
|
+
status: 403,
|
|
560
|
+
headers: PLAIN
|
|
561
|
+
});
|
|
562
|
+
}
|
|
563
|
+
if (declared > config.capBytes - admission.reservedBytes) {
|
|
564
|
+
record("put", key, principal, 403, declared, startedNs, "staging-capacity");
|
|
565
|
+
return new Response(`The staging byte budget is full; nothing was stored.
|
|
566
|
+
`, {
|
|
567
|
+
status: 403,
|
|
568
|
+
headers: PLAIN
|
|
569
|
+
});
|
|
570
|
+
}
|
|
571
|
+
admission.reservedBytes += declared;
|
|
572
|
+
logLine(`ev=staging.reserved key=${logValue(key)} declared_bytes=${declared} reserved_bytes=${admission.reservedBytes} cap=${config.capBytes}`);
|
|
573
|
+
let stagingRetained = false;
|
|
574
|
+
try {
|
|
575
|
+
const staged = await stageEntry(store, key, request.body, declared);
|
|
576
|
+
if (staged.kind !== "ready") {
|
|
577
|
+
stagingRetained = staged.stagingRetained;
|
|
578
|
+
if (staged.kind === "incomplete") {
|
|
579
|
+
record("put", key, principal, 400, staged.bytes, startedNs, "body-shorter-than-declared");
|
|
580
|
+
return new Response(`That body ended after ${staged.bytes} of the ${declared} bytes it declared; nothing was stored.
|
|
581
|
+
`, { status: 400, headers: PLAIN });
|
|
582
|
+
}
|
|
583
|
+
record("put", key, principal, 500, 0, startedNs, staged.reason);
|
|
584
|
+
return new Response(`The cache server could not stage that artifact; see its log.
|
|
585
|
+
`, {
|
|
586
|
+
status: 500,
|
|
587
|
+
headers: PLAIN
|
|
588
|
+
});
|
|
589
|
+
}
|
|
590
|
+
const outcome = await publish(staged);
|
|
591
|
+
stagingRetained = outcome.stagingRetained;
|
|
592
|
+
if (outcome.kind === "stored") {
|
|
593
|
+
record("put", key, principal, 200, outcome.bytes, startedNs, undefined);
|
|
594
|
+
return new Response(`stored
|
|
595
|
+
`, { status: 200, headers: PLAIN });
|
|
596
|
+
}
|
|
597
|
+
if (outcome.kind === "exists") {
|
|
598
|
+
record("put", key, principal, 409, 0, startedNs, "already-stored");
|
|
599
|
+
return new Response(`That key already holds an artifact; a cache entry is never replaced.
|
|
600
|
+
`, {
|
|
601
|
+
status: 409,
|
|
602
|
+
headers: PLAIN
|
|
603
|
+
});
|
|
604
|
+
}
|
|
605
|
+
record("put", key, principal, 500, 0, startedNs, outcome.reason);
|
|
606
|
+
return new Response(`The cache server could not publish or retain that artifact; see its log.
|
|
607
|
+
`, {
|
|
608
|
+
status: 500,
|
|
609
|
+
headers: PLAIN
|
|
610
|
+
});
|
|
611
|
+
} finally {
|
|
612
|
+
if (stagingRetained) {
|
|
613
|
+
logLine(`ev=staging.reservation-retained key=${logValue(key)} declared_bytes=${declared} reserved_bytes=${admission.reservedBytes} reason=cleanup-failed`);
|
|
614
|
+
} else {
|
|
615
|
+
admission.reservedBytes -= declared;
|
|
616
|
+
logLine(`ev=staging.released key=${logValue(key)} declared_bytes=${declared} reserved_bytes=${admission.reservedBytes}`);
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
function authorize(principals, header) {
|
|
621
|
+
if (header === null)
|
|
622
|
+
return;
|
|
623
|
+
let authorized;
|
|
624
|
+
for (const principal of principals) {
|
|
625
|
+
if (secretEquals(header, principal.bearer))
|
|
626
|
+
authorized = principal;
|
|
627
|
+
}
|
|
628
|
+
return authorized;
|
|
629
|
+
}
|
|
630
|
+
function secretEquals(received, expected) {
|
|
631
|
+
if (received.length !== expected.length)
|
|
632
|
+
return false;
|
|
633
|
+
let difference = 0;
|
|
634
|
+
for (let index = 0;index < expected.length; index += 1) {
|
|
635
|
+
difference |= received.charCodeAt(index) ^ expected.charCodeAt(index);
|
|
636
|
+
}
|
|
637
|
+
return difference === 0;
|
|
638
|
+
}
|
|
639
|
+
function unauthenticated() {
|
|
640
|
+
return new Response("Missing or unknown bearer token. Send `Authorization: Bearer <token>` with a token this server was started with (NX_CACHE_SERVER_WRITE_TOKENS or NX_CACHE_SERVER_READ_TOKENS); nx sends the one in NX_SELF_HOSTED_REMOTE_CACHE_ACCESS_TOKEN.\n", { status: 401, headers: PLAIN_UNAUTHENTICATED });
|
|
641
|
+
}
|
|
642
|
+
function record(event, key, principal, status, bytes, startedNs, reason) {
|
|
643
|
+
const microseconds = Math.round((Bun.nanoseconds() - startedNs) / 1000);
|
|
644
|
+
const attribution = principal === undefined ? "-" : principal.label;
|
|
645
|
+
const because = reason === undefined ? "" : ` reason=${logValue(reason)}`;
|
|
646
|
+
logLine(`ev=${event} key=${logValue(key)} principal=${attribution} status=${status} bytes=${bytes} us=${microseconds}${because}`);
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
// src/main.ts
|
|
650
|
+
var exitOnStdinEof = false;
|
|
651
|
+
var argumentProblems = [];
|
|
652
|
+
for (let position = 2;position < Bun.argv.length; position++) {
|
|
653
|
+
if (Bun.argv[position] === "--exit-on-stdin-eof") {
|
|
654
|
+
exitOnStdinEof = true;
|
|
655
|
+
} else {
|
|
656
|
+
argumentProblems.push(`argument ${position - 1} is unsupported; the only supported argument is --exit-on-stdin-eof`);
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
if (argumentProblems.length > 0) {
|
|
660
|
+
console.log(JSON.stringify({ kind: "failed", stage: "config", problems: argumentProblems }));
|
|
661
|
+
console.error("nx-cache-server refuses to start:");
|
|
662
|
+
for (const problem of argumentProblems) {
|
|
663
|
+
console.error(` - ${problem}`);
|
|
664
|
+
}
|
|
665
|
+
process.exit(2);
|
|
666
|
+
}
|
|
667
|
+
var configured = cacheServerConfigFromEnv(Bun.env);
|
|
668
|
+
if (!configured.ok) {
|
|
669
|
+
console.log(JSON.stringify({ kind: "failed", stage: "config", problems: configured.problems }));
|
|
670
|
+
console.error("nx-cache-server refuses to start:");
|
|
671
|
+
for (const problem of configured.problems) {
|
|
672
|
+
console.error(` - ${problem}`);
|
|
673
|
+
}
|
|
674
|
+
process.exit(2);
|
|
675
|
+
}
|
|
676
|
+
var started = await startCacheServer(configured.config);
|
|
677
|
+
if (!started.ok) {
|
|
678
|
+
console.log(JSON.stringify({ kind: "failed", ...started }));
|
|
679
|
+
console.error(`nx-cache-server ${started.stage} failed in ${started.directory}: ${started.reason}`);
|
|
680
|
+
process.exit(1);
|
|
681
|
+
}
|
|
682
|
+
var running = started.server;
|
|
683
|
+
console.log(JSON.stringify({
|
|
684
|
+
kind: "ready",
|
|
685
|
+
origin: running.origin,
|
|
686
|
+
port: running.port,
|
|
687
|
+
reclaimedStagingFiles: running.reclaimedStagingFiles
|
|
688
|
+
}));
|
|
689
|
+
var stopping = false;
|
|
690
|
+
function stop(fields, exitCode = 0) {
|
|
691
|
+
if (stopping)
|
|
692
|
+
return;
|
|
693
|
+
stopping = true;
|
|
694
|
+
logLine(`ev=stopping ${fields}`);
|
|
695
|
+
running.stop().then(() => process.exit(exitCode), (failure) => {
|
|
696
|
+
logLine(`ev=stop.failed reason=${logValue(describe(failure))}`);
|
|
697
|
+
process.exit(1);
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
for (const signal of ["SIGINT", "SIGTERM"]) {
|
|
701
|
+
process.on(signal, () => stop(`signal=${signal}`));
|
|
702
|
+
}
|
|
703
|
+
if (exitOnStdinEof) {
|
|
704
|
+
process.stdin.once("end", () => stop("reason=supervisor-eof"));
|
|
705
|
+
process.stdin.once("error", (failure) => {
|
|
706
|
+
logLine(`ev=supervisor.error reason=${logValue(describe(failure))}`);
|
|
707
|
+
stop("reason=supervisor-error", 1);
|
|
708
|
+
});
|
|
709
|
+
process.stdin.resume();
|
|
710
|
+
}
|