@celilo/e2e 0.19.0 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/registry-server/src/bootstrap.ts +5 -0
- package/registry-server/src/server.test.ts +183 -1
- package/registry-server/src/server.ts +75 -0
- package/registry-server/src/storage.ts +76 -2
- package/registry-server/src/sweep.test.ts +326 -0
- package/registry-server/src/sweep.ts +289 -0
- package/scripts/consumer-paths-changed.ts +3 -0
- package/src/address-plan.test.ts +21 -1
- package/src/container-manager.ts +15 -7
- package/src/docker-compose-generator.ts +66 -22
- package/src/shared-infra.ts +6 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@celilo/e2e",
|
|
3
|
-
"version": "0.19.
|
|
3
|
+
"version": "0.19.1",
|
|
4
4
|
"description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.ts",
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"celilo-e2e-infra": "./bin/e2e-infra"
|
|
19
19
|
},
|
|
20
20
|
"scripts": {
|
|
21
|
-
"test": "bun test --timeout 30000 tests/",
|
|
21
|
+
"test": "bun test --timeout 30000 ./tests/ ./src/ ./npm-registry-server/ ./simulators/",
|
|
22
22
|
"test:completion": "bun test tests/completion",
|
|
23
23
|
"test:integration": "bun test tests-integration/"
|
|
24
24
|
},
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
"README.md"
|
|
38
38
|
],
|
|
39
39
|
"dependencies": {
|
|
40
|
-
"@celilo/capabilities": "^3.
|
|
40
|
+
"@celilo/capabilities": "^3.3.0",
|
|
41
41
|
"@celilo/cli-display": "^0.2.0",
|
|
42
42
|
"@celilo/event-bus": "^0.6.0",
|
|
43
43
|
"@celilo/terraform-fake": "^0.3.1",
|
|
@@ -33,6 +33,8 @@ export interface BootstrapEntry {
|
|
|
33
33
|
* apps/celilo/designs/REGISTRY_BROWSE_UI.md (Phase 2 step 0).
|
|
34
34
|
*/
|
|
35
35
|
description?: string;
|
|
36
|
+
/** Manifest's `icon` field, when present. Served like `description`. */
|
|
37
|
+
icon?: string;
|
|
36
38
|
}
|
|
37
39
|
|
|
38
40
|
/**
|
|
@@ -60,11 +62,14 @@ export function scanBootstrapDir(bootstrapDir: string): Map<string, BootstrapEnt
|
|
|
60
62
|
const name = idMatch?.[1]?.trim() ?? entry.name;
|
|
61
63
|
const version = versionMatch?.[1]?.trim() ?? '0.0.0';
|
|
62
64
|
const description = descriptionMatch?.[1]?.trim().replace(/^['"]|['"]$/g, '');
|
|
65
|
+
const iconMatch = yaml.match(/^icon:\s*(.+?)\s*$/m);
|
|
66
|
+
const icon = iconMatch?.[1]?.trim().replace(/^['"]|['"]$/g, '');
|
|
63
67
|
entries.set(name, {
|
|
64
68
|
name,
|
|
65
69
|
version,
|
|
66
70
|
sourceDir: join(bootstrapDir, entry.name),
|
|
67
71
|
description,
|
|
72
|
+
icon,
|
|
68
73
|
});
|
|
69
74
|
} catch {
|
|
70
75
|
// skip malformed manifests — matches e2e server behavior
|
|
@@ -217,6 +217,64 @@ describe('GET /api/v1/modules (search)', () => {
|
|
|
217
217
|
expect(found?.description).toBe('');
|
|
218
218
|
});
|
|
219
219
|
|
|
220
|
+
test('icon survives publish, the sparse index and a search read', async () => {
|
|
221
|
+
// openspec/changes/module-icons D4 — the icon rides the same path
|
|
222
|
+
// `description` does: publish metadata → index entry → browse endpoints.
|
|
223
|
+
expect((await publish('homebridge', '1.0.0+1', 'valid-token', 'Bridge', '\u22c8')).status).toBe(
|
|
224
|
+
200,
|
|
225
|
+
);
|
|
226
|
+
|
|
227
|
+
const index = await fetch(`${baseUrl}/index/ho/me/homebridge`);
|
|
228
|
+
expect(index.status).toBe(200);
|
|
229
|
+
const line = JSON.parse((await index.text()).trim()) as { icon?: string };
|
|
230
|
+
expect(line.icon).toBe('\u22c8');
|
|
231
|
+
|
|
232
|
+
const res = await fetch(`${baseUrl}/api/v1/modules`);
|
|
233
|
+
const body = (await res.json()) as { modules: Array<{ name: string; icon?: string }> };
|
|
234
|
+
expect(body.modules.find((m) => m.name === 'homebridge')?.icon).toBe('\u22c8');
|
|
235
|
+
|
|
236
|
+
const detail = await fetch(`${baseUrl}/api/v1/modules/homebridge`);
|
|
237
|
+
expect(((await detail.json()) as { icon?: string }).icon).toBe('\u22c8');
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
test('an entry published without an icon reads back as undefined, not a throw', async () => {
|
|
241
|
+
// The old-entry case D4 depends on: index lines written before the field
|
|
242
|
+
// existed carry no `icon`, and a consumer falls back rather than failing.
|
|
243
|
+
//
|
|
244
|
+
// This test passes with or without the icon feature, so it is NOT a gate on
|
|
245
|
+
// it — verified by running it against the pre-#1147 tree. It is here to
|
|
246
|
+
// catch a future change that makes a missing `icon` throw or coerce to '',
|
|
247
|
+
// which is what D4's fallback chain would break on. Do not count it as
|
|
248
|
+
// coverage of the capture path; that is the test above.
|
|
249
|
+
expect((await publish('homebridge', '1.0.0+1', 'valid-token')).status).toBe(200);
|
|
250
|
+
|
|
251
|
+
const res = await fetch(`${baseUrl}/api/v1/modules`);
|
|
252
|
+
const body = (await res.json()) as { modules: Array<{ name: string; icon?: string }> };
|
|
253
|
+
const found = body.modules.find((m) => m.name === 'homebridge');
|
|
254
|
+
expect(found).toBeTruthy();
|
|
255
|
+
expect(found?.icon).toBeUndefined();
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
test('a multi-character icon is dropped while a valid one beside it is kept', async () => {
|
|
259
|
+
// Publish metadata is arbitrary client JSON. The CLI refines the value at
|
|
260
|
+
// manifest-parse time; the server keeps the index from carrying a string
|
|
261
|
+
// that would blow out a fixed-width slot on the browse page.
|
|
262
|
+
//
|
|
263
|
+
// Both halves are asserted deliberately. Checking only that the bad value
|
|
264
|
+
// is absent would pass just as well if the capture path were deleted
|
|
265
|
+
// outright — the guard and the capture would cancel out and the test would
|
|
266
|
+
// stay green having lost the feature. The kept glyph is what stops that.
|
|
267
|
+
expect(
|
|
268
|
+
(await publish('homebridge', '1.0.0+1', 'valid-token', 'Bridge', 'not-a-glyph')).status,
|
|
269
|
+
).toBe(200);
|
|
270
|
+
expect((await publish('caddy', '1.0.0+1', 'valid-token', 'Proxy', '\u25cd')).status).toBe(200);
|
|
271
|
+
|
|
272
|
+
const res = await fetch(`${baseUrl}/api/v1/modules`);
|
|
273
|
+
const body = (await res.json()) as { modules: Array<{ name: string; icon?: string }> };
|
|
274
|
+
expect(body.modules.find((m) => m.name === 'homebridge')?.icon).toBeUndefined();
|
|
275
|
+
expect(body.modules.find((m) => m.name === 'caddy')?.icon).toBe('\u25cd');
|
|
276
|
+
});
|
|
277
|
+
|
|
220
278
|
test('total_downloads reflects actual download count, sort=downloads orders by it', async () => {
|
|
221
279
|
// Publish two modules, hit one's download endpoint several times.
|
|
222
280
|
// The search response should show those counts and sort=downloads
|
|
@@ -490,9 +548,15 @@ async function publish(
|
|
|
490
548
|
vers: string,
|
|
491
549
|
token: string,
|
|
492
550
|
description?: string,
|
|
551
|
+
icon?: string,
|
|
493
552
|
): Promise<Response> {
|
|
494
553
|
const meta = Buffer.from(
|
|
495
|
-
JSON.stringify({
|
|
554
|
+
JSON.stringify({
|
|
555
|
+
name,
|
|
556
|
+
vers,
|
|
557
|
+
...(description ? { description } : {}),
|
|
558
|
+
...(icon ? { icon } : {}),
|
|
559
|
+
}),
|
|
496
560
|
'utf-8',
|
|
497
561
|
);
|
|
498
562
|
const file = Buffer.from('fake netapp bytes');
|
|
@@ -769,3 +833,121 @@ describe('module-owner authorization (ce-1ch)', () => {
|
|
|
769
833
|
expect(res.status).toBe(200);
|
|
770
834
|
});
|
|
771
835
|
});
|
|
836
|
+
|
|
837
|
+
// ── sweep (reclaiming disk from superseded build revisions) ──────────────────
|
|
838
|
+
|
|
839
|
+
describe('POST /api/v1/modules/sweep', () => {
|
|
840
|
+
function requestSweep(token: string, body: unknown = {}): Promise<Response> {
|
|
841
|
+
return fetch(`${baseUrl}/api/v1/modules/sweep`, {
|
|
842
|
+
method: 'POST',
|
|
843
|
+
headers: token ? { Authorization: token, 'Content-Type': 'application/json' } : {},
|
|
844
|
+
body: JSON.stringify(body),
|
|
845
|
+
});
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
test('no token is 401 — a sweep deletes, so it is admin-only', async () => {
|
|
849
|
+
const res = await requestSweep('');
|
|
850
|
+
expect(res.status).toBe(401);
|
|
851
|
+
});
|
|
852
|
+
|
|
853
|
+
test('a scoped (non-admin) token is 401', async () => {
|
|
854
|
+
const mint = await fetch(`${baseUrl}/api/v1/modules/tokens/mint`, {
|
|
855
|
+
method: 'POST',
|
|
856
|
+
headers: { Authorization: 'valid-token', 'Content-Type': 'application/json' },
|
|
857
|
+
body: JSON.stringify({ repo: 'celilo/homebridge', scope: 'homebridge' }),
|
|
858
|
+
});
|
|
859
|
+
const { token } = (await mint.json()) as { token: string };
|
|
860
|
+
|
|
861
|
+
const res = await requestSweep(token);
|
|
862
|
+
expect(res.status).toBe(401);
|
|
863
|
+
});
|
|
864
|
+
|
|
865
|
+
test('removes superseded revisions and leaves the download path working', async () => {
|
|
866
|
+
for (const rev of [1, 2, 3]) await publish('homebridge', `1.0.0+${rev}`, 'valid-token');
|
|
867
|
+
|
|
868
|
+
const res = await requestSweep('valid-token');
|
|
869
|
+
expect(res.status).toBe(200);
|
|
870
|
+
const body = (await res.json()) as { ok: boolean; removedCount: number };
|
|
871
|
+
expect(body.ok).toBe(true);
|
|
872
|
+
expect(body.removedCount).toBe(2);
|
|
873
|
+
|
|
874
|
+
// The surviving revision downloads; a swept one is gone from BOTH the
|
|
875
|
+
// index and the store, so nothing advertises a 404.
|
|
876
|
+
expect((await fetch(`${baseUrl}/api/v1/modules/homebridge/1.0.0+3/download`)).status).toBe(200);
|
|
877
|
+
expect((await fetch(`${baseUrl}/api/v1/modules/homebridge/1.0.0+1/download`)).status).toBe(404);
|
|
878
|
+
|
|
879
|
+
const index = await fetch(`${baseUrl}/index/ho/me/homebridge`);
|
|
880
|
+
const versions = (await index.text())
|
|
881
|
+
.split('\n')
|
|
882
|
+
.filter(Boolean)
|
|
883
|
+
.map((line) => (JSON.parse(line) as { vers: string }).vers);
|
|
884
|
+
expect(versions).toEqual(['1.0.0+3']);
|
|
885
|
+
});
|
|
886
|
+
|
|
887
|
+
test('dry_run reports the plan without deleting anything', async () => {
|
|
888
|
+
for (const rev of [1, 2]) await publish('homebridge', `1.0.0+${rev}`, 'valid-token');
|
|
889
|
+
|
|
890
|
+
const res = await requestSweep('valid-token', { dry_run: true });
|
|
891
|
+
const body = (await res.json()) as { removedCount: number; dryRun: boolean };
|
|
892
|
+
expect(body.removedCount).toBe(1);
|
|
893
|
+
expect(body.dryRun).toBe(true);
|
|
894
|
+
expect((await fetch(`${baseUrl}/api/v1/modules/homebridge/1.0.0+1/download`)).status).toBe(200);
|
|
895
|
+
});
|
|
896
|
+
|
|
897
|
+
test('rejects a keep_build_revisions that would delete a whole release', async () => {
|
|
898
|
+
const res = await requestSweep('valid-token', { keep_build_revisions: 0 });
|
|
899
|
+
expect(res.status).toBe(400);
|
|
900
|
+
});
|
|
901
|
+
|
|
902
|
+
test('honours a larger keep_build_revisions', async () => {
|
|
903
|
+
for (const rev of [1, 2, 3, 4]) await publish('homebridge', `1.0.0+${rev}`, 'valid-token');
|
|
904
|
+
const res = await requestSweep('valid-token', { keep_build_revisions: 2 });
|
|
905
|
+
const body = (await res.json()) as { removedCount: number };
|
|
906
|
+
expect(body.removedCount).toBe(2);
|
|
907
|
+
});
|
|
908
|
+
|
|
909
|
+
test('coexists with a module actually named "sweep"', async () => {
|
|
910
|
+
await publish('sweep', '1.0.0+1', 'valid-token');
|
|
911
|
+
const res = await requestSweep('valid-token');
|
|
912
|
+
expect(res.status).toBe(200);
|
|
913
|
+
// …and the module is still reachable by its own GET route.
|
|
914
|
+
expect((await fetch(`${baseUrl}/api/v1/modules/sweep`)).status).toBe(200);
|
|
915
|
+
});
|
|
916
|
+
});
|
|
917
|
+
|
|
918
|
+
describe('POST /api/v1/modules/sweep — a damaged store', () => {
|
|
919
|
+
function requestSweep(token: string, body: unknown = {}): Promise<Response> {
|
|
920
|
+
return fetch(`${baseUrl}/api/v1/modules/sweep`, {
|
|
921
|
+
method: 'POST',
|
|
922
|
+
headers: { Authorization: token, 'Content-Type': 'application/json' },
|
|
923
|
+
body: JSON.stringify(body),
|
|
924
|
+
});
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
test('reclaims a half-written publish, freeing the version to be published again', async () => {
|
|
928
|
+
// handlePublish stores the package and appends the index with no rollback
|
|
929
|
+
// between them, so an ENOSPC or EACCES in the gap leaves an orphan. The
|
|
930
|
+
// immutability check reads the filesystem, so that version then cannot be
|
|
931
|
+
// published at all — this is the only repair path in the product.
|
|
932
|
+
await publish('homebridge', '1.0.0+1', 'valid-token');
|
|
933
|
+
const rmIndex = await fetch(`${baseUrl}/api/v1/modules/homebridge/1.0.0+1/yank`, {
|
|
934
|
+
method: 'DELETE',
|
|
935
|
+
headers: { Authorization: 'valid-token' },
|
|
936
|
+
});
|
|
937
|
+
expect(rmIndex.status).toBe(200);
|
|
938
|
+
|
|
939
|
+
// Simulate the crash: index line gone, payload left behind.
|
|
940
|
+
rmSync(join(dataDir, 'index'), { recursive: true, force: true });
|
|
941
|
+
|
|
942
|
+
const blocked = await publish('homebridge', '1.0.0+1', 'valid-token');
|
|
943
|
+
expect(blocked.status).toBe(409);
|
|
944
|
+
|
|
945
|
+
const swept = await requestSweep('valid-token');
|
|
946
|
+
const body = (await swept.json()) as { orphanCount: number };
|
|
947
|
+
expect(body.orphanCount).toBe(1);
|
|
948
|
+
|
|
949
|
+
// The version is publishable again.
|
|
950
|
+
const retry = await publish('homebridge', '1.0.0+1', 'valid-token');
|
|
951
|
+
expect(retry.status).toBe(200);
|
|
952
|
+
});
|
|
953
|
+
});
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* PUT /api/v1/modules/new → publish (token auth)
|
|
11
11
|
* DELETE /api/v1/modules/{name}/{ver}/yank → yank
|
|
12
12
|
* PUT /api/v1/modules/{name}/{ver}/unyank → unyank
|
|
13
|
+
* POST /api/v1/modules/sweep → reclaim old revisions (admin)
|
|
13
14
|
*
|
|
14
15
|
* All routes can be prefixed with PATH_PREFIX (e.g. "/registry").
|
|
15
16
|
*
|
|
@@ -34,6 +35,7 @@ import { ModuleOwnerStore, fileModuleOwnerPersistence } from './module-owner-sto
|
|
|
34
35
|
import { type RateLimiter, clientIp, createRateLimiter } from './rate-limit';
|
|
35
36
|
import { ScopedTokenStore, fileScopedTokenPersistence } from './scoped-token-store';
|
|
36
37
|
import { type IndexEntry, RegistryStorage } from './storage';
|
|
38
|
+
import { DEFAULT_KEEP_BUILD_REVISIONS, sweep } from './sweep';
|
|
37
39
|
import { isValidName, isValidVersion, validateNameAndVersion } from './validation';
|
|
38
40
|
|
|
39
41
|
/** Max total publish body size. Real modules are a few MB; 100MB is generous. */
|
|
@@ -313,6 +315,12 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
|
|
|
313
315
|
// not provided we leave it undefined; the search endpoint falls
|
|
314
316
|
// back to bootstrap data or empty.
|
|
315
317
|
const description = typeof meta.description === 'string' ? meta.description : undefined;
|
|
318
|
+
// Icon likewise, but capped at one Unicode scalar. The CLI already refines
|
|
319
|
+
// it at manifest-parse time; this second check is here because the publish
|
|
320
|
+
// metadata is arbitrary client JSON and the value lands in a fixed-width
|
|
321
|
+
// slot on the browse page.
|
|
322
|
+
const icon =
|
|
323
|
+
typeof meta.icon === 'string' && [...meta.icon].length === 1 ? meta.icon : undefined;
|
|
316
324
|
|
|
317
325
|
const validation = validateNameAndVersion(name, vers);
|
|
318
326
|
if (!validation.ok) return err(validation.message);
|
|
@@ -339,12 +347,63 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
|
|
|
339
347
|
cksum: `sha256:${cksum}`,
|
|
340
348
|
yanked: false,
|
|
341
349
|
description,
|
|
350
|
+
icon,
|
|
342
351
|
});
|
|
343
352
|
|
|
344
353
|
console.log(`[registry] published ${name}@${vers} (${fileLen} bytes, sha256:${cksum})`);
|
|
345
354
|
return Response.json({ ok: true, name, vers });
|
|
346
355
|
}
|
|
347
356
|
|
|
357
|
+
/**
|
|
358
|
+
* Reclaim disk from superseded build revisions (admin-only).
|
|
359
|
+
*
|
|
360
|
+
* The counterpart yank never was: yanking flips a boolean and frees nothing,
|
|
361
|
+
* so before this endpoint existed every revision ever published was retained
|
|
362
|
+
* forever and the store grew without bound until the disk filled.
|
|
363
|
+
*
|
|
364
|
+
* Runs IN the process that owns the store, so it cannot race a concurrent
|
|
365
|
+
* publish's index append. `dry_run` reports the same plan without touching
|
|
366
|
+
* anything — the safe way to see what a policy would do on a live store.
|
|
367
|
+
*/
|
|
368
|
+
async function handleSweep(req: Request): Promise<Response> {
|
|
369
|
+
if (!(await authorizeAdminReq(req))) return unauthorized();
|
|
370
|
+
|
|
371
|
+
let body: { keep_build_revisions?: unknown; dry_run?: unknown } = {};
|
|
372
|
+
try {
|
|
373
|
+
const text = await req.text();
|
|
374
|
+
if (text.trim()) body = JSON.parse(text) as typeof body;
|
|
375
|
+
} catch {
|
|
376
|
+
return err('Invalid JSON body');
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
const requested = body.keep_build_revisions;
|
|
380
|
+
if (requested !== undefined && (!Number.isInteger(requested) || (requested as number) < 1)) {
|
|
381
|
+
return err('keep_build_revisions must be an integer >= 1');
|
|
382
|
+
}
|
|
383
|
+
const keepBuildRevisions = (requested as number | undefined) ?? DEFAULT_KEEP_BUILD_REVISIONS;
|
|
384
|
+
const dryRun = body.dry_run === true;
|
|
385
|
+
|
|
386
|
+
const report = sweep(storage, { keepBuildRevisions }, dryRun);
|
|
387
|
+
console.log(
|
|
388
|
+
`[registry] sweep${dryRun ? ' (dry run)' : ''} keep=${keepBuildRevisions}: ` +
|
|
389
|
+
`${report.removedCount} superseded + ${report.orphanCount} orphaned revision(s), ` +
|
|
390
|
+
`${Math.round(report.reclaimedBytes / 1024 / 1024)} MB`,
|
|
391
|
+
);
|
|
392
|
+
// Both of these mean the store is damaged, not merely untidy, and the sweep
|
|
393
|
+
// deliberately did not act on either. Say so where an operator will see it.
|
|
394
|
+
if (report.danglingCount > 0) {
|
|
395
|
+
console.warn(
|
|
396
|
+
`[registry] ${report.danglingCount} index line(s) have no package file — left in place; download 404s until republished`,
|
|
397
|
+
);
|
|
398
|
+
}
|
|
399
|
+
if (report.unreadable.length > 0) {
|
|
400
|
+
console.warn(
|
|
401
|
+
`[registry] skipped entirely (no package files at all — check the mount and DATA_DIR): ${report.unreadable.join(', ')}`,
|
|
402
|
+
);
|
|
403
|
+
}
|
|
404
|
+
return Response.json({ ok: true, ...report });
|
|
405
|
+
}
|
|
406
|
+
|
|
348
407
|
function setYanked(name: string, version: string, yanked: boolean): Response {
|
|
349
408
|
const entries = storage.readIndex(name);
|
|
350
409
|
const entry = entries.find((e) => e.vers === version);
|
|
@@ -505,10 +564,15 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
|
|
|
505
564
|
// description capture and haven't been republished).
|
|
506
565
|
const bootstrapEntry = bootstrap.get(name);
|
|
507
566
|
const description = latest?.description ?? bootstrapEntry?.description ?? '';
|
|
567
|
+
// Icon follows the same order, but has no empty-string tier: absent
|
|
568
|
+
// means "this module declared none", and the consumer resolves it
|
|
569
|
+
// from its own table or a placeholder (module-icons D5).
|
|
570
|
+
const icon = latest?.icon ?? bootstrapEntry?.icon;
|
|
508
571
|
return {
|
|
509
572
|
name,
|
|
510
573
|
max_version: latest?.vers ?? '0.0.0',
|
|
511
574
|
description,
|
|
575
|
+
icon,
|
|
512
576
|
total_downloads: storage.getDownloads(name),
|
|
513
577
|
};
|
|
514
578
|
});
|
|
@@ -551,6 +615,15 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
|
|
|
551
615
|
return handleRevokeToken(req);
|
|
552
616
|
}
|
|
553
617
|
|
|
618
|
+
// Reclaim disk from superseded build revisions (admin-only), grouped
|
|
619
|
+
// with the other admin endpoints. A module NAMED "sweep" is fine here:
|
|
620
|
+
// its metadata route is a GET, so the two never compete.
|
|
621
|
+
if (method === 'POST' && suffix === `${apiBase}/sweep`) {
|
|
622
|
+
const rl = rateLimitOrNull(req, srv);
|
|
623
|
+
if (rl) return rl;
|
|
624
|
+
return handleSweep(req);
|
|
625
|
+
}
|
|
626
|
+
|
|
554
627
|
// Module-owner table management (admin-only — ce-1ch). Matched before the
|
|
555
628
|
// generic `${apiBase}/{name}` handlers so a name of "owners" can't shadow
|
|
556
629
|
// them.
|
|
@@ -588,9 +661,11 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
|
|
|
588
661
|
const latest = entries.filter((v) => !v.yanked).at(-1) ?? entries.at(-1);
|
|
589
662
|
const bootstrapEntry = resolveBootstrap().get(name);
|
|
590
663
|
const description = latest?.description ?? bootstrapEntry?.description ?? '';
|
|
664
|
+
const icon = latest?.icon ?? bootstrapEntry?.icon;
|
|
591
665
|
return Response.json({
|
|
592
666
|
name,
|
|
593
667
|
description,
|
|
668
|
+
icon,
|
|
594
669
|
total_downloads: storage.getDownloads(name),
|
|
595
670
|
versions: entries.map((v) => ({
|
|
596
671
|
num: v.vers,
|
|
@@ -8,7 +8,16 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { createHash } from 'node:crypto';
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
existsSync,
|
|
13
|
+
mkdirSync,
|
|
14
|
+
readFileSync,
|
|
15
|
+
readdirSync,
|
|
16
|
+
renameSync,
|
|
17
|
+
rmSync,
|
|
18
|
+
statSync,
|
|
19
|
+
writeFileSync,
|
|
20
|
+
} from 'node:fs';
|
|
12
21
|
import { dirname, join } from 'node:path';
|
|
13
22
|
|
|
14
23
|
export interface IndexEntry {
|
|
@@ -25,6 +34,14 @@ export interface IndexEntry {
|
|
|
25
34
|
* step 0). Old index files without this field still parse cleanly.
|
|
26
35
|
*/
|
|
27
36
|
description?: string;
|
|
37
|
+
/**
|
|
38
|
+
* One monochrome glyph from the module's `manifest.yml#icon`, captured at
|
|
39
|
+
* publish time. Optional for the same reason `description` is: entries
|
|
40
|
+
* written before the field existed have none, and a consumer reading one
|
|
41
|
+
* without it falls back to its own table rather than failing
|
|
42
|
+
* (openspec/changes/module-icons, D4).
|
|
43
|
+
*/
|
|
44
|
+
icon?: string;
|
|
28
45
|
}
|
|
29
46
|
|
|
30
47
|
export interface ModuleVersion {
|
|
@@ -79,10 +96,28 @@ export class RegistryStorage {
|
|
|
79
96
|
writeFileSync(path, `${JSON.stringify(entry)}\n`, { flag: 'a' });
|
|
80
97
|
}
|
|
81
98
|
|
|
99
|
+
/**
|
|
100
|
+
* Replace a module's whole index file.
|
|
101
|
+
*
|
|
102
|
+
* Writes a sibling temp file and renames it over the original, which is
|
|
103
|
+
* atomic within a directory. A plain truncating write has two failure modes
|
|
104
|
+
* this avoids, and the sweep meets both: a crash mid-write leaves a half-
|
|
105
|
+
* index, and an ENOSPC leaves an EMPTY one — losing every version of the
|
|
106
|
+
* module. The sweep exists precisely because the disk filled up, so "the
|
|
107
|
+
* write fails for want of space" is its expected environment, not a corner.
|
|
108
|
+
* On failure the original file is untouched.
|
|
109
|
+
*/
|
|
82
110
|
updateIndex(name: string, entries: IndexEntry[]): void {
|
|
83
111
|
const path = this.indexPath(name);
|
|
84
112
|
mkdirSync(dirname(path), { recursive: true });
|
|
85
|
-
|
|
113
|
+
const tmp = `${path}.tmp`;
|
|
114
|
+
try {
|
|
115
|
+
writeFileSync(tmp, `${entries.map((e) => JSON.stringify(e)).join('\n')}\n`);
|
|
116
|
+
renameSync(tmp, path);
|
|
117
|
+
} catch (err) {
|
|
118
|
+
rmSync(tmp, { force: true, recursive: true });
|
|
119
|
+
throw err;
|
|
120
|
+
}
|
|
86
121
|
}
|
|
87
122
|
|
|
88
123
|
storePackage(name: string, version: string, data: Buffer): string {
|
|
@@ -96,6 +131,45 @@ export class RegistryStorage {
|
|
|
96
131
|
return existsSync(this.packagePath(name, version));
|
|
97
132
|
}
|
|
98
133
|
|
|
134
|
+
/** Bytes on disk for one version's .netapp, or 0 when it is already gone. */
|
|
135
|
+
packageSize(name: string, version: string): number {
|
|
136
|
+
const path = this.packagePath(name, version);
|
|
137
|
+
if (!existsSync(path)) return 0;
|
|
138
|
+
return statSync(path).size;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Delete one version's payload directory (`modules/{name}/{version}/`).
|
|
143
|
+
* Idempotent: removing a version that is already gone is a no-op, so an
|
|
144
|
+
* interrupted sweep is safe to re-run.
|
|
145
|
+
*/
|
|
146
|
+
removePackage(name: string, version: string): void {
|
|
147
|
+
rmSync(dirname(this.packagePath(name, version)), { recursive: true, force: true });
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Every version with a payload directory on disk, whether or not the index
|
|
152
|
+
* still lists it. The sweep needs the on-DISK set, not the indexed one:
|
|
153
|
+
* removing an index line is what makes a version unreachable, and the file
|
|
154
|
+
* it leaves behind is exactly what has to be reclaimed afterwards.
|
|
155
|
+
*/
|
|
156
|
+
storedVersions(name: string): string[] {
|
|
157
|
+
const dir = join(this.dataDir, 'modules', name);
|
|
158
|
+
if (!existsSync(dir)) return [];
|
|
159
|
+
return readdirSync(dir, { withFileTypes: true })
|
|
160
|
+
.filter((d) => d.isDirectory())
|
|
161
|
+
.map((d) => d.name);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** Module names with a payload directory, including any absent from the index. */
|
|
165
|
+
storedNames(): string[] {
|
|
166
|
+
const dir = join(this.dataDir, 'modules');
|
|
167
|
+
if (!existsSync(dir)) return [];
|
|
168
|
+
return readdirSync(dir, { withFileTypes: true })
|
|
169
|
+
.filter((d) => d.isDirectory())
|
|
170
|
+
.map((d) => d.name);
|
|
171
|
+
}
|
|
172
|
+
|
|
99
173
|
readPackage(name: string, version: string): Buffer | null {
|
|
100
174
|
const path = this.packagePath(name, version);
|
|
101
175
|
return existsSync(path) ? readFileSync(path) : null;
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
|
2
|
+
import { existsSync, mkdirSync, mkdtempSync, rmSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { RegistryStorage } from './storage';
|
|
6
|
+
import { DEFAULT_KEEP_BUILD_REVISIONS, parseRevision, planModuleSweep, sweep } from './sweep';
|
|
7
|
+
|
|
8
|
+
let dataDir: string;
|
|
9
|
+
let storage: RegistryStorage;
|
|
10
|
+
|
|
11
|
+
beforeEach(() => {
|
|
12
|
+
dataDir = mkdtempSync(join(tmpdir(), 'celilo-registry-sweep-'));
|
|
13
|
+
storage = new RegistryStorage(dataDir);
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
afterEach(() => {
|
|
17
|
+
rmSync(dataDir, { recursive: true, force: true });
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
/** Publish a version through the same two calls the real publish path uses. */
|
|
21
|
+
function publish(name: string, vers: string, sizeBytes = 32): void {
|
|
22
|
+
storage.storePackage(name, vers, Buffer.alloc(sizeBytes, 7));
|
|
23
|
+
storage.appendIndex({ name, vers, deps: [], cksum: `sha256:${vers}`, yanked: false });
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* THE invariant. A sweep that trims payloads without trimming index lines
|
|
28
|
+
* leaves the index advertising a version whose file is gone — search lists it
|
|
29
|
+
* and the download 404s. The reverse leaves unreachable bytes on the disk the
|
|
30
|
+
* sweep exists to reclaim. Both directions have to hold.
|
|
31
|
+
*/
|
|
32
|
+
function expectIndexAndPayloadsAgree(name: string): void {
|
|
33
|
+
const indexed = storage.readIndex(name).map((e) => e.vers);
|
|
34
|
+
const stored = storage.storedVersions(name).sort();
|
|
35
|
+
for (const vers of indexed) {
|
|
36
|
+
expect(storage.readPackage(name, vers)).not.toBeNull();
|
|
37
|
+
}
|
|
38
|
+
expect(stored).toEqual([...indexed].sort());
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
describe('parseRevision', () => {
|
|
42
|
+
test('splits a release from its build revision', () => {
|
|
43
|
+
expect(parseRevision('1.0.3+7')).toEqual({ release: '1.0.3', revision: 7 });
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test('keeps a Debian epoch with the release', () => {
|
|
47
|
+
expect(parseRevision('2:1.0.0+1')).toEqual({ release: '2:1.0.0', revision: 1 });
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test('returns null for a version with no build revision', () => {
|
|
51
|
+
expect(parseRevision('1.0.0')).toBeNull();
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
describe('planModuleSweep', () => {
|
|
56
|
+
const entry = (vers: string, yanked = false) => ({
|
|
57
|
+
name: 'm',
|
|
58
|
+
vers,
|
|
59
|
+
deps: [],
|
|
60
|
+
cksum: 'sha256:x',
|
|
61
|
+
yanked,
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Plan against a store where every indexed version HAS its payload — the
|
|
66
|
+
* ordinary case. Passing an empty stored set instead would trip the
|
|
67
|
+
* unreadable guard, which is a different scenario with its own tests below.
|
|
68
|
+
*/
|
|
69
|
+
const planWithPayloads = (entries: ReturnType<typeof entry>[], keepBuildRevisions: number) =>
|
|
70
|
+
planModuleSweep(
|
|
71
|
+
'm',
|
|
72
|
+
entries,
|
|
73
|
+
entries.map((e) => e.vers),
|
|
74
|
+
{ keepBuildRevisions },
|
|
75
|
+
);
|
|
76
|
+
|
|
77
|
+
test('keeps the newest build revision of every release', () => {
|
|
78
|
+
const entries = [
|
|
79
|
+
entry('1.0.0+1'),
|
|
80
|
+
entry('1.0.0+2'),
|
|
81
|
+
entry('1.0.2+8'),
|
|
82
|
+
entry('1.0.2+9'),
|
|
83
|
+
entry('1.0.3+5'),
|
|
84
|
+
entry('1.0.3+6'),
|
|
85
|
+
entry('1.0.3+7'),
|
|
86
|
+
];
|
|
87
|
+
const plan = planWithPayloads(entries, 1);
|
|
88
|
+
expect(plan.keep.map((e) => e.vers)).toEqual(['1.0.0+2', '1.0.2+9', '1.0.3+7']);
|
|
89
|
+
expect(plan.remove).toEqual(['1.0.0+1', '1.0.2+8', '1.0.3+5', '1.0.3+6']);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
test('never removes the only revision of a release', () => {
|
|
93
|
+
const entries = [entry('0.9.0+2'), entry('1.0.0+1')];
|
|
94
|
+
const plan = planWithPayloads(entries, 1);
|
|
95
|
+
expect(plan.remove).toEqual([]);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
test('a module can never be swept out of existence', () => {
|
|
99
|
+
const entries = [entry('1.0.0+1')];
|
|
100
|
+
const plan = planWithPayloads(entries, 1);
|
|
101
|
+
expect(plan.keep).toHaveLength(1);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test('keeps more revisions per release when asked', () => {
|
|
105
|
+
const entries = [entry('1.0.3+5'), entry('1.0.3+6'), entry('1.0.3+7')];
|
|
106
|
+
const plan = planWithPayloads(entries, 2);
|
|
107
|
+
expect(plan.keep.map((e) => e.vers)).toEqual(['1.0.3+6', '1.0.3+7']);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
test('keepBuildRevisions below 1 is clamped — no policy deletes a whole release', () => {
|
|
111
|
+
// Needs MORE THAN ONE release to be a real check. With a single release
|
|
112
|
+
// the latest-line retention below masks a missing clamp; with two, an
|
|
113
|
+
// unclamped 0 wipes out every release except the last.
|
|
114
|
+
const entries = [entry('1.0.0+1'), entry('1.0.0+2'), entry('1.0.3+5'), entry('1.0.3+6')];
|
|
115
|
+
const plan = planWithPayloads(entries, 0);
|
|
116
|
+
expect(plan.keep.map((e) => e.vers)).toEqual(['1.0.0+2', '1.0.3+6']);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
test('retains what latestVersion resolves to even when the index is out of order', () => {
|
|
120
|
+
// Last line wins for RegistryClient.latestVersion regardless of ordering,
|
|
121
|
+
// so a disordered index must not lose the version clients actually fetch.
|
|
122
|
+
const entries = [entry('1.0.3+7'), entry('1.0.3+6'), entry('1.0.3+5')];
|
|
123
|
+
const plan = planWithPayloads(entries, 1);
|
|
124
|
+
expect(plan.keep.map((e) => e.vers)).toContain('1.0.3+5');
|
|
125
|
+
expect(plan.keep.map((e) => e.vers)).toContain('1.0.3+7');
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
test('retains the last NON-yanked line, which is what clients resolve to', () => {
|
|
129
|
+
const entries = [entry('1.0.3+5'), entry('1.0.3+6'), entry('1.0.3+7', true)];
|
|
130
|
+
const plan = planWithPayloads(entries, 1);
|
|
131
|
+
expect(plan.keep.map((e) => e.vers)).toContain('1.0.3+6');
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test('keeps a version it cannot parse rather than deleting it', () => {
|
|
135
|
+
const entries = [entry('nonsense'), entry('1.0.0+1'), entry('1.0.0+2')];
|
|
136
|
+
const plan = planWithPayloads(entries, 1);
|
|
137
|
+
expect(plan.remove).toEqual(['1.0.0+1']);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
test('reports a payload with no index line as an orphan', () => {
|
|
141
|
+
const plan = planModuleSweep('m', [entry('1.0.0+2')], ['1.0.0+1', '1.0.0+2'], {
|
|
142
|
+
keepBuildRevisions: 1,
|
|
143
|
+
});
|
|
144
|
+
expect(plan.orphans).toEqual(['1.0.0+1']);
|
|
145
|
+
});
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
describe('sweep over a fixture store', () => {
|
|
149
|
+
test('index and payloads agree afterwards, and every release survives', () => {
|
|
150
|
+
for (const rev of [1, 2, 3, 4, 5]) publish('celilo-registry', `1.0.3+${rev}`);
|
|
151
|
+
for (const rev of [1, 2]) publish('celilo-registry', `1.0.2+${rev}`);
|
|
152
|
+
publish('forgejo', '2.1.0+1');
|
|
153
|
+
|
|
154
|
+
const report = sweep(storage, { keepBuildRevisions: DEFAULT_KEEP_BUILD_REVISIONS });
|
|
155
|
+
|
|
156
|
+
expect(report.removedCount).toBe(5);
|
|
157
|
+
expectIndexAndPayloadsAgree('celilo-registry');
|
|
158
|
+
expectIndexAndPayloadsAgree('forgejo');
|
|
159
|
+
// Original index order is preserved: the fixture published 1.0.3 before
|
|
160
|
+
// 1.0.2, and the sweep filters lines rather than re-sorting them.
|
|
161
|
+
expect(storage.readIndex('celilo-registry').map((e) => e.vers)).toEqual(['1.0.3+5', '1.0.2+2']);
|
|
162
|
+
expect(storage.readIndex('forgejo').map((e) => e.vers)).toEqual(['2.1.0+1']);
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
test('reclaims the bytes it says it reclaims', () => {
|
|
166
|
+
publish('m', '1.0.0+1', 4096);
|
|
167
|
+
publish('m', '1.0.0+2', 4096);
|
|
168
|
+
const report = sweep(storage, { keepBuildRevisions: 1 });
|
|
169
|
+
expect(report.reclaimedBytes).toBe(4096);
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
test('a dry run changes nothing', () => {
|
|
173
|
+
for (const rev of [1, 2, 3]) publish('m', `1.0.0+${rev}`);
|
|
174
|
+
const report = sweep(storage, { keepBuildRevisions: 1 }, true);
|
|
175
|
+
|
|
176
|
+
expect(report.removedCount).toBe(2);
|
|
177
|
+
expect(storage.readIndex('m')).toHaveLength(3);
|
|
178
|
+
expect(storage.storedVersions('m').sort()).toEqual(['1.0.0+1', '1.0.0+2', '1.0.0+3']);
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
test('is idempotent — a second sweep finds nothing to do', () => {
|
|
182
|
+
for (const rev of [1, 2, 3]) publish('m', `1.0.0+${rev}`);
|
|
183
|
+
sweep(storage, { keepBuildRevisions: 1 });
|
|
184
|
+
const second = sweep(storage, { keepBuildRevisions: 1 });
|
|
185
|
+
|
|
186
|
+
expect(second.removedCount).toBe(0);
|
|
187
|
+
expect(second.orphanCount).toBe(0);
|
|
188
|
+
expect(second.reclaimedBytes).toBe(0);
|
|
189
|
+
expectIndexAndPayloadsAgree('m');
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
test('a sweep interrupted after the index rewrite self-heals on the next run', () => {
|
|
193
|
+
for (const rev of [1, 2, 3]) publish('m', `1.0.0+${rev}`);
|
|
194
|
+
// The crash window: index trimmed, payloads not yet unlinked. The version
|
|
195
|
+
// is unlisted but still on disk — wasted bytes, not a broken registry.
|
|
196
|
+
storage.updateIndex('m', storage.readIndex('m').slice(-1));
|
|
197
|
+
expect(storage.storedVersions('m')).toHaveLength(3);
|
|
198
|
+
|
|
199
|
+
const report = sweep(storage, { keepBuildRevisions: 1 });
|
|
200
|
+
|
|
201
|
+
expect(report.orphanCount).toBe(2);
|
|
202
|
+
expectIndexAndPayloadsAgree('m');
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
test('a version left unlisted stays downloadable until its payload goes', () => {
|
|
206
|
+
// Why index-first is the safe order: the download route reads the payload
|
|
207
|
+
// directly and never consults the index, so the window an interrupted
|
|
208
|
+
// sweep opens serves stale-but-valid bytes rather than 404ing.
|
|
209
|
+
publish('m', '1.0.0+1');
|
|
210
|
+
storage.updateIndex('m', []);
|
|
211
|
+
expect(storage.readPackage('m', '1.0.0+1')).not.toBeNull();
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
test('reclaims a half-written publish, which is what makes the version publishable again', () => {
|
|
215
|
+
// Both states below exist on the production registry, created 2026-08-22 by
|
|
216
|
+
// real failures: an ENOSPC that killed a run after storePackage, and an
|
|
217
|
+
// EACCES on appendIndex. handlePublish has no rollback between the two.
|
|
218
|
+
//
|
|
219
|
+
// The consequence is worse than wasted bytes: `packageExists` reads the
|
|
220
|
+
// FILESYSTEM, so the orphan makes its own version permanently
|
|
221
|
+
// unpublishable — "versions are immutable". Removing the orphan is the
|
|
222
|
+
// repair, and nothing else in the product performs it.
|
|
223
|
+
storage.storePackage('celilo-mgmt', '0.6.2+2', Buffer.alloc(2048, 1));
|
|
224
|
+
publish('celilo-mgmt', '0.6.1+1');
|
|
225
|
+
expect(storage.packageExists('celilo-mgmt', '0.6.2+2')).toBe(true);
|
|
226
|
+
|
|
227
|
+
const report = sweep(storage, { keepBuildRevisions: 1 });
|
|
228
|
+
|
|
229
|
+
expect(report.orphanCount).toBe(1);
|
|
230
|
+
expect(storage.packageExists('celilo-mgmt', '0.6.2+2')).toBe(false);
|
|
231
|
+
expectIndexAndPayloadsAgree('celilo-mgmt');
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
test('REPORTS an index line with no payload and does not remove it', () => {
|
|
235
|
+
// The dangerous direction. Removing the line would actually repair the
|
|
236
|
+
// module — a dangling entry already 404s — but a missing payload is
|
|
237
|
+
// indistinguishable from an unreadable store, so it is reported only.
|
|
238
|
+
publish('m', '1.0.0+1');
|
|
239
|
+
publish('m', '1.0.0+2');
|
|
240
|
+
storage.removePackage('m', '1.0.0+1');
|
|
241
|
+
|
|
242
|
+
const report = sweep(storage, { keepBuildRevisions: 1 });
|
|
243
|
+
|
|
244
|
+
expect(report.danglingCount).toBe(1);
|
|
245
|
+
expect(report.modules[0]?.dangling).toEqual(['1.0.0+1']);
|
|
246
|
+
// Still listed: the sweep reported it rather than acting on it.
|
|
247
|
+
expect(storage.readIndex('m').map((e) => e.vers)).toContain('1.0.0+1');
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
test('touches nothing when a module has index lines and no payloads at all', () => {
|
|
251
|
+
// An unmounted volume or a wrong DATA_DIR looks exactly like this. Acting
|
|
252
|
+
// on it would trim the index of a module whose files are merely
|
|
253
|
+
// unreachable.
|
|
254
|
+
for (const rev of [1, 2, 3]) publish('m', `1.0.0+${rev}`);
|
|
255
|
+
for (const rev of [1, 2, 3]) storage.removePackage('m', `1.0.0+${rev}`);
|
|
256
|
+
|
|
257
|
+
const report = sweep(storage, { keepBuildRevisions: 1 });
|
|
258
|
+
|
|
259
|
+
expect(report.unreadable).toEqual(['m']);
|
|
260
|
+
expect(report.removedCount).toBe(0);
|
|
261
|
+
expect(storage.readIndex('m')).toHaveLength(3);
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
test('a dangling entry does not stop the rest of the sweep', () => {
|
|
265
|
+
publish('m', '1.0.0+1');
|
|
266
|
+
publish('m', '1.0.0+2');
|
|
267
|
+
publish('m', '1.0.0+3');
|
|
268
|
+
storage.removePackage('m', '1.0.0+1');
|
|
269
|
+
|
|
270
|
+
const report = sweep(storage, { keepBuildRevisions: 1 });
|
|
271
|
+
|
|
272
|
+
expect(report.danglingCount).toBe(1);
|
|
273
|
+
// 1.0.0+2 is superseded and still had its payload, so it is swept normally.
|
|
274
|
+
expect(report.removedCount).toBe(1);
|
|
275
|
+
expect(storage.readIndex('m').map((e) => e.vers)).toEqual(['1.0.0+1', '1.0.0+3']);
|
|
276
|
+
});
|
|
277
|
+
|
|
278
|
+
test('never sweeps the last SERVABLE version because a newer line is dangling', () => {
|
|
279
|
+
// The failure this guards: retention computed over every index line lets a
|
|
280
|
+
// dangling NEWEST entry satisfy both the per-release rule and the
|
|
281
|
+
// latest-line rule, so every servable older version reads as superseded.
|
|
282
|
+
// The module ends up listed, with one index line, and undownloadable.
|
|
283
|
+
publish('m', '1.0.0+1');
|
|
284
|
+
publish('m', '1.0.0+2');
|
|
285
|
+
publish('m', '1.0.0+3');
|
|
286
|
+
storage.removePackage('m', '1.0.0+3'); // the newest line is now dangling
|
|
287
|
+
|
|
288
|
+
sweep(storage, { keepBuildRevisions: 1 });
|
|
289
|
+
|
|
290
|
+
// Something is still downloadable, which is the whole point.
|
|
291
|
+
const servable = storage
|
|
292
|
+
.readIndex('m')
|
|
293
|
+
.map((e) => e.vers)
|
|
294
|
+
.filter((v) => storage.packageExists('m', v));
|
|
295
|
+
expect(servable.length).toBeGreaterThan(0);
|
|
296
|
+
expect(servable).toContain('1.0.0+2');
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
test('an empty store sweeps cleanly', () => {
|
|
300
|
+
const report = sweep(storage, { keepBuildRevisions: 1 });
|
|
301
|
+
expect(report.removedCount).toBe(0);
|
|
302
|
+
expect(report.modules).toEqual([]);
|
|
303
|
+
});
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
describe('updateIndex atomicity', () => {
|
|
307
|
+
test('leaves no temp file behind', () => {
|
|
308
|
+
publish('m', '1.0.0+1');
|
|
309
|
+
storage.updateIndex('m', storage.readIndex('m'));
|
|
310
|
+
expect(existsSync(`${storage.indexPath('m')}.tmp`)).toBe(false);
|
|
311
|
+
});
|
|
312
|
+
|
|
313
|
+
test('a write that cannot land leaves the original index intact', () => {
|
|
314
|
+
publish('m', '1.0.0+1');
|
|
315
|
+
publish('m', '1.0.0+2');
|
|
316
|
+
const before = storage.readIndex('m');
|
|
317
|
+
// Block the temp path so the new content cannot be written. A truncating
|
|
318
|
+
// in-place write ignores the obstruction and replaces the index anyway;
|
|
319
|
+
// the rename path fails with the original still on disk. This stands in
|
|
320
|
+
// for the failure that actually matters — ENOSPC on the full disk this
|
|
321
|
+
// sweep exists to relieve — which needs an fs seam to reproduce directly.
|
|
322
|
+
mkdirSync(`${storage.indexPath('m')}.tmp`, { recursive: true });
|
|
323
|
+
expect(() => storage.updateIndex('m', before.slice(0, 1))).toThrow();
|
|
324
|
+
expect(storage.readIndex('m')).toEqual(before);
|
|
325
|
+
});
|
|
326
|
+
});
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reclaiming disk from superseded build revisions.
|
|
3
|
+
*
|
|
4
|
+
* The registry had no delete path at all: `yank` sets a boolean in the index
|
|
5
|
+
* and frees nothing, so every revision ever published was retained forever.
|
|
6
|
+
* A `+N` build revision is auto-assigned on every publish, so the store grew
|
|
7
|
+
* with every release until the disk filled and the release pipeline stopped.
|
|
8
|
+
*
|
|
9
|
+
* ── What the policy keeps, and why it cannot break a fetch ──
|
|
10
|
+
*
|
|
11
|
+
* A version is `X.Y.Z+N`: a released semver plus a build revision. The bloat
|
|
12
|
+
* is the rebuilds, not the releases — a module embedding compiled binaries
|
|
13
|
+
* republished twenty times carries twenty ~73 MB payloads of one release.
|
|
14
|
+
*
|
|
15
|
+
* So the sweep keeps the newest `keepBuildRevisions` of EVERY release and
|
|
16
|
+
* drops the rest. No release ever disappears, which is what makes the safety
|
|
17
|
+
* argument checkable rather than a judgement call:
|
|
18
|
+
*
|
|
19
|
+
* - The only version a celilo client can ask for is the one
|
|
20
|
+
* `RegistryClient.latestVersion` picks — the last non-yanked line in the
|
|
21
|
+
* index. There is no `--version` flag, no lockfile, and `IndexEntry.deps`
|
|
22
|
+
* is always `[]`, so nothing pins a specific revision.
|
|
23
|
+
* - {@link planModuleSweep} retains that line explicitly, on top of the
|
|
24
|
+
* per-release rule, so the property holds even for an index whose lines
|
|
25
|
+
* are out of publish order.
|
|
26
|
+
* - Every release keeps a downloadable payload, so an operator rolling back
|
|
27
|
+
* to an older release still can.
|
|
28
|
+
*
|
|
29
|
+
* ── Ordering, and why it is index-first ──
|
|
30
|
+
*
|
|
31
|
+
* The download route reads the payload file directly and never consults the
|
|
32
|
+
* index. That asymmetry decides the order:
|
|
33
|
+
*
|
|
34
|
+
* - index line first, then payload — the window is "unlisted but still
|
|
35
|
+
* served". Nothing 404s. A crash here leaves an orphan payload, which
|
|
36
|
+
* wastes the disk we are reclaiming but is otherwise correct.
|
|
37
|
+
* - payload first, then index — the window is "listed but 404s", an
|
|
38
|
+
* actively wrong registry. A crash here leaves it that way.
|
|
39
|
+
*
|
|
40
|
+
* So: index first. The orphan a crash leaves behind is then reclaimed by the
|
|
41
|
+
* NEXT run, which is what makes the sweep idempotent and safe to interrupt.
|
|
42
|
+
* The orphan pass runs FIRST for a second reason: it is pure unlink and needs
|
|
43
|
+
* no free space, so it can still make progress on a disk with nothing left to
|
|
44
|
+
* write with — the state that produced this code.
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
import type { IndexEntry, RegistryStorage } from './storage';
|
|
48
|
+
|
|
49
|
+
export interface SweepPolicy {
|
|
50
|
+
/**
|
|
51
|
+
* Build revisions to keep per released semver. Must be >= 1 — keeping zero
|
|
52
|
+
* would delete a whole release, which this policy never does.
|
|
53
|
+
*/
|
|
54
|
+
keepBuildRevisions: number;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export const DEFAULT_KEEP_BUILD_REVISIONS = 1;
|
|
58
|
+
|
|
59
|
+
export interface ModuleSweepPlan {
|
|
60
|
+
name: string;
|
|
61
|
+
/** Index entries to retain, in their original order. */
|
|
62
|
+
keep: IndexEntry[];
|
|
63
|
+
/** Versions whose index line and payload both go. */
|
|
64
|
+
remove: string[];
|
|
65
|
+
/**
|
|
66
|
+
* Payloads on disk with no index line. Unreachable by search, and the
|
|
67
|
+
* publisher's immutability check still sees them — `packageExists` reads the
|
|
68
|
+
* FILESYSTEM, so an orphan makes its version unpublishable forever with no
|
|
69
|
+
* repair path anywhere else in the product. Safe to delete, and deleting it
|
|
70
|
+
* IS the repair.
|
|
71
|
+
*/
|
|
72
|
+
orphans: string[];
|
|
73
|
+
/**
|
|
74
|
+
* Index lines whose payload is missing. REPORTED, NEVER REMOVED.
|
|
75
|
+
*
|
|
76
|
+
* This is the direction the sweep must never create, and the reason it takes
|
|
77
|
+
* the index line before the payload. Removing the line would in fact repair
|
|
78
|
+
* the module — a dangling entry already 404s on download, and dropping it
|
|
79
|
+
* makes `latestVersion` resolve to a version that can actually be served.
|
|
80
|
+
* It is left alone anyway, because a missing payload and an unreadable store
|
|
81
|
+
* are indistinguishable from here: an unmounted volume or a wrong DATA_DIR
|
|
82
|
+
* makes every payload look absent, and an auto-remove would then delete the
|
|
83
|
+
* whole index. Reporting costs nothing and cannot destroy anything.
|
|
84
|
+
*/
|
|
85
|
+
dangling: string[];
|
|
86
|
+
/**
|
|
87
|
+
* The module's index lists versions and NOT ONE has a payload on disk. That
|
|
88
|
+
* is a broken or unmounted store rather than N independent losses, so the
|
|
89
|
+
* sweep does nothing at all to this module — see {@link ModuleSweepPlan.dangling}.
|
|
90
|
+
*/
|
|
91
|
+
unreadable: boolean;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export interface SweepReport {
|
|
95
|
+
modules: Array<{
|
|
96
|
+
name: string;
|
|
97
|
+
removed: string[];
|
|
98
|
+
orphans: string[];
|
|
99
|
+
dangling: string[];
|
|
100
|
+
bytes: number;
|
|
101
|
+
}>;
|
|
102
|
+
removedCount: number;
|
|
103
|
+
orphanCount: number;
|
|
104
|
+
/** Index lines with no payload. Reported for a human; never acted on. */
|
|
105
|
+
danglingCount: number;
|
|
106
|
+
/** Modules skipped entirely because their whole payload set is missing. */
|
|
107
|
+
unreadable: string[];
|
|
108
|
+
reclaimedBytes: number;
|
|
109
|
+
dryRun: boolean;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Split `X.Y.Z+N` (or `E:X.Y.Z+N`) into the release it belongs to and its
|
|
114
|
+
* build revision. Returns null for anything that does not parse — the caller
|
|
115
|
+
* treats that as "keep", because a version the sweep cannot reason about is
|
|
116
|
+
* not one it should delete.
|
|
117
|
+
*/
|
|
118
|
+
export function parseRevision(vers: string): { release: string; revision: number } | null {
|
|
119
|
+
const plus = vers.lastIndexOf('+');
|
|
120
|
+
if (plus <= 0) return null;
|
|
121
|
+
const release = vers.slice(0, plus);
|
|
122
|
+
const revision = Number(vers.slice(plus + 1));
|
|
123
|
+
if (!Number.isInteger(revision) || revision < 0) return null;
|
|
124
|
+
return { release, revision };
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Decide what one module keeps. Pure — no filesystem, no storage.
|
|
129
|
+
*
|
|
130
|
+
* `storedVersions` is what is on DISK, which is deliberately not the same set
|
|
131
|
+
* as `entries`: a previous interrupted run can leave a payload whose index
|
|
132
|
+
* line is already gone, and reclaiming it is the whole reason this takes both.
|
|
133
|
+
*/
|
|
134
|
+
export function planModuleSweep(
|
|
135
|
+
name: string,
|
|
136
|
+
entries: IndexEntry[],
|
|
137
|
+
storedVersions: string[],
|
|
138
|
+
policy: SweepPolicy,
|
|
139
|
+
): ModuleSweepPlan {
|
|
140
|
+
const keepBuildRevisions = Math.max(1, Math.trunc(policy.keepBuildRevisions));
|
|
141
|
+
|
|
142
|
+
const stored = new Set(storedVersions);
|
|
143
|
+
const dangling = entries.filter((e) => !stored.has(e.vers)).map((e) => e.vers);
|
|
144
|
+
|
|
145
|
+
// Every listed version missing its payload is one broken store, not N
|
|
146
|
+
// independent losses. Touch nothing.
|
|
147
|
+
if (entries.length > 0 && dangling.length === entries.length) {
|
|
148
|
+
return { name, keep: entries, remove: [], orphans: [], dangling, unreadable: true };
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Retention is computed over the SERVABLE entries only — the ones whose
|
|
153
|
+
* payload is actually there.
|
|
154
|
+
*
|
|
155
|
+
* Doing it over every index line is subtly wrong in a way that bites exactly
|
|
156
|
+
* when the store is already damaged. If a module's NEWEST line is dangling,
|
|
157
|
+
* that line satisfies both the per-release rule and the latest-line rule, so
|
|
158
|
+
* every older version — the servable ones — becomes "superseded" and is
|
|
159
|
+
* swept. The module is then listed, has one index line, and cannot be
|
|
160
|
+
* downloaded at all. The sweep would not have created the dangling entry,
|
|
161
|
+
* but it would have removed the last thing that still worked.
|
|
162
|
+
*/
|
|
163
|
+
const servable = entries.filter((e) => stored.has(e.vers));
|
|
164
|
+
|
|
165
|
+
const retained = new Set<string>();
|
|
166
|
+
|
|
167
|
+
// The newest `keepBuildRevisions` of each release.
|
|
168
|
+
const byRelease = new Map<string, Array<{ vers: string; revision: number }>>();
|
|
169
|
+
for (const entry of servable) {
|
|
170
|
+
const parsed = parseRevision(entry.vers);
|
|
171
|
+
if (!parsed) {
|
|
172
|
+
// Unparseable — never a publish this server accepted, so it is hand-
|
|
173
|
+
// written state. Keep it and move on.
|
|
174
|
+
retained.add(entry.vers);
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
const group = byRelease.get(parsed.release) ?? [];
|
|
178
|
+
group.push({ vers: entry.vers, revision: parsed.revision });
|
|
179
|
+
byRelease.set(parsed.release, group);
|
|
180
|
+
}
|
|
181
|
+
for (const group of byRelease.values()) {
|
|
182
|
+
group.sort((a, b) => b.revision - a.revision);
|
|
183
|
+
for (const { vers } of group.slice(0, keepBuildRevisions)) retained.add(vers);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// What every client actually resolves to, retained explicitly rather than
|
|
187
|
+
// inferred from the rule above. The rule already covers it for an index in
|
|
188
|
+
// publish order; stating it means a disordered or hand-edited index cannot
|
|
189
|
+
// turn into an unfetchable module.
|
|
190
|
+
const lastServable = servable.at(-1);
|
|
191
|
+
if (lastServable) retained.add(lastServable.vers);
|
|
192
|
+
const lastUnyanked = [...servable].reverse().find((e) => !e.yanked);
|
|
193
|
+
if (lastUnyanked) retained.add(lastUnyanked.vers);
|
|
194
|
+
|
|
195
|
+
// Only ever removes something it can see. A dangling line is left in the
|
|
196
|
+
// index untouched and reported instead.
|
|
197
|
+
const remove = servable.filter((e) => !retained.has(e.vers)).map((e) => e.vers);
|
|
198
|
+
const removing = new Set(remove);
|
|
199
|
+
const indexed = new Set(entries.map((e) => e.vers));
|
|
200
|
+
|
|
201
|
+
return {
|
|
202
|
+
name,
|
|
203
|
+
keep: entries.filter((e) => !removing.has(e.vers)),
|
|
204
|
+
remove,
|
|
205
|
+
orphans: storedVersions.filter((v) => !indexed.has(v)),
|
|
206
|
+
dangling,
|
|
207
|
+
unreadable: false,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Apply one module's plan. Orphans first (needs no free space), then the
|
|
213
|
+
* index rewrite, then the payloads the rewrite just unlisted.
|
|
214
|
+
*/
|
|
215
|
+
function applyModuleSweep(
|
|
216
|
+
storage: RegistryStorage,
|
|
217
|
+
plan: ModuleSweepPlan,
|
|
218
|
+
dryRun: boolean,
|
|
219
|
+
): number {
|
|
220
|
+
let bytes = 0;
|
|
221
|
+
|
|
222
|
+
for (const version of plan.orphans) {
|
|
223
|
+
bytes += storage.packageSize(plan.name, version);
|
|
224
|
+
if (!dryRun) storage.removePackage(plan.name, version);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
for (const version of plan.remove) {
|
|
228
|
+
bytes += storage.packageSize(plan.name, version);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
if (plan.remove.length > 0 && !dryRun) {
|
|
232
|
+
storage.updateIndex(plan.name, plan.keep);
|
|
233
|
+
for (const version of plan.remove) storage.removePackage(plan.name, version);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
return bytes;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** Sweep every module in the store. */
|
|
240
|
+
export function sweep(storage: RegistryStorage, policy: SweepPolicy, dryRun = false): SweepReport {
|
|
241
|
+
const modules: SweepReport['modules'] = [];
|
|
242
|
+
let removedCount = 0;
|
|
243
|
+
let orphanCount = 0;
|
|
244
|
+
let reclaimedBytes = 0;
|
|
245
|
+
|
|
246
|
+
const unreadable: string[] = [];
|
|
247
|
+
let danglingCount = 0;
|
|
248
|
+
|
|
249
|
+
for (const name of storage.storedNames()) {
|
|
250
|
+
const plan = planModuleSweep(
|
|
251
|
+
name,
|
|
252
|
+
storage.readIndex(name),
|
|
253
|
+
storage.storedVersions(name),
|
|
254
|
+
policy,
|
|
255
|
+
);
|
|
256
|
+
|
|
257
|
+
if (plan.unreadable) {
|
|
258
|
+
unreadable.push(name);
|
|
259
|
+
continue;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
danglingCount += plan.dangling.length;
|
|
263
|
+
if (plan.remove.length === 0 && plan.orphans.length === 0 && plan.dangling.length === 0) {
|
|
264
|
+
continue;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
const bytes = applyModuleSweep(storage, plan, dryRun);
|
|
268
|
+
modules.push({
|
|
269
|
+
name,
|
|
270
|
+
removed: plan.remove,
|
|
271
|
+
orphans: plan.orphans,
|
|
272
|
+
dangling: plan.dangling,
|
|
273
|
+
bytes,
|
|
274
|
+
});
|
|
275
|
+
removedCount += plan.remove.length;
|
|
276
|
+
orphanCount += plan.orphans.length;
|
|
277
|
+
reclaimedBytes += bytes;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
return {
|
|
281
|
+
modules,
|
|
282
|
+
removedCount,
|
|
283
|
+
orphanCount,
|
|
284
|
+
danglingCount,
|
|
285
|
+
unreadable,
|
|
286
|
+
reclaimedBytes,
|
|
287
|
+
dryRun,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
@@ -22,6 +22,9 @@
|
|
|
22
22
|
export const CONSUMER_PATHS = [
|
|
23
23
|
'packages/e2e/**',
|
|
24
24
|
'packages/registry-server/**',
|
|
25
|
+
// The smoke test IS this gate's body — the only place publishModule and
|
|
26
|
+
// friends run in the consumer install shape (celilo#1142).
|
|
27
|
+
'e2e/tests/smoke.test.ts',
|
|
25
28
|
// The publish driver (moved from the dead apps/celilo/scripts/publish.ts).
|
|
26
29
|
'apps/celilo/src/cli/commands/publish/**',
|
|
27
30
|
'scripts/publish.ts',
|
package/src/address-plan.test.ts
CHANGED
|
@@ -50,6 +50,25 @@ const SCANNED_PATHS = ['packages/e2e', 'e2e/tests', 'modules/*/e2e'];
|
|
|
50
50
|
*/
|
|
51
51
|
const EXCLUDED = [/^packages\/e2e\/CHANGELOG\.md$/, /address-plan\.test\.ts$/];
|
|
52
52
|
|
|
53
|
+
/**
|
|
54
|
+
* A single LINE may exempt itself with `#539-ok: <reason>`.
|
|
55
|
+
*
|
|
56
|
+
* There is one legitimate reason to write a banned prefix: enumerating an
|
|
57
|
+
* RFC 1918 block as a CLASS, which any correct private-address predicate must
|
|
58
|
+
* do. `192.168.` is both the operator's LAN (banned) and 192.168.0.0/16
|
|
59
|
+
* (unavoidable), and RETIRED_PREFIXES cannot tell them apart. The tell that
|
|
60
|
+
* this is the prefix list's blind spot rather than a real judgement: `10.` sits
|
|
61
|
+
* beside `192.168.` in exactly such a predicate and escapes only because `10.`
|
|
62
|
+
* was never added to the list.
|
|
63
|
+
*
|
|
64
|
+
* Deliberately per-line and not per-file. Excluding a file would blind the gate
|
|
65
|
+
* to every OTHER address in it, which is how a suppression becomes the next
|
|
66
|
+
* leak. The reason is required — a marker with nothing after the colon does not
|
|
67
|
+
* match, because an exemption that names only a line number is the next
|
|
68
|
+
* reader's mystery.
|
|
69
|
+
*/
|
|
70
|
+
const INLINE_EXEMPTION = /#539-ok:\s*\S/;
|
|
71
|
+
|
|
53
72
|
function grepRetired(): string[] {
|
|
54
73
|
const pattern = RETIRED_PREFIXES.map(toPattern).join('|');
|
|
55
74
|
let out = '';
|
|
@@ -66,7 +85,8 @@ function grepRetired(): string[] {
|
|
|
66
85
|
return out
|
|
67
86
|
.split('\n')
|
|
68
87
|
.filter(Boolean)
|
|
69
|
-
.filter((line) => !EXCLUDED.some((re) => re.test(line.split(':')[0])))
|
|
88
|
+
.filter((line) => !EXCLUDED.some((re) => re.test(line.split(':')[0] as string)))
|
|
89
|
+
.filter((line) => !INLINE_EXEMPTION.test(line));
|
|
70
90
|
}
|
|
71
91
|
|
|
72
92
|
describe('simulated address plan (#539)', () => {
|
package/src/container-manager.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type ExecSyncOptions, execSync, spawn } from 'node:child_process';
|
|
2
|
-
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
3
3
|
import { tmpdir } from 'node:os';
|
|
4
4
|
import { basename, join, resolve } from 'node:path';
|
|
5
5
|
import { parse as parseYaml } from 'yaml';
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
firewallZoneLegs,
|
|
11
11
|
generateTestComposeYaml,
|
|
12
12
|
getAllMachines,
|
|
13
|
+
registryUploadsHostDir,
|
|
13
14
|
} from './docker-compose-generator';
|
|
14
15
|
import { explainBuildFailure } from './doctor';
|
|
15
16
|
import { type ModuleHost, parseModuleHost } from './module-host';
|
|
@@ -1005,12 +1006,19 @@ function buildNetworkHandle(
|
|
|
1005
1006
|
}
|
|
1006
1007
|
|
|
1007
1008
|
try {
|
|
1008
|
-
//
|
|
1009
|
-
//
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1009
|
+
// Write HOST-side into the dir bound at the registry's /uploads. The
|
|
1010
|
+
// registry rescans that dir on every request, so it needs no HTTP
|
|
1011
|
+
// handshake and no restart.
|
|
1012
|
+
//
|
|
1013
|
+
// Deliberately not a `docker compose cp` into the container: /uploads
|
|
1014
|
+
// is a READ-ONLY bind in a consumer install, so the container write
|
|
1015
|
+
// failed for every npm-installed consumer while passing forever in the
|
|
1016
|
+
// monorepo, where the same path is not a mount at all (celilo#1142).
|
|
1017
|
+
// registryUploadsHostDir() is the same function the compose generator
|
|
1018
|
+
// builds the mount from, so the two cannot drift apart again.
|
|
1019
|
+
const uploadsDir = registryUploadsHostDir();
|
|
1020
|
+
mkdirSync(uploadsDir, { recursive: true });
|
|
1021
|
+
copyFileSync(netappPath, join(uploadsDir, `${moduleId}.netapp`));
|
|
1014
1022
|
} finally {
|
|
1015
1023
|
if (cleanup)
|
|
1016
1024
|
try {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { existsSync } from 'node:fs';
|
|
2
|
-
import { join } from 'node:path';
|
|
1
|
+
import { existsSync, mkdirSync, readdirSync, rmSync } from 'node:fs';
|
|
2
|
+
import { basename, join } from 'node:path';
|
|
3
3
|
import { stringify } from 'yaml';
|
|
4
4
|
import { normalizeObservers, observerEnv, observerPlacement } from './observer';
|
|
5
5
|
import { PROXMOX_SIM_IP, SIMULATOR_IPS } from './simulator-ips';
|
|
@@ -13,32 +13,76 @@ import {
|
|
|
13
13
|
zoneIp,
|
|
14
14
|
} from './types';
|
|
15
15
|
|
|
16
|
+
const PACKAGE_DIR = join(import.meta.dir, '..');
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Is this @celilo/e2e running from the monorepo, or from an npm install?
|
|
20
|
+
*
|
|
21
|
+
* `<pkgDir>/../../modules` is `<repo>/modules` in the monorepo and
|
|
22
|
+
* `<consumer>/node_modules/modules` (nonexistent) in a consumer install.
|
|
23
|
+
*/
|
|
24
|
+
function isMonorepoInstall(): boolean {
|
|
25
|
+
return existsSync(join(PACKAGE_DIR, '..', '..', 'modules'));
|
|
26
|
+
}
|
|
27
|
+
|
|
16
28
|
/**
|
|
17
|
-
*
|
|
18
|
-
*
|
|
29
|
+
* The host dir bound read-only at the registry's `/uploads` — and therefore
|
|
30
|
+
* the dir `publishModule` writes a `.netapp` into. The registry rescans it on
|
|
31
|
+
* every request, so a host-side write is picked up with no container write and
|
|
32
|
+
* no HTTP handshake.
|
|
33
|
+
*
|
|
34
|
+
* Both install shapes bind one. They differ only in WHICH, because they differ
|
|
35
|
+
* in where standard modules come from:
|
|
36
|
+
*
|
|
37
|
+
* - **Monorepo dev** — modules are served live from `/modules`
|
|
38
|
+
* (`../../modules`, BOOTSTRAP_MODULES_DIR mode), so edits to a module's
|
|
39
|
+
* source flow through without a manual repack. `/uploads` is therefore a
|
|
40
|
+
* dedicated drop-zone, emptied on every shared-infra start (see
|
|
41
|
+
* `prepareRegistryDropZone`) — a leftover `.netapp` would shadow live source.
|
|
19
42
|
*
|
|
20
|
-
* - **
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
43
|
+
* - **npm-installed consumer** (lunacycle and friends) — `../../modules` does
|
|
44
|
+
* not exist, so there is no source to serve. `/uploads` IS the netapp cache
|
|
45
|
+
* `cele2e build-infra` fills from the public registry (see
|
|
46
|
+
* stageNetappsFromRegistry in cli/build.ts), and a published module lands
|
|
47
|
+
* beside the standard ones. Consumers vendor nothing.
|
|
24
48
|
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
|
|
32
|
-
|
|
49
|
+
* Read this through `registryUploadsHostDir()` rather than re-deriving it.
|
|
50
|
+
* publishModule used to `docker cp` into the container's `/uploads` instead,
|
|
51
|
+
* which works only where that path is NOT a mount — so it passed forever in
|
|
52
|
+
* the monorepo and failed for every consumer against the read-only bind
|
|
53
|
+
* (celilo#1142). One function now decides the mount and the write target
|
|
54
|
+
* together, so they cannot disagree again.
|
|
55
|
+
*/
|
|
56
|
+
export function registryUploadsHostDir(): string {
|
|
57
|
+
return join(PACKAGE_DIR, isMonorepoInstall() ? 'uploads' : 'netapps');
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Make the drop-zone ready for a shared-infra bring-up.
|
|
33
62
|
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
63
|
+
* Creates it: Docker silently creates a MISSING bind source as an empty
|
|
64
|
+
* root-owned directory, which the host then cannot write into.
|
|
65
|
+
*
|
|
66
|
+
* Empties it, but only in the monorepo, where `/uploads` is dedicated — a
|
|
67
|
+
* `.netapp` left by a previous run would shadow the live module source it was
|
|
68
|
+
* packed from. In a consumer install the same dir IS the build-infra netapp
|
|
69
|
+
* cache, so clearing it would delete every standard module. (Before
|
|
70
|
+
* celilo#1142 published netapps lived in the registry's container layer and
|
|
71
|
+
* died with the container, so starting empty is the behaviour being kept, not
|
|
72
|
+
* a new one.)
|
|
38
73
|
*/
|
|
74
|
+
export function prepareRegistryDropZone(): void {
|
|
75
|
+
const dir = registryUploadsHostDir();
|
|
76
|
+
mkdirSync(dir, { recursive: true });
|
|
77
|
+
if (!isMonorepoInstall()) return;
|
|
78
|
+
for (const file of readdirSync(dir)) {
|
|
79
|
+
if (file.endsWith('.netapp')) rmSync(join(dir, file), { force: true });
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
39
83
|
function getRegistryVolumes(): string[] {
|
|
40
|
-
const
|
|
41
|
-
return
|
|
84
|
+
const uploads = `./${basename(registryUploadsHostDir())}:/uploads:ro`;
|
|
85
|
+
return isMonorepoInstall() ? ['../../modules:/modules:ro', uploads] : [uploads];
|
|
42
86
|
}
|
|
43
87
|
|
|
44
88
|
const NETWORK_DRIVER_OPTS = {
|
package/src/shared-infra.ts
CHANGED
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
SHARED_NETWORKS,
|
|
18
18
|
SHARED_PROJECT_NAME,
|
|
19
19
|
generateSharedInfraYaml,
|
|
20
|
+
prepareRegistryDropZone,
|
|
20
21
|
} from './docker-compose-generator';
|
|
21
22
|
import { ensureRegistryServerBundle, ensureTerraformFakeBundle } from './registry-bundle';
|
|
22
23
|
|
|
@@ -161,6 +162,11 @@ export async function ensureSharedInfra(): Promise<void> {
|
|
|
161
162
|
const yaml = generateSharedInfraYaml();
|
|
162
163
|
writeFileSync(join(e2eDir, SHARED_COMPOSE_FILE), yaml);
|
|
163
164
|
|
|
165
|
+
// The registry's /uploads bind source must exist before `up` (a missing
|
|
166
|
+
// bind source becomes an empty root-owned dir), and in the monorepo it
|
|
167
|
+
// starts empty so no stale .netapp shadows live module source.
|
|
168
|
+
prepareRegistryDropZone();
|
|
169
|
+
|
|
164
170
|
// Seed the live DNS zone files from their templates BEFORE the compose
|
|
165
171
|
// mounts them. config/dns/{iamtheinternet.org,example.net}.zone are
|
|
166
172
|
// gitignored runtime state (scrubDnsZones rewrites them per-test); on a
|