@celilo/e2e 0.19.0 → 0.19.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@celilo/e2e",
3
- "version": "0.19.0",
3
+ "version": "0.19.1",
4
4
  "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
5
5
  "type": "module",
6
6
  "main": "./src/index.ts",
@@ -18,7 +18,7 @@
18
18
  "celilo-e2e-infra": "./bin/e2e-infra"
19
19
  },
20
20
  "scripts": {
21
- "test": "bun test --timeout 30000 tests/",
21
+ "test": "bun test --timeout 30000 ./tests/ ./src/ ./npm-registry-server/ ./simulators/",
22
22
  "test:completion": "bun test tests/completion",
23
23
  "test:integration": "bun test tests-integration/"
24
24
  },
@@ -37,7 +37,7 @@
37
37
  "README.md"
38
38
  ],
39
39
  "dependencies": {
40
- "@celilo/capabilities": "^3.1.0",
40
+ "@celilo/capabilities": "^3.3.0",
41
41
  "@celilo/cli-display": "^0.2.0",
42
42
  "@celilo/event-bus": "^0.6.0",
43
43
  "@celilo/terraform-fake": "^0.3.1",
@@ -33,6 +33,8 @@ export interface BootstrapEntry {
33
33
  * apps/celilo/designs/REGISTRY_BROWSE_UI.md (Phase 2 step 0).
34
34
  */
35
35
  description?: string;
36
+ /** Manifest's `icon` field, when present. Served like `description`. */
37
+ icon?: string;
36
38
  }
37
39
 
38
40
  /**
@@ -60,11 +62,14 @@ export function scanBootstrapDir(bootstrapDir: string): Map<string, BootstrapEnt
60
62
  const name = idMatch?.[1]?.trim() ?? entry.name;
61
63
  const version = versionMatch?.[1]?.trim() ?? '0.0.0';
62
64
  const description = descriptionMatch?.[1]?.trim().replace(/^['"]|['"]$/g, '');
65
+ const iconMatch = yaml.match(/^icon:\s*(.+?)\s*$/m);
66
+ const icon = iconMatch?.[1]?.trim().replace(/^['"]|['"]$/g, '');
63
67
  entries.set(name, {
64
68
  name,
65
69
  version,
66
70
  sourceDir: join(bootstrapDir, entry.name),
67
71
  description,
72
+ icon,
68
73
  });
69
74
  } catch {
70
75
  // skip malformed manifests — matches e2e server behavior
@@ -217,6 +217,64 @@ describe('GET /api/v1/modules (search)', () => {
217
217
  expect(found?.description).toBe('');
218
218
  });
219
219
 
220
+ test('icon survives publish, the sparse index and a search read', async () => {
221
+ // openspec/changes/module-icons D4 — the icon rides the same path
222
+ // `description` does: publish metadata → index entry → browse endpoints.
223
+ expect((await publish('homebridge', '1.0.0+1', 'valid-token', 'Bridge', '\u22c8')).status).toBe(
224
+ 200,
225
+ );
226
+
227
+ const index = await fetch(`${baseUrl}/index/ho/me/homebridge`);
228
+ expect(index.status).toBe(200);
229
+ const line = JSON.parse((await index.text()).trim()) as { icon?: string };
230
+ expect(line.icon).toBe('\u22c8');
231
+
232
+ const res = await fetch(`${baseUrl}/api/v1/modules`);
233
+ const body = (await res.json()) as { modules: Array<{ name: string; icon?: string }> };
234
+ expect(body.modules.find((m) => m.name === 'homebridge')?.icon).toBe('\u22c8');
235
+
236
+ const detail = await fetch(`${baseUrl}/api/v1/modules/homebridge`);
237
+ expect(((await detail.json()) as { icon?: string }).icon).toBe('\u22c8');
238
+ });
239
+
240
+ test('an entry published without an icon reads back as undefined, not a throw', async () => {
241
+ // The old-entry case D4 depends on: index lines written before the field
242
+ // existed carry no `icon`, and a consumer falls back rather than failing.
243
+ //
244
+ // This test passes with or without the icon feature, so it is NOT a gate on
245
+ // it — verified by running it against the pre-#1147 tree. It is here to
246
+ // catch a future change that makes a missing `icon` throw or coerce to '',
247
+ // which is what D4's fallback chain would break on. Do not count it as
248
+ // coverage of the capture path; that is the test above.
249
+ expect((await publish('homebridge', '1.0.0+1', 'valid-token')).status).toBe(200);
250
+
251
+ const res = await fetch(`${baseUrl}/api/v1/modules`);
252
+ const body = (await res.json()) as { modules: Array<{ name: string; icon?: string }> };
253
+ const found = body.modules.find((m) => m.name === 'homebridge');
254
+ expect(found).toBeTruthy();
255
+ expect(found?.icon).toBeUndefined();
256
+ });
257
+
258
+ test('a multi-character icon is dropped while a valid one beside it is kept', async () => {
259
+ // Publish metadata is arbitrary client JSON. The CLI refines the value at
260
+ // manifest-parse time; the server keeps the index from carrying a string
261
+ // that would blow out a fixed-width slot on the browse page.
262
+ //
263
+ // Both halves are asserted deliberately. Checking only that the bad value
264
+ // is absent would pass just as well if the capture path were deleted
265
+ // outright — the guard and the capture would cancel out and the test would
266
+ // stay green having lost the feature. The kept glyph is what stops that.
267
+ expect(
268
+ (await publish('homebridge', '1.0.0+1', 'valid-token', 'Bridge', 'not-a-glyph')).status,
269
+ ).toBe(200);
270
+ expect((await publish('caddy', '1.0.0+1', 'valid-token', 'Proxy', '\u25cd')).status).toBe(200);
271
+
272
+ const res = await fetch(`${baseUrl}/api/v1/modules`);
273
+ const body = (await res.json()) as { modules: Array<{ name: string; icon?: string }> };
274
+ expect(body.modules.find((m) => m.name === 'homebridge')?.icon).toBeUndefined();
275
+ expect(body.modules.find((m) => m.name === 'caddy')?.icon).toBe('\u25cd');
276
+ });
277
+
220
278
  test('total_downloads reflects actual download count, sort=downloads orders by it', async () => {
221
279
  // Publish two modules, hit one's download endpoint several times.
222
280
  // The search response should show those counts and sort=downloads
@@ -490,9 +548,15 @@ async function publish(
490
548
  vers: string,
491
549
  token: string,
492
550
  description?: string,
551
+ icon?: string,
493
552
  ): Promise<Response> {
494
553
  const meta = Buffer.from(
495
- JSON.stringify({ name, vers, ...(description ? { description } : {}) }),
554
+ JSON.stringify({
555
+ name,
556
+ vers,
557
+ ...(description ? { description } : {}),
558
+ ...(icon ? { icon } : {}),
559
+ }),
496
560
  'utf-8',
497
561
  );
498
562
  const file = Buffer.from('fake netapp bytes');
@@ -769,3 +833,121 @@ describe('module-owner authorization (ce-1ch)', () => {
769
833
  expect(res.status).toBe(200);
770
834
  });
771
835
  });
836
+
837
+ // ── sweep (reclaiming disk from superseded build revisions) ──────────────────
838
+
839
+ describe('POST /api/v1/modules/sweep', () => {
840
+ function requestSweep(token: string, body: unknown = {}): Promise<Response> {
841
+ return fetch(`${baseUrl}/api/v1/modules/sweep`, {
842
+ method: 'POST',
843
+ headers: token ? { Authorization: token, 'Content-Type': 'application/json' } : {},
844
+ body: JSON.stringify(body),
845
+ });
846
+ }
847
+
848
+ test('no token is 401 — a sweep deletes, so it is admin-only', async () => {
849
+ const res = await requestSweep('');
850
+ expect(res.status).toBe(401);
851
+ });
852
+
853
+ test('a scoped (non-admin) token is 401', async () => {
854
+ const mint = await fetch(`${baseUrl}/api/v1/modules/tokens/mint`, {
855
+ method: 'POST',
856
+ headers: { Authorization: 'valid-token', 'Content-Type': 'application/json' },
857
+ body: JSON.stringify({ repo: 'celilo/homebridge', scope: 'homebridge' }),
858
+ });
859
+ const { token } = (await mint.json()) as { token: string };
860
+
861
+ const res = await requestSweep(token);
862
+ expect(res.status).toBe(401);
863
+ });
864
+
865
+ test('removes superseded revisions and leaves the download path working', async () => {
866
+ for (const rev of [1, 2, 3]) await publish('homebridge', `1.0.0+${rev}`, 'valid-token');
867
+
868
+ const res = await requestSweep('valid-token');
869
+ expect(res.status).toBe(200);
870
+ const body = (await res.json()) as { ok: boolean; removedCount: number };
871
+ expect(body.ok).toBe(true);
872
+ expect(body.removedCount).toBe(2);
873
+
874
+ // The surviving revision downloads; a swept one is gone from BOTH the
875
+ // index and the store, so nothing advertises a 404.
876
+ expect((await fetch(`${baseUrl}/api/v1/modules/homebridge/1.0.0+3/download`)).status).toBe(200);
877
+ expect((await fetch(`${baseUrl}/api/v1/modules/homebridge/1.0.0+1/download`)).status).toBe(404);
878
+
879
+ const index = await fetch(`${baseUrl}/index/ho/me/homebridge`);
880
+ const versions = (await index.text())
881
+ .split('\n')
882
+ .filter(Boolean)
883
+ .map((line) => (JSON.parse(line) as { vers: string }).vers);
884
+ expect(versions).toEqual(['1.0.0+3']);
885
+ });
886
+
887
+ test('dry_run reports the plan without deleting anything', async () => {
888
+ for (const rev of [1, 2]) await publish('homebridge', `1.0.0+${rev}`, 'valid-token');
889
+
890
+ const res = await requestSweep('valid-token', { dry_run: true });
891
+ const body = (await res.json()) as { removedCount: number; dryRun: boolean };
892
+ expect(body.removedCount).toBe(1);
893
+ expect(body.dryRun).toBe(true);
894
+ expect((await fetch(`${baseUrl}/api/v1/modules/homebridge/1.0.0+1/download`)).status).toBe(200);
895
+ });
896
+
897
+ test('rejects a keep_build_revisions that would delete a whole release', async () => {
898
+ const res = await requestSweep('valid-token', { keep_build_revisions: 0 });
899
+ expect(res.status).toBe(400);
900
+ });
901
+
902
+ test('honours a larger keep_build_revisions', async () => {
903
+ for (const rev of [1, 2, 3, 4]) await publish('homebridge', `1.0.0+${rev}`, 'valid-token');
904
+ const res = await requestSweep('valid-token', { keep_build_revisions: 2 });
905
+ const body = (await res.json()) as { removedCount: number };
906
+ expect(body.removedCount).toBe(2);
907
+ });
908
+
909
+ test('coexists with a module actually named "sweep"', async () => {
910
+ await publish('sweep', '1.0.0+1', 'valid-token');
911
+ const res = await requestSweep('valid-token');
912
+ expect(res.status).toBe(200);
913
+ // …and the module is still reachable by its own GET route.
914
+ expect((await fetch(`${baseUrl}/api/v1/modules/sweep`)).status).toBe(200);
915
+ });
916
+ });
917
+
918
+ describe('POST /api/v1/modules/sweep — a damaged store', () => {
919
+ function requestSweep(token: string, body: unknown = {}): Promise<Response> {
920
+ return fetch(`${baseUrl}/api/v1/modules/sweep`, {
921
+ method: 'POST',
922
+ headers: { Authorization: token, 'Content-Type': 'application/json' },
923
+ body: JSON.stringify(body),
924
+ });
925
+ }
926
+
927
+ test('reclaims a half-written publish, freeing the version to be published again', async () => {
928
+ // handlePublish stores the package and appends the index with no rollback
929
+ // between them, so an ENOSPC or EACCES in the gap leaves an orphan. The
930
+ // immutability check reads the filesystem, so that version then cannot be
931
+ // published at all — this is the only repair path in the product.
932
+ await publish('homebridge', '1.0.0+1', 'valid-token');
933
+ const rmIndex = await fetch(`${baseUrl}/api/v1/modules/homebridge/1.0.0+1/yank`, {
934
+ method: 'DELETE',
935
+ headers: { Authorization: 'valid-token' },
936
+ });
937
+ expect(rmIndex.status).toBe(200);
938
+
939
+ // Simulate the crash: index line gone, payload left behind.
940
+ rmSync(join(dataDir, 'index'), { recursive: true, force: true });
941
+
942
+ const blocked = await publish('homebridge', '1.0.0+1', 'valid-token');
943
+ expect(blocked.status).toBe(409);
944
+
945
+ const swept = await requestSweep('valid-token');
946
+ const body = (await swept.json()) as { orphanCount: number };
947
+ expect(body.orphanCount).toBe(1);
948
+
949
+ // The version is publishable again.
950
+ const retry = await publish('homebridge', '1.0.0+1', 'valid-token');
951
+ expect(retry.status).toBe(200);
952
+ });
953
+ });
@@ -10,6 +10,7 @@
10
10
  * PUT /api/v1/modules/new → publish (token auth)
11
11
  * DELETE /api/v1/modules/{name}/{ver}/yank → yank
12
12
  * PUT /api/v1/modules/{name}/{ver}/unyank → unyank
13
+ * POST /api/v1/modules/sweep → reclaim old revisions (admin)
13
14
  *
14
15
  * All routes can be prefixed with PATH_PREFIX (e.g. "/registry").
15
16
  *
@@ -34,6 +35,7 @@ import { ModuleOwnerStore, fileModuleOwnerPersistence } from './module-owner-sto
34
35
  import { type RateLimiter, clientIp, createRateLimiter } from './rate-limit';
35
36
  import { ScopedTokenStore, fileScopedTokenPersistence } from './scoped-token-store';
36
37
  import { type IndexEntry, RegistryStorage } from './storage';
38
+ import { DEFAULT_KEEP_BUILD_REVISIONS, sweep } from './sweep';
37
39
  import { isValidName, isValidVersion, validateNameAndVersion } from './validation';
38
40
 
39
41
  /** Max total publish body size. Real modules are a few MB; 100MB is generous. */
@@ -313,6 +315,12 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
313
315
  // not provided we leave it undefined; the search endpoint falls
314
316
  // back to bootstrap data or empty.
315
317
  const description = typeof meta.description === 'string' ? meta.description : undefined;
318
+ // Icon likewise, but capped at one Unicode scalar. The CLI already refines
319
+ // it at manifest-parse time; this second check is here because the publish
320
+ // metadata is arbitrary client JSON and the value lands in a fixed-width
321
+ // slot on the browse page.
322
+ const icon =
323
+ typeof meta.icon === 'string' && [...meta.icon].length === 1 ? meta.icon : undefined;
316
324
 
317
325
  const validation = validateNameAndVersion(name, vers);
318
326
  if (!validation.ok) return err(validation.message);
@@ -339,12 +347,63 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
339
347
  cksum: `sha256:${cksum}`,
340
348
  yanked: false,
341
349
  description,
350
+ icon,
342
351
  });
343
352
 
344
353
  console.log(`[registry] published ${name}@${vers} (${fileLen} bytes, sha256:${cksum})`);
345
354
  return Response.json({ ok: true, name, vers });
346
355
  }
347
356
 
357
+ /**
358
+ * Reclaim disk from superseded build revisions (admin-only).
359
+ *
360
+ * The counterpart yank never was: yanking flips a boolean and frees nothing,
361
+ * so before this endpoint existed every revision ever published was retained
362
+ * forever and the store grew without bound until the disk filled.
363
+ *
364
+ * Runs IN the process that owns the store, so it cannot race a concurrent
365
+ * publish's index append. `dry_run` reports the same plan without touching
366
+ * anything — the safe way to see what a policy would do on a live store.
367
+ */
368
+ async function handleSweep(req: Request): Promise<Response> {
369
+ if (!(await authorizeAdminReq(req))) return unauthorized();
370
+
371
+ let body: { keep_build_revisions?: unknown; dry_run?: unknown } = {};
372
+ try {
373
+ const text = await req.text();
374
+ if (text.trim()) body = JSON.parse(text) as typeof body;
375
+ } catch {
376
+ return err('Invalid JSON body');
377
+ }
378
+
379
+ const requested = body.keep_build_revisions;
380
+ if (requested !== undefined && (!Number.isInteger(requested) || (requested as number) < 1)) {
381
+ return err('keep_build_revisions must be an integer >= 1');
382
+ }
383
+ const keepBuildRevisions = (requested as number | undefined) ?? DEFAULT_KEEP_BUILD_REVISIONS;
384
+ const dryRun = body.dry_run === true;
385
+
386
+ const report = sweep(storage, { keepBuildRevisions }, dryRun);
387
+ console.log(
388
+ `[registry] sweep${dryRun ? ' (dry run)' : ''} keep=${keepBuildRevisions}: ` +
389
+ `${report.removedCount} superseded + ${report.orphanCount} orphaned revision(s), ` +
390
+ `${Math.round(report.reclaimedBytes / 1024 / 1024)} MB`,
391
+ );
392
+ // Both of these mean the store is damaged, not merely untidy, and the sweep
393
+ // deliberately did not act on either. Say so where an operator will see it.
394
+ if (report.danglingCount > 0) {
395
+ console.warn(
396
+ `[registry] ${report.danglingCount} index line(s) have no package file — left in place; download 404s until republished`,
397
+ );
398
+ }
399
+ if (report.unreadable.length > 0) {
400
+ console.warn(
401
+ `[registry] skipped entirely (no package files at all — check the mount and DATA_DIR): ${report.unreadable.join(', ')}`,
402
+ );
403
+ }
404
+ return Response.json({ ok: true, ...report });
405
+ }
406
+
348
407
  function setYanked(name: string, version: string, yanked: boolean): Response {
349
408
  const entries = storage.readIndex(name);
350
409
  const entry = entries.find((e) => e.vers === version);
@@ -505,10 +564,15 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
505
564
  // description capture and haven't been republished).
506
565
  const bootstrapEntry = bootstrap.get(name);
507
566
  const description = latest?.description ?? bootstrapEntry?.description ?? '';
567
+ // Icon follows the same order, but has no empty-string tier: absent
568
+ // means "this module declared none", and the consumer resolves it
569
+ // from its own table or a placeholder (module-icons D5).
570
+ const icon = latest?.icon ?? bootstrapEntry?.icon;
508
571
  return {
509
572
  name,
510
573
  max_version: latest?.vers ?? '0.0.0',
511
574
  description,
575
+ icon,
512
576
  total_downloads: storage.getDownloads(name),
513
577
  };
514
578
  });
@@ -551,6 +615,15 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
551
615
  return handleRevokeToken(req);
552
616
  }
553
617
 
618
+ // Reclaim disk from superseded build revisions (admin-only), grouped
619
+ // with the other admin endpoints. A module NAMED "sweep" is fine here:
620
+ // its metadata route is a GET, so the two never compete.
621
+ if (method === 'POST' && suffix === `${apiBase}/sweep`) {
622
+ const rl = rateLimitOrNull(req, srv);
623
+ if (rl) return rl;
624
+ return handleSweep(req);
625
+ }
626
+
554
627
  // Module-owner table management (admin-only — ce-1ch). Matched before the
555
628
  // generic `${apiBase}/{name}` handlers so a name of "owners" can't shadow
556
629
  // them.
@@ -588,9 +661,11 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
588
661
  const latest = entries.filter((v) => !v.yanked).at(-1) ?? entries.at(-1);
589
662
  const bootstrapEntry = resolveBootstrap().get(name);
590
663
  const description = latest?.description ?? bootstrapEntry?.description ?? '';
664
+ const icon = latest?.icon ?? bootstrapEntry?.icon;
591
665
  return Response.json({
592
666
  name,
593
667
  description,
668
+ icon,
594
669
  total_downloads: storage.getDownloads(name),
595
670
  versions: entries.map((v) => ({
596
671
  num: v.vers,
@@ -8,7 +8,16 @@
8
8
  */
9
9
 
10
10
  import { createHash } from 'node:crypto';
11
- import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
11
+ import {
12
+ existsSync,
13
+ mkdirSync,
14
+ readFileSync,
15
+ readdirSync,
16
+ renameSync,
17
+ rmSync,
18
+ statSync,
19
+ writeFileSync,
20
+ } from 'node:fs';
12
21
  import { dirname, join } from 'node:path';
13
22
 
14
23
  export interface IndexEntry {
@@ -25,6 +34,14 @@ export interface IndexEntry {
25
34
  * step 0). Old index files without this field still parse cleanly.
26
35
  */
27
36
  description?: string;
37
+ /**
38
+ * One monochrome glyph from the module's `manifest.yml#icon`, captured at
39
+ * publish time. Optional for the same reason `description` is: entries
40
+ * written before the field existed have none, and a consumer reading one
41
+ * without it falls back to its own table rather than failing
42
+ * (openspec/changes/module-icons, D4).
43
+ */
44
+ icon?: string;
28
45
  }
29
46
 
30
47
  export interface ModuleVersion {
@@ -79,10 +96,28 @@ export class RegistryStorage {
79
96
  writeFileSync(path, `${JSON.stringify(entry)}\n`, { flag: 'a' });
80
97
  }
81
98
 
99
+ /**
100
+ * Replace a module's whole index file.
101
+ *
102
+ * Writes a sibling temp file and renames it over the original, which is
103
+ * atomic within a directory. A plain truncating write has two failure modes
104
+ * this avoids, and the sweep meets both: a crash mid-write leaves a half-
105
+ * index, and an ENOSPC leaves an EMPTY one — losing every version of the
106
+ * module. The sweep exists precisely because the disk filled up, so "the
107
+ * write fails for want of space" is its expected environment, not a corner.
108
+ * On failure the original file is untouched.
109
+ */
82
110
  updateIndex(name: string, entries: IndexEntry[]): void {
83
111
  const path = this.indexPath(name);
84
112
  mkdirSync(dirname(path), { recursive: true });
85
- writeFileSync(path, `${entries.map((e) => JSON.stringify(e)).join('\n')}\n`);
113
+ const tmp = `${path}.tmp`;
114
+ try {
115
+ writeFileSync(tmp, `${entries.map((e) => JSON.stringify(e)).join('\n')}\n`);
116
+ renameSync(tmp, path);
117
+ } catch (err) {
118
+ rmSync(tmp, { force: true, recursive: true });
119
+ throw err;
120
+ }
86
121
  }
87
122
 
88
123
  storePackage(name: string, version: string, data: Buffer): string {
@@ -96,6 +131,45 @@ export class RegistryStorage {
96
131
  return existsSync(this.packagePath(name, version));
97
132
  }
98
133
 
134
+ /** Bytes on disk for one version's .netapp, or 0 when it is already gone. */
135
+ packageSize(name: string, version: string): number {
136
+ const path = this.packagePath(name, version);
137
+ if (!existsSync(path)) return 0;
138
+ return statSync(path).size;
139
+ }
140
+
141
+ /**
142
+ * Delete one version's payload directory (`modules/{name}/{version}/`).
143
+ * Idempotent: removing a version that is already gone is a no-op, so an
144
+ * interrupted sweep is safe to re-run.
145
+ */
146
+ removePackage(name: string, version: string): void {
147
+ rmSync(dirname(this.packagePath(name, version)), { recursive: true, force: true });
148
+ }
149
+
150
+ /**
151
+ * Every version with a payload directory on disk, whether or not the index
152
+ * still lists it. The sweep needs the on-DISK set, not the indexed one:
153
+ * removing an index line is what makes a version unreachable, and the file
154
+ * it leaves behind is exactly what has to be reclaimed afterwards.
155
+ */
156
+ storedVersions(name: string): string[] {
157
+ const dir = join(this.dataDir, 'modules', name);
158
+ if (!existsSync(dir)) return [];
159
+ return readdirSync(dir, { withFileTypes: true })
160
+ .filter((d) => d.isDirectory())
161
+ .map((d) => d.name);
162
+ }
163
+
164
+ /** Module names with a payload directory, including any absent from the index. */
165
+ storedNames(): string[] {
166
+ const dir = join(this.dataDir, 'modules');
167
+ if (!existsSync(dir)) return [];
168
+ return readdirSync(dir, { withFileTypes: true })
169
+ .filter((d) => d.isDirectory())
170
+ .map((d) => d.name);
171
+ }
172
+
99
173
  readPackage(name: string, version: string): Buffer | null {
100
174
  const path = this.packagePath(name, version);
101
175
  return existsSync(path) ? readFileSync(path) : null;
@@ -0,0 +1,326 @@
1
+ import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
2
+ import { existsSync, mkdirSync, mkdtempSync, rmSync } from 'node:fs';
3
+ import { tmpdir } from 'node:os';
4
+ import { join } from 'node:path';
5
+ import { RegistryStorage } from './storage';
6
+ import { DEFAULT_KEEP_BUILD_REVISIONS, parseRevision, planModuleSweep, sweep } from './sweep';
7
+
8
+ let dataDir: string;
9
+ let storage: RegistryStorage;
10
+
11
+ beforeEach(() => {
12
+ dataDir = mkdtempSync(join(tmpdir(), 'celilo-registry-sweep-'));
13
+ storage = new RegistryStorage(dataDir);
14
+ });
15
+
16
+ afterEach(() => {
17
+ rmSync(dataDir, { recursive: true, force: true });
18
+ });
19
+
20
+ /** Publish a version through the same two calls the real publish path uses. */
21
+ function publish(name: string, vers: string, sizeBytes = 32): void {
22
+ storage.storePackage(name, vers, Buffer.alloc(sizeBytes, 7));
23
+ storage.appendIndex({ name, vers, deps: [], cksum: `sha256:${vers}`, yanked: false });
24
+ }
25
+
26
+ /**
27
+ * THE invariant. A sweep that trims payloads without trimming index lines
28
+ * leaves the index advertising a version whose file is gone — search lists it
29
+ * and the download 404s. The reverse leaves unreachable bytes on the disk the
30
+ * sweep exists to reclaim. Both directions have to hold.
31
+ */
32
+ function expectIndexAndPayloadsAgree(name: string): void {
33
+ const indexed = storage.readIndex(name).map((e) => e.vers);
34
+ const stored = storage.storedVersions(name).sort();
35
+ for (const vers of indexed) {
36
+ expect(storage.readPackage(name, vers)).not.toBeNull();
37
+ }
38
+ expect(stored).toEqual([...indexed].sort());
39
+ }
40
+
41
+ describe('parseRevision', () => {
42
+ test('splits a release from its build revision', () => {
43
+ expect(parseRevision('1.0.3+7')).toEqual({ release: '1.0.3', revision: 7 });
44
+ });
45
+
46
+ test('keeps a Debian epoch with the release', () => {
47
+ expect(parseRevision('2:1.0.0+1')).toEqual({ release: '2:1.0.0', revision: 1 });
48
+ });
49
+
50
+ test('returns null for a version with no build revision', () => {
51
+ expect(parseRevision('1.0.0')).toBeNull();
52
+ });
53
+ });
54
+
55
+ describe('planModuleSweep', () => {
56
+ const entry = (vers: string, yanked = false) => ({
57
+ name: 'm',
58
+ vers,
59
+ deps: [],
60
+ cksum: 'sha256:x',
61
+ yanked,
62
+ });
63
+
64
+ /**
65
+ * Plan against a store where every indexed version HAS its payload — the
66
+ * ordinary case. Passing an empty stored set instead would trip the
67
+ * unreadable guard, which is a different scenario with its own tests below.
68
+ */
69
+ const planWithPayloads = (entries: ReturnType<typeof entry>[], keepBuildRevisions: number) =>
70
+ planModuleSweep(
71
+ 'm',
72
+ entries,
73
+ entries.map((e) => e.vers),
74
+ { keepBuildRevisions },
75
+ );
76
+
77
+ test('keeps the newest build revision of every release', () => {
78
+ const entries = [
79
+ entry('1.0.0+1'),
80
+ entry('1.0.0+2'),
81
+ entry('1.0.2+8'),
82
+ entry('1.0.2+9'),
83
+ entry('1.0.3+5'),
84
+ entry('1.0.3+6'),
85
+ entry('1.0.3+7'),
86
+ ];
87
+ const plan = planWithPayloads(entries, 1);
88
+ expect(plan.keep.map((e) => e.vers)).toEqual(['1.0.0+2', '1.0.2+9', '1.0.3+7']);
89
+ expect(plan.remove).toEqual(['1.0.0+1', '1.0.2+8', '1.0.3+5', '1.0.3+6']);
90
+ });
91
+
92
+ test('never removes the only revision of a release', () => {
93
+ const entries = [entry('0.9.0+2'), entry('1.0.0+1')];
94
+ const plan = planWithPayloads(entries, 1);
95
+ expect(plan.remove).toEqual([]);
96
+ });
97
+
98
+ test('a module can never be swept out of existence', () => {
99
+ const entries = [entry('1.0.0+1')];
100
+ const plan = planWithPayloads(entries, 1);
101
+ expect(plan.keep).toHaveLength(1);
102
+ });
103
+
104
+ test('keeps more revisions per release when asked', () => {
105
+ const entries = [entry('1.0.3+5'), entry('1.0.3+6'), entry('1.0.3+7')];
106
+ const plan = planWithPayloads(entries, 2);
107
+ expect(plan.keep.map((e) => e.vers)).toEqual(['1.0.3+6', '1.0.3+7']);
108
+ });
109
+
110
+ test('keepBuildRevisions below 1 is clamped — no policy deletes a whole release', () => {
111
+ // Needs MORE THAN ONE release to be a real check. With a single release
112
+ // the latest-line retention below masks a missing clamp; with two, an
113
+ // unclamped 0 wipes out every release except the last.
114
+ const entries = [entry('1.0.0+1'), entry('1.0.0+2'), entry('1.0.3+5'), entry('1.0.3+6')];
115
+ const plan = planWithPayloads(entries, 0);
116
+ expect(plan.keep.map((e) => e.vers)).toEqual(['1.0.0+2', '1.0.3+6']);
117
+ });
118
+
119
+ test('retains what latestVersion resolves to even when the index is out of order', () => {
120
+ // Last line wins for RegistryClient.latestVersion regardless of ordering,
121
+ // so a disordered index must not lose the version clients actually fetch.
122
+ const entries = [entry('1.0.3+7'), entry('1.0.3+6'), entry('1.0.3+5')];
123
+ const plan = planWithPayloads(entries, 1);
124
+ expect(plan.keep.map((e) => e.vers)).toContain('1.0.3+5');
125
+ expect(plan.keep.map((e) => e.vers)).toContain('1.0.3+7');
126
+ });
127
+
128
+ test('retains the last NON-yanked line, which is what clients resolve to', () => {
129
+ const entries = [entry('1.0.3+5'), entry('1.0.3+6'), entry('1.0.3+7', true)];
130
+ const plan = planWithPayloads(entries, 1);
131
+ expect(plan.keep.map((e) => e.vers)).toContain('1.0.3+6');
132
+ });
133
+
134
+ test('keeps a version it cannot parse rather than deleting it', () => {
135
+ const entries = [entry('nonsense'), entry('1.0.0+1'), entry('1.0.0+2')];
136
+ const plan = planWithPayloads(entries, 1);
137
+ expect(plan.remove).toEqual(['1.0.0+1']);
138
+ });
139
+
140
+ test('reports a payload with no index line as an orphan', () => {
141
+ const plan = planModuleSweep('m', [entry('1.0.0+2')], ['1.0.0+1', '1.0.0+2'], {
142
+ keepBuildRevisions: 1,
143
+ });
144
+ expect(plan.orphans).toEqual(['1.0.0+1']);
145
+ });
146
+ });
147
+
148
+ describe('sweep over a fixture store', () => {
149
+ test('index and payloads agree afterwards, and every release survives', () => {
150
+ for (const rev of [1, 2, 3, 4, 5]) publish('celilo-registry', `1.0.3+${rev}`);
151
+ for (const rev of [1, 2]) publish('celilo-registry', `1.0.2+${rev}`);
152
+ publish('forgejo', '2.1.0+1');
153
+
154
+ const report = sweep(storage, { keepBuildRevisions: DEFAULT_KEEP_BUILD_REVISIONS });
155
+
156
+ expect(report.removedCount).toBe(5);
157
+ expectIndexAndPayloadsAgree('celilo-registry');
158
+ expectIndexAndPayloadsAgree('forgejo');
159
+ // Original index order is preserved: the fixture published 1.0.3 before
160
+ // 1.0.2, and the sweep filters lines rather than re-sorting them.
161
+ expect(storage.readIndex('celilo-registry').map((e) => e.vers)).toEqual(['1.0.3+5', '1.0.2+2']);
162
+ expect(storage.readIndex('forgejo').map((e) => e.vers)).toEqual(['2.1.0+1']);
163
+ });
164
+
165
+ test('reclaims the bytes it says it reclaims', () => {
166
+ publish('m', '1.0.0+1', 4096);
167
+ publish('m', '1.0.0+2', 4096);
168
+ const report = sweep(storage, { keepBuildRevisions: 1 });
169
+ expect(report.reclaimedBytes).toBe(4096);
170
+ });
171
+
172
+ test('a dry run changes nothing', () => {
173
+ for (const rev of [1, 2, 3]) publish('m', `1.0.0+${rev}`);
174
+ const report = sweep(storage, { keepBuildRevisions: 1 }, true);
175
+
176
+ expect(report.removedCount).toBe(2);
177
+ expect(storage.readIndex('m')).toHaveLength(3);
178
+ expect(storage.storedVersions('m').sort()).toEqual(['1.0.0+1', '1.0.0+2', '1.0.0+3']);
179
+ });
180
+
181
+ test('is idempotent — a second sweep finds nothing to do', () => {
182
+ for (const rev of [1, 2, 3]) publish('m', `1.0.0+${rev}`);
183
+ sweep(storage, { keepBuildRevisions: 1 });
184
+ const second = sweep(storage, { keepBuildRevisions: 1 });
185
+
186
+ expect(second.removedCount).toBe(0);
187
+ expect(second.orphanCount).toBe(0);
188
+ expect(second.reclaimedBytes).toBe(0);
189
+ expectIndexAndPayloadsAgree('m');
190
+ });
191
+
192
+ test('a sweep interrupted after the index rewrite self-heals on the next run', () => {
193
+ for (const rev of [1, 2, 3]) publish('m', `1.0.0+${rev}`);
194
+ // The crash window: index trimmed, payloads not yet unlinked. The version
195
+ // is unlisted but still on disk — wasted bytes, not a broken registry.
196
+ storage.updateIndex('m', storage.readIndex('m').slice(-1));
197
+ expect(storage.storedVersions('m')).toHaveLength(3);
198
+
199
+ const report = sweep(storage, { keepBuildRevisions: 1 });
200
+
201
+ expect(report.orphanCount).toBe(2);
202
+ expectIndexAndPayloadsAgree('m');
203
+ });
204
+
205
+ test('a version left unlisted stays downloadable until its payload goes', () => {
206
+ // Why index-first is the safe order: the download route reads the payload
207
+ // directly and never consults the index, so the window an interrupted
208
+ // sweep opens serves stale-but-valid bytes rather than 404ing.
209
+ publish('m', '1.0.0+1');
210
+ storage.updateIndex('m', []);
211
+ expect(storage.readPackage('m', '1.0.0+1')).not.toBeNull();
212
+ });
213
+
214
+ test('reclaims a half-written publish, which is what makes the version publishable again', () => {
215
+ // Both states below exist on the production registry, created 2026-08-22 by
216
+ // real failures: an ENOSPC that killed a run after storePackage, and an
217
+ // EACCES on appendIndex. handlePublish has no rollback between the two.
218
+ //
219
+ // The consequence is worse than wasted bytes: `packageExists` reads the
220
+ // FILESYSTEM, so the orphan makes its own version permanently
221
+ // unpublishable — "versions are immutable". Removing the orphan is the
222
+ // repair, and nothing else in the product performs it.
223
+ storage.storePackage('celilo-mgmt', '0.6.2+2', Buffer.alloc(2048, 1));
224
+ publish('celilo-mgmt', '0.6.1+1');
225
+ expect(storage.packageExists('celilo-mgmt', '0.6.2+2')).toBe(true);
226
+
227
+ const report = sweep(storage, { keepBuildRevisions: 1 });
228
+
229
+ expect(report.orphanCount).toBe(1);
230
+ expect(storage.packageExists('celilo-mgmt', '0.6.2+2')).toBe(false);
231
+ expectIndexAndPayloadsAgree('celilo-mgmt');
232
+ });
233
+
234
+ test('REPORTS an index line with no payload and does not remove it', () => {
235
+ // The dangerous direction. Removing the line would actually repair the
236
+ // module — a dangling entry already 404s — but a missing payload is
237
+ // indistinguishable from an unreadable store, so it is reported only.
238
+ publish('m', '1.0.0+1');
239
+ publish('m', '1.0.0+2');
240
+ storage.removePackage('m', '1.0.0+1');
241
+
242
+ const report = sweep(storage, { keepBuildRevisions: 1 });
243
+
244
+ expect(report.danglingCount).toBe(1);
245
+ expect(report.modules[0]?.dangling).toEqual(['1.0.0+1']);
246
+ // Still listed: the sweep reported it rather than acting on it.
247
+ expect(storage.readIndex('m').map((e) => e.vers)).toContain('1.0.0+1');
248
+ });
249
+
250
+ test('touches nothing when a module has index lines and no payloads at all', () => {
251
+ // An unmounted volume or a wrong DATA_DIR looks exactly like this. Acting
252
+ // on it would trim the index of a module whose files are merely
253
+ // unreachable.
254
+ for (const rev of [1, 2, 3]) publish('m', `1.0.0+${rev}`);
255
+ for (const rev of [1, 2, 3]) storage.removePackage('m', `1.0.0+${rev}`);
256
+
257
+ const report = sweep(storage, { keepBuildRevisions: 1 });
258
+
259
+ expect(report.unreadable).toEqual(['m']);
260
+ expect(report.removedCount).toBe(0);
261
+ expect(storage.readIndex('m')).toHaveLength(3);
262
+ });
263
+
264
+ test('a dangling entry does not stop the rest of the sweep', () => {
265
+ publish('m', '1.0.0+1');
266
+ publish('m', '1.0.0+2');
267
+ publish('m', '1.0.0+3');
268
+ storage.removePackage('m', '1.0.0+1');
269
+
270
+ const report = sweep(storage, { keepBuildRevisions: 1 });
271
+
272
+ expect(report.danglingCount).toBe(1);
273
+ // 1.0.0+2 is superseded and still had its payload, so it is swept normally.
274
+ expect(report.removedCount).toBe(1);
275
+ expect(storage.readIndex('m').map((e) => e.vers)).toEqual(['1.0.0+1', '1.0.0+3']);
276
+ });
277
+
278
+ test('never sweeps the last SERVABLE version because a newer line is dangling', () => {
279
+ // The failure this guards: retention computed over every index line lets a
280
+ // dangling NEWEST entry satisfy both the per-release rule and the
281
+ // latest-line rule, so every servable older version reads as superseded.
282
+ // The module ends up listed, with one index line, and undownloadable.
283
+ publish('m', '1.0.0+1');
284
+ publish('m', '1.0.0+2');
285
+ publish('m', '1.0.0+3');
286
+ storage.removePackage('m', '1.0.0+3'); // the newest line is now dangling
287
+
288
+ sweep(storage, { keepBuildRevisions: 1 });
289
+
290
+ // Something is still downloadable, which is the whole point.
291
+ const servable = storage
292
+ .readIndex('m')
293
+ .map((e) => e.vers)
294
+ .filter((v) => storage.packageExists('m', v));
295
+ expect(servable.length).toBeGreaterThan(0);
296
+ expect(servable).toContain('1.0.0+2');
297
+ });
298
+
299
+ test('an empty store sweeps cleanly', () => {
300
+ const report = sweep(storage, { keepBuildRevisions: 1 });
301
+ expect(report.removedCount).toBe(0);
302
+ expect(report.modules).toEqual([]);
303
+ });
304
+ });
305
+
306
+ describe('updateIndex atomicity', () => {
307
+ test('leaves no temp file behind', () => {
308
+ publish('m', '1.0.0+1');
309
+ storage.updateIndex('m', storage.readIndex('m'));
310
+ expect(existsSync(`${storage.indexPath('m')}.tmp`)).toBe(false);
311
+ });
312
+
313
+ test('a write that cannot land leaves the original index intact', () => {
314
+ publish('m', '1.0.0+1');
315
+ publish('m', '1.0.0+2');
316
+ const before = storage.readIndex('m');
317
+ // Block the temp path so the new content cannot be written. A truncating
318
+ // in-place write ignores the obstruction and replaces the index anyway;
319
+ // the rename path fails with the original still on disk. This stands in
320
+ // for the failure that actually matters — ENOSPC on the full disk this
321
+ // sweep exists to relieve — which needs an fs seam to reproduce directly.
322
+ mkdirSync(`${storage.indexPath('m')}.tmp`, { recursive: true });
323
+ expect(() => storage.updateIndex('m', before.slice(0, 1))).toThrow();
324
+ expect(storage.readIndex('m')).toEqual(before);
325
+ });
326
+ });
@@ -0,0 +1,289 @@
1
+ /**
2
+ * Reclaiming disk from superseded build revisions.
3
+ *
4
+ * The registry had no delete path at all: `yank` sets a boolean in the index
5
+ * and frees nothing, so every revision ever published was retained forever.
6
+ * A `+N` build revision is auto-assigned on every publish, so the store grew
7
+ * with every release until the disk filled and the release pipeline stopped.
8
+ *
9
+ * ── What the policy keeps, and why it cannot break a fetch ──
10
+ *
11
+ * A version is `X.Y.Z+N`: a released semver plus a build revision. The bloat
12
+ * is the rebuilds, not the releases — a module embedding compiled binaries
13
+ * republished twenty times carries twenty ~73 MB payloads of one release.
14
+ *
15
+ * So the sweep keeps the newest `keepBuildRevisions` of EVERY release and
16
+ * drops the rest. No release ever disappears, which is what makes the safety
17
+ * argument checkable rather than a judgement call:
18
+ *
19
+ * - The only version a celilo client can ask for is the one
20
+ * `RegistryClient.latestVersion` picks — the last non-yanked line in the
21
+ * index. There is no `--version` flag, no lockfile, and `IndexEntry.deps`
22
+ * is always `[]`, so nothing pins a specific revision.
23
+ * - {@link planModuleSweep} retains that line explicitly, on top of the
24
+ * per-release rule, so the property holds even for an index whose lines
25
+ * are out of publish order.
26
+ * - Every release keeps a downloadable payload, so an operator rolling back
27
+ * to an older release still can.
28
+ *
29
+ * ── Ordering, and why it is index-first ──
30
+ *
31
+ * The download route reads the payload file directly and never consults the
32
+ * index. That asymmetry decides the order:
33
+ *
34
+ * - index line first, then payload — the window is "unlisted but still
35
+ * served". Nothing 404s. A crash here leaves an orphan payload, which
36
+ * wastes the disk we are reclaiming but is otherwise correct.
37
+ * - payload first, then index — the window is "listed but 404s", an
38
+ * actively wrong registry. A crash here leaves it that way.
39
+ *
40
+ * So: index first. The orphan a crash leaves behind is then reclaimed by the
41
+ * NEXT run, which is what makes the sweep idempotent and safe to interrupt.
42
+ * The orphan pass runs FIRST for a second reason: it is pure unlink and needs
43
+ * no free space, so it can still make progress on a disk with nothing left to
44
+ * write with — the state that produced this code.
45
+ */
46
+
47
+ import type { IndexEntry, RegistryStorage } from './storage';
48
+
49
+ export interface SweepPolicy {
50
+ /**
51
+ * Build revisions to keep per released semver. Must be >= 1 — keeping zero
52
+ * would delete a whole release, which this policy never does.
53
+ */
54
+ keepBuildRevisions: number;
55
+ }
56
+
57
+ export const DEFAULT_KEEP_BUILD_REVISIONS = 1;
58
+
59
+ export interface ModuleSweepPlan {
60
+ name: string;
61
+ /** Index entries to retain, in their original order. */
62
+ keep: IndexEntry[];
63
+ /** Versions whose index line and payload both go. */
64
+ remove: string[];
65
+ /**
66
+ * Payloads on disk with no index line. Unreachable by search, and the
67
+ * publisher's immutability check still sees them — `packageExists` reads the
68
+ * FILESYSTEM, so an orphan makes its version unpublishable forever with no
69
+ * repair path anywhere else in the product. Safe to delete, and deleting it
70
+ * IS the repair.
71
+ */
72
+ orphans: string[];
73
+ /**
74
+ * Index lines whose payload is missing. REPORTED, NEVER REMOVED.
75
+ *
76
+ * This is the direction the sweep must never create, and the reason it takes
77
+ * the index line before the payload. Removing the line would in fact repair
78
+ * the module — a dangling entry already 404s on download, and dropping it
79
+ * makes `latestVersion` resolve to a version that can actually be served.
80
+ * It is left alone anyway, because a missing payload and an unreadable store
81
+ * are indistinguishable from here: an unmounted volume or a wrong DATA_DIR
82
+ * makes every payload look absent, and an auto-remove would then delete the
83
+ * whole index. Reporting costs nothing and cannot destroy anything.
84
+ */
85
+ dangling: string[];
86
+ /**
87
+ * The module's index lists versions and NOT ONE has a payload on disk. That
88
+ * is a broken or unmounted store rather than N independent losses, so the
89
+ * sweep does nothing at all to this module — see {@link ModuleSweepPlan.dangling}.
90
+ */
91
+ unreadable: boolean;
92
+ }
93
+
94
+ export interface SweepReport {
95
+ modules: Array<{
96
+ name: string;
97
+ removed: string[];
98
+ orphans: string[];
99
+ dangling: string[];
100
+ bytes: number;
101
+ }>;
102
+ removedCount: number;
103
+ orphanCount: number;
104
+ /** Index lines with no payload. Reported for a human; never acted on. */
105
+ danglingCount: number;
106
+ /** Modules skipped entirely because their whole payload set is missing. */
107
+ unreadable: string[];
108
+ reclaimedBytes: number;
109
+ dryRun: boolean;
110
+ }
111
+
112
+ /**
113
+ * Split `X.Y.Z+N` (or `E:X.Y.Z+N`) into the release it belongs to and its
114
+ * build revision. Returns null for anything that does not parse — the caller
115
+ * treats that as "keep", because a version the sweep cannot reason about is
116
+ * not one it should delete.
117
+ */
118
+ export function parseRevision(vers: string): { release: string; revision: number } | null {
119
+ const plus = vers.lastIndexOf('+');
120
+ if (plus <= 0) return null;
121
+ const release = vers.slice(0, plus);
122
+ const revision = Number(vers.slice(plus + 1));
123
+ if (!Number.isInteger(revision) || revision < 0) return null;
124
+ return { release, revision };
125
+ }
126
+
127
+ /**
128
+ * Decide what one module keeps. Pure — no filesystem, no storage.
129
+ *
130
+ * `storedVersions` is what is on DISK, which is deliberately not the same set
131
+ * as `entries`: a previous interrupted run can leave a payload whose index
132
+ * line is already gone, and reclaiming it is the whole reason this takes both.
133
+ */
134
+ export function planModuleSweep(
135
+ name: string,
136
+ entries: IndexEntry[],
137
+ storedVersions: string[],
138
+ policy: SweepPolicy,
139
+ ): ModuleSweepPlan {
140
+ const keepBuildRevisions = Math.max(1, Math.trunc(policy.keepBuildRevisions));
141
+
142
+ const stored = new Set(storedVersions);
143
+ const dangling = entries.filter((e) => !stored.has(e.vers)).map((e) => e.vers);
144
+
145
+ // Every listed version missing its payload is one broken store, not N
146
+ // independent losses. Touch nothing.
147
+ if (entries.length > 0 && dangling.length === entries.length) {
148
+ return { name, keep: entries, remove: [], orphans: [], dangling, unreadable: true };
149
+ }
150
+
151
+ /**
152
+ * Retention is computed over the SERVABLE entries only — the ones whose
153
+ * payload is actually there.
154
+ *
155
+ * Doing it over every index line is subtly wrong in a way that bites exactly
156
+ * when the store is already damaged. If a module's NEWEST line is dangling,
157
+ * that line satisfies both the per-release rule and the latest-line rule, so
158
+ * every older version — the servable ones — becomes "superseded" and is
159
+ * swept. The module is then listed, has one index line, and cannot be
160
+ * downloaded at all. The sweep would not have created the dangling entry,
161
+ * but it would have removed the last thing that still worked.
162
+ */
163
+ const servable = entries.filter((e) => stored.has(e.vers));
164
+
165
+ const retained = new Set<string>();
166
+
167
+ // The newest `keepBuildRevisions` of each release.
168
+ const byRelease = new Map<string, Array<{ vers: string; revision: number }>>();
169
+ for (const entry of servable) {
170
+ const parsed = parseRevision(entry.vers);
171
+ if (!parsed) {
172
+ // Unparseable — never a publish this server accepted, so it is hand-
173
+ // written state. Keep it and move on.
174
+ retained.add(entry.vers);
175
+ continue;
176
+ }
177
+ const group = byRelease.get(parsed.release) ?? [];
178
+ group.push({ vers: entry.vers, revision: parsed.revision });
179
+ byRelease.set(parsed.release, group);
180
+ }
181
+ for (const group of byRelease.values()) {
182
+ group.sort((a, b) => b.revision - a.revision);
183
+ for (const { vers } of group.slice(0, keepBuildRevisions)) retained.add(vers);
184
+ }
185
+
186
+ // What every client actually resolves to, retained explicitly rather than
187
+ // inferred from the rule above. The rule already covers it for an index in
188
+ // publish order; stating it means a disordered or hand-edited index cannot
189
+ // turn into an unfetchable module.
190
+ const lastServable = servable.at(-1);
191
+ if (lastServable) retained.add(lastServable.vers);
192
+ const lastUnyanked = [...servable].reverse().find((e) => !e.yanked);
193
+ if (lastUnyanked) retained.add(lastUnyanked.vers);
194
+
195
+ // Only ever removes something it can see. A dangling line is left in the
196
+ // index untouched and reported instead.
197
+ const remove = servable.filter((e) => !retained.has(e.vers)).map((e) => e.vers);
198
+ const removing = new Set(remove);
199
+ const indexed = new Set(entries.map((e) => e.vers));
200
+
201
+ return {
202
+ name,
203
+ keep: entries.filter((e) => !removing.has(e.vers)),
204
+ remove,
205
+ orphans: storedVersions.filter((v) => !indexed.has(v)),
206
+ dangling,
207
+ unreadable: false,
208
+ };
209
+ }
210
+
211
+ /**
212
+ * Apply one module's plan. Orphans first (needs no free space), then the
213
+ * index rewrite, then the payloads the rewrite just unlisted.
214
+ */
215
+ function applyModuleSweep(
216
+ storage: RegistryStorage,
217
+ plan: ModuleSweepPlan,
218
+ dryRun: boolean,
219
+ ): number {
220
+ let bytes = 0;
221
+
222
+ for (const version of plan.orphans) {
223
+ bytes += storage.packageSize(plan.name, version);
224
+ if (!dryRun) storage.removePackage(plan.name, version);
225
+ }
226
+
227
+ for (const version of plan.remove) {
228
+ bytes += storage.packageSize(plan.name, version);
229
+ }
230
+
231
+ if (plan.remove.length > 0 && !dryRun) {
232
+ storage.updateIndex(plan.name, plan.keep);
233
+ for (const version of plan.remove) storage.removePackage(plan.name, version);
234
+ }
235
+
236
+ return bytes;
237
+ }
238
+
239
+ /** Sweep every module in the store. */
240
+ export function sweep(storage: RegistryStorage, policy: SweepPolicy, dryRun = false): SweepReport {
241
+ const modules: SweepReport['modules'] = [];
242
+ let removedCount = 0;
243
+ let orphanCount = 0;
244
+ let reclaimedBytes = 0;
245
+
246
+ const unreadable: string[] = [];
247
+ let danglingCount = 0;
248
+
249
+ for (const name of storage.storedNames()) {
250
+ const plan = planModuleSweep(
251
+ name,
252
+ storage.readIndex(name),
253
+ storage.storedVersions(name),
254
+ policy,
255
+ );
256
+
257
+ if (plan.unreadable) {
258
+ unreadable.push(name);
259
+ continue;
260
+ }
261
+
262
+ danglingCount += plan.dangling.length;
263
+ if (plan.remove.length === 0 && plan.orphans.length === 0 && plan.dangling.length === 0) {
264
+ continue;
265
+ }
266
+
267
+ const bytes = applyModuleSweep(storage, plan, dryRun);
268
+ modules.push({
269
+ name,
270
+ removed: plan.remove,
271
+ orphans: plan.orphans,
272
+ dangling: plan.dangling,
273
+ bytes,
274
+ });
275
+ removedCount += plan.remove.length;
276
+ orphanCount += plan.orphans.length;
277
+ reclaimedBytes += bytes;
278
+ }
279
+
280
+ return {
281
+ modules,
282
+ removedCount,
283
+ orphanCount,
284
+ danglingCount,
285
+ unreadable,
286
+ reclaimedBytes,
287
+ dryRun,
288
+ };
289
+ }
@@ -22,6 +22,9 @@
22
22
  export const CONSUMER_PATHS = [
23
23
  'packages/e2e/**',
24
24
  'packages/registry-server/**',
25
+ // The smoke test IS this gate's body — the only place publishModule and
26
+ // friends run in the consumer install shape (celilo#1142).
27
+ 'e2e/tests/smoke.test.ts',
25
28
  // The publish driver (moved from the dead apps/celilo/scripts/publish.ts).
26
29
  'apps/celilo/src/cli/commands/publish/**',
27
30
  'scripts/publish.ts',
@@ -50,6 +50,25 @@ const SCANNED_PATHS = ['packages/e2e', 'e2e/tests', 'modules/*/e2e'];
50
50
  */
51
51
  const EXCLUDED = [/^packages\/e2e\/CHANGELOG\.md$/, /address-plan\.test\.ts$/];
52
52
 
53
+ /**
54
+ * A single LINE may exempt itself with `#539-ok: <reason>`.
55
+ *
56
+ * There is one legitimate reason to write a banned prefix: enumerating an
57
+ * RFC 1918 block as a CLASS, which any correct private-address predicate must
58
+ * do. `192.168.` is both the operator's LAN (banned) and 192.168.0.0/16
59
+ * (unavoidable), and RETIRED_PREFIXES cannot tell them apart. The tell that
60
+ * this is the prefix list's blind spot rather than a real judgement: `10.` sits
61
+ * beside `192.168.` in exactly such a predicate and escapes only because `10.`
62
+ * was never added to the list.
63
+ *
64
+ * Deliberately per-line and not per-file. Excluding a file would blind the gate
65
+ * to every OTHER address in it, which is how a suppression becomes the next
66
+ * leak. The reason is required — a marker with nothing after the colon does not
67
+ * match, because an exemption that names only a line number is the next
68
+ * reader's mystery.
69
+ */
70
+ const INLINE_EXEMPTION = /#539-ok:\s*\S/;
71
+
53
72
  function grepRetired(): string[] {
54
73
  const pattern = RETIRED_PREFIXES.map(toPattern).join('|');
55
74
  let out = '';
@@ -66,7 +85,8 @@ function grepRetired(): string[] {
66
85
  return out
67
86
  .split('\n')
68
87
  .filter(Boolean)
69
- .filter((line) => !EXCLUDED.some((re) => re.test(line.split(':')[0])));
88
+ .filter((line) => !EXCLUDED.some((re) => re.test(line.split(':')[0] as string)))
89
+ .filter((line) => !INLINE_EXEMPTION.test(line));
70
90
  }
71
91
 
72
92
  describe('simulated address plan (#539)', () => {
@@ -1,5 +1,5 @@
1
1
  import { type ExecSyncOptions, execSync, spawn } from 'node:child_process';
2
- import { existsSync, readFileSync, writeFileSync } from 'node:fs';
2
+ import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
3
3
  import { tmpdir } from 'node:os';
4
4
  import { basename, join, resolve } from 'node:path';
5
5
  import { parse as parseYaml } from 'yaml';
@@ -10,6 +10,7 @@ import {
10
10
  firewallZoneLegs,
11
11
  generateTestComposeYaml,
12
12
  getAllMachines,
13
+ registryUploadsHostDir,
13
14
  } from './docker-compose-generator';
14
15
  import { explainBuildFailure } from './doctor';
15
16
  import { type ModuleHost, parseModuleHost } from './module-host';
@@ -1005,12 +1006,19 @@ function buildNetworkHandle(
1005
1006
  }
1006
1007
 
1007
1008
  try {
1008
- // Copy directly into the registry container's /uploads dir.
1009
- // The registry rebuilds its index on every request, so no HTTP handshake needed.
1010
- run(
1011
- `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} cp ${JSON.stringify(netappPath)} registry:/uploads/${moduleId}.netapp`,
1012
- { cwd: composeDir, timeout: 120_000 },
1013
- );
1009
+ // Write HOST-side into the dir bound at the registry's /uploads. The
1010
+ // registry rescans that dir on every request, so it needs no HTTP
1011
+ // handshake and no restart.
1012
+ //
1013
+ // Deliberately not a `docker compose cp` into the container: /uploads
1014
+ // is a READ-ONLY bind in a consumer install, so the container write
1015
+ // failed for every npm-installed consumer while passing forever in the
1016
+ // monorepo, where the same path is not a mount at all (celilo#1142).
1017
+ // registryUploadsHostDir() is the same function the compose generator
1018
+ // builds the mount from, so the two cannot drift apart again.
1019
+ const uploadsDir = registryUploadsHostDir();
1020
+ mkdirSync(uploadsDir, { recursive: true });
1021
+ copyFileSync(netappPath, join(uploadsDir, `${moduleId}.netapp`));
1014
1022
  } finally {
1015
1023
  if (cleanup)
1016
1024
  try {
@@ -1,5 +1,5 @@
1
- import { existsSync } from 'node:fs';
2
- import { join } from 'node:path';
1
+ import { existsSync, mkdirSync, readdirSync, rmSync } from 'node:fs';
2
+ import { basename, join } from 'node:path';
3
3
  import { stringify } from 'yaml';
4
4
  import { normalizeObservers, observerEnv, observerPlacement } from './observer';
5
5
  import { PROXMOX_SIM_IP, SIMULATOR_IPS } from './simulator-ips';
@@ -13,32 +13,76 @@ import {
13
13
  zoneIp,
14
14
  } from './types';
15
15
 
16
+ const PACKAGE_DIR = join(import.meta.dir, '..');
17
+
18
+ /**
19
+ * Is this @celilo/e2e running from the monorepo, or from an npm install?
20
+ *
21
+ * `<pkgDir>/../../modules` is `<repo>/modules` in the monorepo and
22
+ * `<consumer>/node_modules/modules` (nonexistent) in a consumer install.
23
+ */
24
+ function isMonorepoInstall(): boolean {
25
+ return existsSync(join(PACKAGE_DIR, '..', '..', 'modules'));
26
+ }
27
+
16
28
  /**
17
- * Volumes for the registry container. Picks between source-bootstrap and
18
- * uploads-bootstrap based on where the @celilo/e2e package lives:
29
+ * The host dir bound read-only at the registry's `/uploads` — and therefore
30
+ * the dir `publishModule` writes a `.netapp` into. The registry rescans it on
31
+ * every request, so a host-side write is picked up with no container write and
32
+ * no HTTP handshake.
33
+ *
34
+ * Both install shapes bind one. They differ only in WHICH, because they differ
35
+ * in where standard modules come from:
36
+ *
37
+ * - **Monorepo dev** — modules are served live from `/modules`
38
+ * (`../../modules`, BOOTSTRAP_MODULES_DIR mode), so edits to a module's
39
+ * source flow through without a manual repack. `/uploads` is therefore a
40
+ * dedicated drop-zone, emptied on every shared-infra start (see
41
+ * `prepareRegistryDropZone`) — a leftover `.netapp` would shadow live source.
19
42
  *
20
- * - **Monorepo dev** — `../../modules` from the compose file (which lives
21
- * in `packages/e2e/`) resolves to `<repo>/modules`. The registry runs
22
- * in BOOTSTRAP_MODULES_DIR mode and packs source modules on the fly,
23
- * so edits to a module's source flow through without manual repack.
43
+ * - **npm-installed consumer** (lunacycle and friends) — `../../modules` does
44
+ * not exist, so there is no source to serve. `/uploads` IS the netapp cache
45
+ * `cele2e build-infra` fills from the public registry (see
46
+ * stageNetappsFromRegistry in cli/build.ts), and a published module lands
47
+ * beside the standard ones. Consumers vendor nothing.
24
48
  *
25
- * - **npm-installed consumer** (lunacycle and friends) — `../../modules`
26
- * would resolve to `<consumer>/node_modules/modules`, which doesn't
27
- * exist. Docker fails the mount with "permission denied" trying to
28
- * create that path. Instead we bind the local `./netapps/` dir to
29
- * `/uploads`, and the registry's scanUploadsDir picks up the .netapp
30
- * files. That dir is populated at `cele2e build-infra` time by fetching
31
- * the standard-module netapps from the public registry (see
32
- * stageNetappsFromRegistry in cli/build.ts) — consumers vendor nothing.
49
+ * Read this through `registryUploadsHostDir()` rather than re-deriving it.
50
+ * publishModule used to `docker cp` into the container's `/uploads` instead,
51
+ * which works only where that path is NOT a mount — so it passed forever in
52
+ * the monorepo and failed for every consumer against the read-only bind
53
+ * (celilo#1142). One function now decides the mount and the write target
54
+ * together, so they cannot disagree again.
55
+ */
56
+ export function registryUploadsHostDir(): string {
57
+ return join(PACKAGE_DIR, isMonorepoInstall() ? 'uploads' : 'netapps');
58
+ }
59
+
60
+ /**
61
+ * Make the drop-zone ready for a shared-infra bring-up.
33
62
  *
34
- * Detection: check whether the would-be monorepo modules dir is real.
35
- * `import.meta.dir` is `<pkgDir>/src/`, so `../../../modules` from there
36
- * is `<repo>/modules` in the monorepo and `<consumer>/node_modules/modules`
37
- * (nonexistent) in a consumer install.
63
+ * Creates it: Docker silently creates a MISSING bind source as an empty
64
+ * root-owned directory, which the host then cannot write into.
65
+ *
66
+ * Empties it, but only in the monorepo, where `/uploads` is dedicated — a
67
+ * `.netapp` left by a previous run would shadow the live module source it was
68
+ * packed from. In a consumer install the same dir IS the build-infra netapp
69
+ * cache, so clearing it would delete every standard module. (Before
70
+ * celilo#1142 published netapps lived in the registry's container layer and
71
+ * died with the container, so starting empty is the behaviour being kept, not
72
+ * a new one.)
38
73
  */
74
+ export function prepareRegistryDropZone(): void {
75
+ const dir = registryUploadsHostDir();
76
+ mkdirSync(dir, { recursive: true });
77
+ if (!isMonorepoInstall()) return;
78
+ for (const file of readdirSync(dir)) {
79
+ if (file.endsWith('.netapp')) rmSync(join(dir, file), { force: true });
80
+ }
81
+ }
82
+
39
83
  function getRegistryVolumes(): string[] {
40
- const monorepoModules = join(import.meta.dir, '..', '..', '..', 'modules');
41
- return existsSync(monorepoModules) ? ['../../modules:/modules:ro'] : ['./netapps:/uploads:ro'];
84
+ const uploads = `./${basename(registryUploadsHostDir())}:/uploads:ro`;
85
+ return isMonorepoInstall() ? ['../../modules:/modules:ro', uploads] : [uploads];
42
86
  }
43
87
 
44
88
  const NETWORK_DRIVER_OPTS = {
@@ -17,6 +17,7 @@ import {
17
17
  SHARED_NETWORKS,
18
18
  SHARED_PROJECT_NAME,
19
19
  generateSharedInfraYaml,
20
+ prepareRegistryDropZone,
20
21
  } from './docker-compose-generator';
21
22
  import { ensureRegistryServerBundle, ensureTerraformFakeBundle } from './registry-bundle';
22
23
 
@@ -161,6 +162,11 @@ export async function ensureSharedInfra(): Promise<void> {
161
162
  const yaml = generateSharedInfraYaml();
162
163
  writeFileSync(join(e2eDir, SHARED_COMPOSE_FILE), yaml);
163
164
 
165
+ // The registry's /uploads bind source must exist before `up` (a missing
166
+ // bind source becomes an empty root-owned dir), and in the monorepo it
167
+ // starts empty so no stale .netapp shadows live module source.
168
+ prepareRegistryDropZone();
169
+
164
170
  // Seed the live DNS zone files from their templates BEFORE the compose
165
171
  // mounts them. config/dns/{iamtheinternet.org,example.net}.zone are
166
172
  // gitignored runtime state (scrubDnsZones rewrites them per-test); on a