@celilo/e2e 0.18.0 → 0.19.1

package/config/routing/fw-isp-routes.sh

@@ -24,9 +24,19 @@ iptables -P FORWARD ACCEPT
 # DNS
 echo "nameserver 203.0.113.1" > /etc/resolv.conf
 
-# Start dnsmasq DHCP server on internal network
-dnsmasq --conf-dir=/etc/dnsmasq.d --keep-in-foreground --log-facility=- &
-echo "dnsmasq DHCP server started"
+# Start dnsmasq DHCP server on internal network.
+#
+# Skippable, because celilo can serve DHCP itself (`modules/dnsmasq-dhcp`) and
+# two DHCP servers on one broadcast domain race: whichever answers a DISCOVER
+# first wins, and a test asserting which one served a lease would be asserting
+# a coin toss. An operator moving DHCP to celilo turns the router's off, so a
+# suite that tests celilo's DHCP models that by setting ROUTER_DHCP=off.
+if [ "${ROUTER_DHCP:-on}" = "off" ]; then
+  echo "dnsmasq DHCP server NOT started (ROUTER_DHCP=off; celilo serves DHCP here)"
+else
+  dnsmasq --conf-dir=/etc/dnsmasq.d --keep-in-foreground --log-facility=- &
+  echo "dnsmasq DHCP server started"
+fi
 
 # Start greenwave simulator
 if [ -f /simulator/server.ts ]; then

package/docker/Dockerfile.management

@@ -35,6 +35,12 @@ RUN apt-get update && apt-get install -y \
     procps \
     zsh \
     gnupg \
+    # The hook jail (openspec/changes/hook-process-boundary, stage 2) runs every
+    # hook under bubblewrap. Without it the management box runs hooks unjailed,
+    # which is a valid mode but not the one the e2e suite is there to exercise.
+    # The container needs security_opt relaxations to let it build a namespace —
+    # see the measured ladder in docker-compose-generator.ts.
+    bubblewrap \
     && rm -rf /var/lib/apt/lists/*
 
 # --- Terraform, plus an OFFLINE provider mirror -------------------------------

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.18.0",
+  "version": "0.19.1",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",
@@ -18,7 +18,7 @@
     "celilo-e2e-infra": "./bin/e2e-infra"
   },
   "scripts": {
-    "test": "bun test --timeout 30000 tests/",
+    "test": "bun test --timeout 30000 ./tests/ ./src/ ./npm-registry-server/ ./simulators/",
     "test:completion": "bun test tests/completion",
     "test:integration": "bun test tests-integration/"
   },
@@ -37,7 +37,7 @@
     "README.md"
   ],
   "dependencies": {
-    "@celilo/capabilities": "^3.0.0",
+    "@celilo/capabilities": "^3.3.0",
     "@celilo/cli-display": "^0.2.0",
     "@celilo/event-bus": "^0.6.0",
     "@celilo/terraform-fake": "^0.3.1",

package/registry-server/src/bootstrap.ts

@@ -33,6 +33,8 @@ export interface BootstrapEntry {
    * apps/celilo/designs/REGISTRY_BROWSE_UI.md (Phase 2 step 0).
    */
   description?: string;
+  /** Manifest's `icon` field, when present. Served like `description`. */
+  icon?: string;
 }
 
 /**
@@ -60,11 +62,14 @@ export function scanBootstrapDir(bootstrapDir: string): Map<string, BootstrapEnt
       const name = idMatch?.[1]?.trim() ?? entry.name;
       const version = versionMatch?.[1]?.trim() ?? '0.0.0';
       const description = descriptionMatch?.[1]?.trim().replace(/^['"]|['"]$/g, '');
+      const iconMatch = yaml.match(/^icon:\s*(.+?)\s*$/m);
+      const icon = iconMatch?.[1]?.trim().replace(/^['"]|['"]$/g, '');
       entries.set(name, {
         name,
         version,
         sourceDir: join(bootstrapDir, entry.name),
         description,
+        icon,
       });
     } catch {
       // skip malformed manifests — matches e2e server behavior

package/registry-server/src/server.test.ts

@@ -217,6 +217,64 @@ describe('GET /api/v1/modules (search)', () => {
     expect(found?.description).toBe('');
   });
 
+  test('icon survives publish, the sparse index and a search read', async () => {
+    // openspec/changes/module-icons D4 — the icon rides the same path
+    // `description` does: publish metadata → index entry → browse endpoints.
+    expect((await publish('homebridge', '1.0.0+1', 'valid-token', 'Bridge', '\u22c8')).status).toBe(
+      200,
+    );
+
+    const index = await fetch(`${baseUrl}/index/ho/me/homebridge`);
+    expect(index.status).toBe(200);
+    const line = JSON.parse((await index.text()).trim()) as { icon?: string };
+    expect(line.icon).toBe('\u22c8');
+
+    const res = await fetch(`${baseUrl}/api/v1/modules`);
+    const body = (await res.json()) as { modules: Array<{ name: string; icon?: string }> };
+    expect(body.modules.find((m) => m.name === 'homebridge')?.icon).toBe('\u22c8');
+
+    const detail = await fetch(`${baseUrl}/api/v1/modules/homebridge`);
+    expect(((await detail.json()) as { icon?: string }).icon).toBe('\u22c8');
+  });
+
+  test('an entry published without an icon reads back as undefined, not a throw', async () => {
+    // The old-entry case D4 depends on: index lines written before the field
+    // existed carry no `icon`, and a consumer falls back rather than failing.
+    //
+    // This test passes with or without the icon feature, so it is NOT a gate on
+    // it — verified by running it against the pre-#1147 tree. It is here to
+    // catch a future change that makes a missing `icon` throw or coerce to '',
+    // which is what D4's fallback chain would break on. Do not count it as
+    // coverage of the capture path; that is the test above.
+    expect((await publish('homebridge', '1.0.0+1', 'valid-token')).status).toBe(200);
+
+    const res = await fetch(`${baseUrl}/api/v1/modules`);
+    const body = (await res.json()) as { modules: Array<{ name: string; icon?: string }> };
+    const found = body.modules.find((m) => m.name === 'homebridge');
+    expect(found).toBeTruthy();
+    expect(found?.icon).toBeUndefined();
+  });
+
+  test('a multi-character icon is dropped while a valid one beside it is kept', async () => {
+    // Publish metadata is arbitrary client JSON. The CLI refines the value at
+    // manifest-parse time; the server keeps the index from carrying a string
+    // that would blow out a fixed-width slot on the browse page.
+    //
+    // Both halves are asserted deliberately. Checking only that the bad value
+    // is absent would pass just as well if the capture path were deleted
+    // outright — the guard and the capture would cancel out and the test would
+    // stay green having lost the feature. The kept glyph is what stops that.
+    expect(
+      (await publish('homebridge', '1.0.0+1', 'valid-token', 'Bridge', 'not-a-glyph')).status,
+    ).toBe(200);
+    expect((await publish('caddy', '1.0.0+1', 'valid-token', 'Proxy', '\u25cd')).status).toBe(200);
+
+    const res = await fetch(`${baseUrl}/api/v1/modules`);
+    const body = (await res.json()) as { modules: Array<{ name: string; icon?: string }> };
+    expect(body.modules.find((m) => m.name === 'homebridge')?.icon).toBeUndefined();
+    expect(body.modules.find((m) => m.name === 'caddy')?.icon).toBe('\u25cd');
+  });
+
   test('total_downloads reflects actual download count, sort=downloads orders by it', async () => {
     // Publish two modules, hit one's download endpoint several times.
     // The search response should show those counts and sort=downloads
@@ -490,9 +548,15 @@ async function publish(
   vers: string,
   token: string,
   description?: string,
+  icon?: string,
 ): Promise<Response> {
   const meta = Buffer.from(
-    JSON.stringify({ name, vers, ...(description ? { description } : {}) }),
+    JSON.stringify({
+      name,
+      vers,
+      ...(description ? { description } : {}),
+      ...(icon ? { icon } : {}),
+    }),
     'utf-8',
   );
   const file = Buffer.from('fake netapp bytes');
@@ -769,3 +833,121 @@ describe('module-owner authorization (ce-1ch)', () => {
     expect(res.status).toBe(200);
   });
 });
+
+// ── sweep (reclaiming disk from superseded build revisions) ──────────────────
+
+describe('POST /api/v1/modules/sweep', () => {
+  function requestSweep(token: string, body: unknown = {}): Promise<Response> {
+    return fetch(`${baseUrl}/api/v1/modules/sweep`, {
+      method: 'POST',
+      headers: token ? { Authorization: token, 'Content-Type': 'application/json' } : {},
+      body: JSON.stringify(body),
+    });
+  }
+
+  test('no token is 401 — a sweep deletes, so it is admin-only', async () => {
+    const res = await requestSweep('');
+    expect(res.status).toBe(401);
+  });
+
+  test('a scoped (non-admin) token is 401', async () => {
+    const mint = await fetch(`${baseUrl}/api/v1/modules/tokens/mint`, {
+      method: 'POST',
+      headers: { Authorization: 'valid-token', 'Content-Type': 'application/json' },
+      body: JSON.stringify({ repo: 'celilo/homebridge', scope: 'homebridge' }),
+    });
+    const { token } = (await mint.json()) as { token: string };
+
+    const res = await requestSweep(token);
+    expect(res.status).toBe(401);
+  });
+
+  test('removes superseded revisions and leaves the download path working', async () => {
+    for (const rev of [1, 2, 3]) await publish('homebridge', `1.0.0+${rev}`, 'valid-token');
+
+    const res = await requestSweep('valid-token');
+    expect(res.status).toBe(200);
+    const body = (await res.json()) as { ok: boolean; removedCount: number };
+    expect(body.ok).toBe(true);
+    expect(body.removedCount).toBe(2);
+
+    // The surviving revision downloads; a swept one is gone from BOTH the
+    // index and the store, so nothing advertises a 404.
+    expect((await fetch(`${baseUrl}/api/v1/modules/homebridge/1.0.0+3/download`)).status).toBe(200);
+    expect((await fetch(`${baseUrl}/api/v1/modules/homebridge/1.0.0+1/download`)).status).toBe(404);
+
+    const index = await fetch(`${baseUrl}/index/ho/me/homebridge`);
+    const versions = (await index.text())
+      .split('\n')
+      .filter(Boolean)
+      .map((line) => (JSON.parse(line) as { vers: string }).vers);
+    expect(versions).toEqual(['1.0.0+3']);
+  });
+
+  test('dry_run reports the plan without deleting anything', async () => {
+    for (const rev of [1, 2]) await publish('homebridge', `1.0.0+${rev}`, 'valid-token');
+
+    const res = await requestSweep('valid-token', { dry_run: true });
+    const body = (await res.json()) as { removedCount: number; dryRun: boolean };
+    expect(body.removedCount).toBe(1);
+    expect(body.dryRun).toBe(true);
+    expect((await fetch(`${baseUrl}/api/v1/modules/homebridge/1.0.0+1/download`)).status).toBe(200);
+  });
+
+  test('rejects a keep_build_revisions that would delete a whole release', async () => {
+    const res = await requestSweep('valid-token', { keep_build_revisions: 0 });
+    expect(res.status).toBe(400);
+  });
+
+  test('honours a larger keep_build_revisions', async () => {
+    for (const rev of [1, 2, 3, 4]) await publish('homebridge', `1.0.0+${rev}`, 'valid-token');
+    const res = await requestSweep('valid-token', { keep_build_revisions: 2 });
+    const body = (await res.json()) as { removedCount: number };
+    expect(body.removedCount).toBe(2);
+  });
+
+  test('coexists with a module actually named "sweep"', async () => {
+    await publish('sweep', '1.0.0+1', 'valid-token');
+    const res = await requestSweep('valid-token');
+    expect(res.status).toBe(200);
+    // …and the module is still reachable by its own GET route.
+    expect((await fetch(`${baseUrl}/api/v1/modules/sweep`)).status).toBe(200);
+  });
+});
+
+describe('POST /api/v1/modules/sweep — a damaged store', () => {
+  function requestSweep(token: string, body: unknown = {}): Promise<Response> {
+    return fetch(`${baseUrl}/api/v1/modules/sweep`, {
+      method: 'POST',
+      headers: { Authorization: token, 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+  }
+
+  test('reclaims a half-written publish, freeing the version to be published again', async () => {
+    // handlePublish stores the package and appends the index with no rollback
+    // between them, so an ENOSPC or EACCES in the gap leaves an orphan. The
+    // immutability check reads the filesystem, so that version then cannot be
+    // published at all — this is the only repair path in the product.
+    await publish('homebridge', '1.0.0+1', 'valid-token');
+    const rmIndex = await fetch(`${baseUrl}/api/v1/modules/homebridge/1.0.0+1/yank`, {
+      method: 'DELETE',
+      headers: { Authorization: 'valid-token' },
+    });
+    expect(rmIndex.status).toBe(200);
+
+    // Simulate the crash: index line gone, payload left behind.
+    rmSync(join(dataDir, 'index'), { recursive: true, force: true });
+
+    const blocked = await publish('homebridge', '1.0.0+1', 'valid-token');
+    expect(blocked.status).toBe(409);
+
+    const swept = await requestSweep('valid-token');
+    const body = (await swept.json()) as { orphanCount: number };
+    expect(body.orphanCount).toBe(1);
+
+    // The version is publishable again.
+    const retry = await publish('homebridge', '1.0.0+1', 'valid-token');
+    expect(retry.status).toBe(200);
+  });
+});

package/registry-server/src/server.ts

@@ -10,6 +10,7 @@
  *   PUT  /api/v1/modules/new                             → publish (token auth)
  *   DELETE /api/v1/modules/{name}/{ver}/yank             → yank
  *   PUT  /api/v1/modules/{name}/{ver}/unyank             → unyank
+ *   POST /api/v1/modules/sweep                           → reclaim old revisions (admin)
  *
  * All routes can be prefixed with PATH_PREFIX (e.g. "/registry").
  *
@@ -34,6 +35,7 @@ import { ModuleOwnerStore, fileModuleOwnerPersistence } from './module-owner-sto
 import { type RateLimiter, clientIp, createRateLimiter } from './rate-limit';
 import { ScopedTokenStore, fileScopedTokenPersistence } from './scoped-token-store';
 import { type IndexEntry, RegistryStorage } from './storage';
+import { DEFAULT_KEEP_BUILD_REVISIONS, sweep } from './sweep';
 import { isValidName, isValidVersion, validateNameAndVersion } from './validation';
 
 /** Max total publish body size. Real modules are a few MB; 100MB is generous. */
@@ -313,6 +315,12 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
     // not provided we leave it undefined; the search endpoint falls
     // back to bootstrap data or empty.
     const description = typeof meta.description === 'string' ? meta.description : undefined;
+    // Icon likewise, but capped at one Unicode scalar. The CLI already refines
+    // it at manifest-parse time; this second check is here because the publish
+    // metadata is arbitrary client JSON and the value lands in a fixed-width
+    // slot on the browse page.
+    const icon =
+      typeof meta.icon === 'string' && [...meta.icon].length === 1 ? meta.icon : undefined;
 
     const validation = validateNameAndVersion(name, vers);
     if (!validation.ok) return err(validation.message);
@@ -339,12 +347,63 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
       cksum: `sha256:${cksum}`,
       yanked: false,
       description,
+      icon,
     });
 
     console.log(`[registry] published ${name}@${vers} (${fileLen} bytes, sha256:${cksum})`);
     return Response.json({ ok: true, name, vers });
   }
 
+  /**
+   * Reclaim disk from superseded build revisions (admin-only).
+   *
+   * The counterpart yank never was: yanking flips a boolean and frees nothing,
+   * so before this endpoint existed every revision ever published was retained
+   * forever and the store grew without bound until the disk filled.
+   *
+   * Runs IN the process that owns the store, so it cannot race a concurrent
+   * publish's index append. `dry_run` reports the same plan without touching
+   * anything — the safe way to see what a policy would do on a live store.
+   */
+  async function handleSweep(req: Request): Promise<Response> {
+    if (!(await authorizeAdminReq(req))) return unauthorized();
+
+    let body: { keep_build_revisions?: unknown; dry_run?: unknown } = {};
+    try {
+      const text = await req.text();
+      if (text.trim()) body = JSON.parse(text) as typeof body;
+    } catch {
+      return err('Invalid JSON body');
+    }
+
+    const requested = body.keep_build_revisions;
+    if (requested !== undefined && (!Number.isInteger(requested) || (requested as number) < 1)) {
+      return err('keep_build_revisions must be an integer >= 1');
+    }
+    const keepBuildRevisions = (requested as number | undefined) ?? DEFAULT_KEEP_BUILD_REVISIONS;
+    const dryRun = body.dry_run === true;
+
+    const report = sweep(storage, { keepBuildRevisions }, dryRun);
+    console.log(
+      `[registry] sweep${dryRun ? ' (dry run)' : ''} keep=${keepBuildRevisions}: ` +
+        `${report.removedCount} superseded + ${report.orphanCount} orphaned revision(s), ` +
+        `${Math.round(report.reclaimedBytes / 1024 / 1024)} MB`,
+    );
+    // Both of these mean the store is damaged, not merely untidy, and the sweep
+    // deliberately did not act on either. Say so where an operator will see it.
+    if (report.danglingCount > 0) {
+      console.warn(
+        `[registry] ${report.danglingCount} index line(s) have no package file — left in place; download 404s until republished`,
+      );
+    }
+    if (report.unreadable.length > 0) {
+      console.warn(
+        `[registry] skipped entirely (no package files at all — check the mount and DATA_DIR): ${report.unreadable.join(', ')}`,
+      );
+    }
+    return Response.json({ ok: true, ...report });
+  }
+
   function setYanked(name: string, version: string, yanked: boolean): Response {
     const entries = storage.readIndex(name);
     const entry = entries.find((e) => e.vers === version);
@@ -505,10 +564,15 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
           //      description capture and haven't been republished).
           const bootstrapEntry = bootstrap.get(name);
           const description = latest?.description ?? bootstrapEntry?.description ?? '';
+          // Icon follows the same order, but has no empty-string tier: absent
+          // means "this module declared none", and the consumer resolves it
+          // from its own table or a placeholder (module-icons D5).
+          const icon = latest?.icon ?? bootstrapEntry?.icon;
           return {
             name,
             max_version: latest?.vers ?? '0.0.0',
             description,
+            icon,
             total_downloads: storage.getDownloads(name),
           };
         });
@@ -551,6 +615,15 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
         return handleRevokeToken(req);
       }
 
+      // Reclaim disk from superseded build revisions (admin-only), grouped
+      // with the other admin endpoints. A module NAMED "sweep" is fine here:
+      // its metadata route is a GET, so the two never compete.
+      if (method === 'POST' && suffix === `${apiBase}/sweep`) {
+        const rl = rateLimitOrNull(req, srv);
+        if (rl) return rl;
+        return handleSweep(req);
+      }
+
       // Module-owner table management (admin-only — ce-1ch). Matched before the
       // generic `${apiBase}/{name}` handlers so a name of "owners" can't shadow
       // them.
@@ -588,9 +661,11 @@ export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve
           const latest = entries.filter((v) => !v.yanked).at(-1) ?? entries.at(-1);
           const bootstrapEntry = resolveBootstrap().get(name);
           const description = latest?.description ?? bootstrapEntry?.description ?? '';
+          const icon = latest?.icon ?? bootstrapEntry?.icon;
           return Response.json({
             name,
             description,
+            icon,
             total_downloads: storage.getDownloads(name),
             versions: entries.map((v) => ({
               num: v.vers,

package/registry-server/src/storage.ts

@@ -8,7 +8,16 @@
  */
 
 import { createHash } from 'node:crypto';
-import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  renameSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs';
 import { dirname, join } from 'node:path';
 
 export interface IndexEntry {
@@ -25,6 +34,14 @@ export interface IndexEntry {
    * step 0). Old index files without this field still parse cleanly.
    */
   description?: string;
+  /**
+   * One monochrome glyph from the module's `manifest.yml#icon`, captured at
+   * publish time. Optional for the same reason `description` is: entries
+   * written before the field existed have none, and a consumer reading one
+   * without it falls back to its own table rather than failing
+   * (openspec/changes/module-icons, D4).
+   */
+  icon?: string;
 }
 
 export interface ModuleVersion {
@@ -79,10 +96,28 @@ export class RegistryStorage {
     writeFileSync(path, `${JSON.stringify(entry)}\n`, { flag: 'a' });
   }
 
+  /**
+   * Replace a module's whole index file.
+   *
+   * Writes a sibling temp file and renames it over the original, which is
+   * atomic within a directory. A plain truncating write has two failure modes
+   * this avoids, and the sweep meets both: a crash mid-write leaves a half-
+   * index, and an ENOSPC leaves an EMPTY one — losing every version of the
+   * module. The sweep exists precisely because the disk filled up, so "the
+   * write fails for want of space" is its expected environment, not a corner.
+   * On failure the original file is untouched.
+   */
   updateIndex(name: string, entries: IndexEntry[]): void {
     const path = this.indexPath(name);
     mkdirSync(dirname(path), { recursive: true });
-    writeFileSync(path, `${entries.map((e) => JSON.stringify(e)).join('\n')}\n`);
+    const tmp = `${path}.tmp`;
+    try {
+      writeFileSync(tmp, `${entries.map((e) => JSON.stringify(e)).join('\n')}\n`);
+      renameSync(tmp, path);
+    } catch (err) {
+      rmSync(tmp, { force: true, recursive: true });
+      throw err;
+    }
   }
 
   storePackage(name: string, version: string, data: Buffer): string {
@@ -96,6 +131,45 @@ export class RegistryStorage {
     return existsSync(this.packagePath(name, version));
   }
 
+  /** Bytes on disk for one version's .netapp, or 0 when it is already gone. */
+  packageSize(name: string, version: string): number {
+    const path = this.packagePath(name, version);
+    if (!existsSync(path)) return 0;
+    return statSync(path).size;
+  }
+
+  /**
+   * Delete one version's payload directory (`modules/{name}/{version}/`).
+   * Idempotent: removing a version that is already gone is a no-op, so an
+   * interrupted sweep is safe to re-run.
+   */
+  removePackage(name: string, version: string): void {
+    rmSync(dirname(this.packagePath(name, version)), { recursive: true, force: true });
+  }
+
+  /**
+   * Every version with a payload directory on disk, whether or not the index
+   * still lists it. The sweep needs the on-DISK set, not the indexed one:
+   * removing an index line is what makes a version unreachable, and the file
+   * it leaves behind is exactly what has to be reclaimed afterwards.
+   */
+  storedVersions(name: string): string[] {
+    const dir = join(this.dataDir, 'modules', name);
+    if (!existsSync(dir)) return [];
+    return readdirSync(dir, { withFileTypes: true })
+      .filter((d) => d.isDirectory())
+      .map((d) => d.name);
+  }
+
+  /** Module names with a payload directory, including any absent from the index. */
+  storedNames(): string[] {
+    const dir = join(this.dataDir, 'modules');
+    if (!existsSync(dir)) return [];
+    return readdirSync(dir, { withFileTypes: true })
+      .filter((d) => d.isDirectory())
+      .map((d) => d.name);
+  }
+
   readPackage(name: string, version: string): Buffer | null {
     const path = this.packagePath(name, version);
     return existsSync(path) ? readFileSync(path) : null;