@celilo/e2e 0.9.3 → 0.10.0

package/registry-server/src/storage.test.ts

@@ -0,0 +1,148 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { existsSync, mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { IndexEntry } from './storage';
+import { RegistryStorage } from './storage';
+
+let dataDir: string;
+let storage: RegistryStorage;
+
+beforeEach(() => {
+  dataDir = mkdtempSync(join(tmpdir(), 'celilo-registry-test-'));
+  storage = new RegistryStorage(dataDir);
+});
+
+afterEach(() => {
+  rmSync(dataDir, { recursive: true, force: true });
+});
+
+// ── indexPath ─────────────────────────────────────────────────────────────────
+
+describe('RegistryStorage.indexPath', () => {
+  test('1-char name → index/1/{name}', () => {
+    expect(storage.indexPath('a')).toBe(join(dataDir, 'index', '1', 'a'));
+  });
+
+  test('2-char name → index/2/{name}', () => {
+    expect(storage.indexPath('ab')).toBe(join(dataDir, 'index', '2', 'ab'));
+  });
+
+  test('3-char name → index/3/{first2}/{name}', () => {
+    expect(storage.indexPath('abc')).toBe(join(dataDir, 'index', '3', 'ab', 'abc'));
+  });
+
+  test('4-char name → index/{01}/{23}/{name}', () => {
+    expect(storage.indexPath('abcd')).toBe(join(dataDir, 'index', 'ab', 'cd', 'abcd'));
+  });
+
+  test('long name uses first 4 chars for directories', () => {
+    expect(storage.indexPath('homebridge')).toBe(
+      join(dataDir, 'index', 'ho', 'me', 'homebridge'),
+    );
+  });
+
+  test('real module names produce expected paths', () => {
+    expect(storage.indexPath('caddy')).toBe(join(dataDir, 'index', 'ca', 'dd', 'caddy'));
+    expect(storage.indexPath('celilo-registry')).toBe(
+      join(dataDir, 'index', 'ce', 'li', 'celilo-registry'),
+    );
+  });
+});
+
+// ── readIndex / appendIndex ───────────────────────────────────────────────────
+
+describe('RegistryStorage index read/write', () => {
+  function entry(vers: string, yanked = false): IndexEntry {
+    return { name: 'homebridge', vers, deps: [], cksum: 'abc123', yanked };
+  }
+
+  test('readIndex returns empty array for unknown module', () => {
+    expect(storage.readIndex('nonexistent')).toEqual([]);
+  });
+
+  test('appendIndex then readIndex round-trips entries', () => {
+    storage.appendIndex(entry('1.0.0+1'));
+    storage.appendIndex(entry('1.0.0+2'));
+    const result = storage.readIndex('homebridge');
+    expect(result).toHaveLength(2);
+    expect(result[0].vers).toBe('1.0.0+1');
+    expect(result[1].vers).toBe('1.0.0+2');
+  });
+
+  test('updateIndex replaces all entries', () => {
+    storage.appendIndex(entry('1.0.0+1'));
+    storage.appendIndex(entry('1.0.0+2'));
+    storage.updateIndex('homebridge', [entry('1.0.0+2', true), entry('1.0.0+3')]);
+    const result = storage.readIndex('homebridge');
+    expect(result).toHaveLength(2);
+    expect(result[0].yanked).toBe(true);
+    expect(result[1].vers).toBe('1.0.0+3');
+  });
+
+  test('appendIndex creates parent directories', () => {
+    storage.appendIndex(entry('1.0.0+1'));
+    expect(existsSync(storage.indexPath('homebridge'))).toBe(true);
+  });
+});
+
+// ── storePackage / readPackage ────────────────────────────────────────────────
+
+describe('RegistryStorage package store/read', () => {
+  test('storePackage returns sha256 hex', () => {
+    const data = Buffer.from('fake netapp bytes');
+    const cksum = storage.storePackage('homebridge', '1.0.0+1', data);
+    expect(cksum).toMatch(/^[0-9a-f]{64}$/);
+  });
+
+  test('packageExists returns false before store, true after', () => {
+    expect(storage.packageExists('homebridge', '1.0.0+1')).toBe(false);
+    storage.storePackage('homebridge', '1.0.0+1', Buffer.from('data'));
+    expect(storage.packageExists('homebridge', '1.0.0+1')).toBe(true);
+  });
+
+  test('readPackage returns stored bytes', () => {
+    const data = Buffer.from('fake netapp bytes');
+    storage.storePackage('homebridge', '1.0.0+1', data);
+    const result = storage.readPackage('homebridge', '1.0.0+1');
+    expect(result?.toString()).toBe('fake netapp bytes');
+  });
+
+  test('readPackage returns null for missing version', () => {
+    expect(storage.readPackage('homebridge', '9.9.9+1')).toBeNull();
+  });
+
+  test('sha256 is consistent with stored data', () => {
+    const data = Buffer.from('deterministic content');
+    const cksum = storage.storePackage('homebridge', '1.0.0+1', data);
+    expect(cksum).toBe(storage.sha256(data));
+  });
+});
+
+// ── listModules / getModule ───────────────────────────────────────────────────
+
+describe('RegistryStorage listModules / getModule', () => {
+  test('listModules returns empty when no modules stored', () => {
+    expect(storage.listModules()).toEqual([]);
+  });
+
+  test('listModules returns stored modules with their index entries', () => {
+    storage.storePackage('homebridge', '1.0.0+1', Buffer.from('a'));
+    storage.appendIndex({ name: 'homebridge', vers: '1.0.0+1', deps: [], cksum: 'x', yanked: false });
+    const list = storage.listModules();
+    expect(list).toHaveLength(1);
+    expect(list[0].name).toBe('homebridge');
+    expect(list[0].versions).toHaveLength(1);
+  });
+
+  test('getModule returns null for unknown module', () => {
+    expect(storage.getModule('nonexistent')).toBeNull();
+  });
+
+  test('getModule returns module with versions when index has entries', () => {
+    storage.appendIndex({ name: 'caddy', vers: '2.0.0+1', deps: [], cksum: 'y', yanked: false });
+    const mod = storage.getModule('caddy');
+    expect(mod?.name).toBe('caddy');
+    expect(mod?.versions).toHaveLength(1);
+  });
+});

package/registry-server/src/storage.ts

@@ -0,0 +1,150 @@
+/**
+ * File-backed storage for the registry.
+ *
+ * Layout under DATA_DIR:
+ *   modules/{name}/{version}/{name}-{version}.netapp   — package files
+ *   index/{1,2,ab/cd}/{name}                            — sparse index files (one JSON line per version)
+ *   downloads/{name}                                    — per-module download counter (single integer)
+ */
+
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { createHash } from 'node:crypto';
+import { dirname, join } from 'node:path';
+
+export interface IndexEntry {
+  name: string;
+  vers: string;
+  deps: string[];
+  cksum: string;
+  yanked: boolean;
+  /**
+   * One-line summary from the module's `manifest.yml#description`,
+   * captured at publish time. Optional because pre-Phase-2 published
+   * modules have no description recorded and need a republish to
+   * backfill — see apps/celilo/designs/REGISTRY_BROWSE_UI.md (Phase 2
+   * step 0). Old index files without this field still parse cleanly.
+   */
+  description?: string;
+}
+
+export interface ModuleVersion {
+  name: string;
+  vers: string;
+  cksum: string;
+  yanked: boolean;
+  uploadedAt: string;
+}
+
+export class RegistryStorage {
+  private readonly dataDir: string;
+
+  constructor(dataDir: string) {
+    this.dataDir = dataDir;
+    mkdirSync(join(dataDir, 'modules'), { recursive: true });
+    mkdirSync(join(dataDir, 'index'), { recursive: true });
+  }
+
+  // Returns Cargo-style index path for a module name.
+  // 1-char names → index/1/{name}
+  // 2-char names → index/2/{name}
+  // 3-char names → index/3/{first-two-chars}/{name}
+  // 4+ char names → index/{chars 0-1}/{chars 2-3}/{name}
+  indexPath(name: string): string {
+    if (name.length === 1) return join(this.dataDir, 'index', '1', name);
+    if (name.length === 2) return join(this.dataDir, 'index', '2', name);
+    if (name.length === 3) return join(this.dataDir, 'index', '3', name.slice(0, 2), name);
+    return join(this.dataDir, 'index', name.slice(0, 2), name.slice(2, 4), name);
+  }
+
+  packagePath(name: string, version: string): string {
+    return join(this.dataDir, 'modules', name, version, `${name}-${version}.netapp`);
+  }
+
+  sha256(data: Buffer): string {
+    return createHash('sha256').update(data).digest('hex');
+  }
+
+  readIndex(name: string): IndexEntry[] {
+    const path = this.indexPath(name);
+    if (!existsSync(path)) return [];
+    return readFileSync(path, 'utf-8')
+      .split('\n')
+      .filter(Boolean)
+      .map((line) => JSON.parse(line) as IndexEntry);
+  }
+
+  appendIndex(entry: IndexEntry): void {
+    const path = this.indexPath(entry.name);
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, `${JSON.stringify(entry)}\n`, { flag: 'a' });
+  }
+
+  updateIndex(name: string, entries: IndexEntry[]): void {
+    const path = this.indexPath(name);
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, `${entries.map((e) => JSON.stringify(e)).join('\n')}\n`);
+  }
+
+  storePackage(name: string, version: string, data: Buffer): string {
+    const path = this.packagePath(name, version);
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, data);
+    return this.sha256(data);
+  }
+
+  packageExists(name: string, version: string): boolean {
+    return existsSync(this.packagePath(name, version));
+  }
+
+  readPackage(name: string, version: string): Buffer | null {
+    const path = this.packagePath(name, version);
+    return existsSync(path) ? readFileSync(path) : null;
+  }
+
+  listModules(): Array<{ name: string; versions: IndexEntry[] }> {
+    const modulesDir = join(this.dataDir, 'modules');
+    if (!existsSync(modulesDir)) return [];
+    const names = readdirSync(modulesDir, { withFileTypes: true })
+      .filter((d) => d.isDirectory())
+      .map((d) => d.name);
+    return names.map((name) => ({ name, versions: this.readIndex(name) }));
+  }
+
+  getModule(name: string): { name: string; versions: IndexEntry[] } | null {
+    const versions = this.readIndex(name);
+    return versions.length > 0 ? { name, versions } : null;
+  }
+
+  /**
+   * Per-module download counter. Stored as a single integer in
+   * `<dataDir>/downloads/<name>`. Read-modify-write, NOT atomic
+   * across concurrent processes — a lost increment under contention
+   * is acceptable for a low-traffic registry. If we ever need real
+   * concurrency safety this is the place to swap in fcntl locks or
+   * a sqlite counter.
+   *
+   * Validation: `name` is checked by the caller (server.ts uses
+   * `isValidName` before reaching this layer); we still constrain
+   * the path lookup to the downloads/ subdirectory as defense in
+   * depth against path traversal in `name`.
+   */
+  private downloadsPath(name: string): string {
+    return join(this.dataDir, 'downloads', name);
+  }
+
+  getDownloads(name: string): number {
+    const path = this.downloadsPath(name);
+    if (!existsSync(path)) return 0;
+    const raw = readFileSync(path, 'utf-8').trim();
+    const n = Number.parseInt(raw, 10);
+    return Number.isFinite(n) && n >= 0 ? n : 0;
+  }
+
+  incrementDownloads(name: string): number {
+    const path = this.downloadsPath(name);
+    mkdirSync(dirname(path), { recursive: true });
+    const next = this.getDownloads(name) + 1;
+    writeFileSync(path, `${next}\n`);
+    return next;
+  }
+}

package/registry-server/src/validation.test.ts

@@ -0,0 +1,86 @@
+import { describe, expect, test } from 'bun:test';
+import { isValidName, isValidVersion, validateNameAndVersion } from './validation';
+
+describe('isValidName', () => {
+  test.each(['homebridge', 'my-module', 'a', 'a1', 'dns-external', 'x-y-z-2'])(
+    'accepts %s',
+    (name) => {
+      expect(isValidName(name)).toBe(true);
+    },
+  );
+
+  test.each([
+    ['empty', ''],
+    ['uppercase', 'Homebridge'],
+    ['underscore', 'my_module'],
+    ['leading hyphen', '-foo'],
+    ['trailing hyphen', 'foo-'],
+    ['double hyphen', 'foo--bar'],
+    ['space', 'foo bar'],
+    ['path traversal', '../etc'],
+    ['slash', 'foo/bar'],
+    ['dot', 'foo.bar'],
+    ['null byte', 'foo\0bar'],
+    ['newline', 'foo\nbar'],
+    ['carriage return', 'foo\rbar'],
+    ['URL-encoded slash after decode would be', '..%2F..'],
+    ['over 128 chars', `${'a'.repeat(129)}`],
+  ])('rejects %s', (_label, name) => {
+    expect(isValidName(name)).toBe(false);
+  });
+});
+
+describe('isValidVersion', () => {
+  test.each(['1.0.0+1', '1.4.2+1', '0.0.1+99', '1.5.0+1', '2:1.0.0+1', '0.0.0+1'])(
+    'accepts %s',
+    (vers) => {
+      expect(isValidVersion(vers)).toBe(true);
+    },
+  );
+
+  test.each([
+    ['empty', ''],
+    ['no pkgrev', '1.0.0'],
+    ['pkgrev not numeric', '1.0.0+abc'],
+    ['semver not numeric', 'a.b.c+1'],
+    ['hyphen pkgrev', '1.0.0-1'],
+    ['path traversal', '../../etc'],
+    ['slash', '1.0.0+1/foo'],
+    ['dot-dot', '1.0.0+1/..'],
+    ['null byte', '1.0.0+1\0'],
+    ['newline', '1.0.0+1\n'],
+    ['space', '1.0.0 +1'],
+    ['extra dot', '1.0.0.0+1'],
+    ['negative pkgrev', '1.0.0+-1'],
+    ['four parts', '1.2.3.4+1'],
+    ['pkgrev missing', '1.0.0+'],
+    ['URL-encoded bad version', '..%2F..%2Fetc'],
+    ['over 128 chars', `${'1'.repeat(129)}.0.0+1`],
+  ])('rejects %s', (_label, vers) => {
+    expect(isValidVersion(vers)).toBe(false);
+  });
+});
+
+describe('validateNameAndVersion', () => {
+  test('accepts a matched valid pair', () => {
+    expect(validateNameAndVersion('homebridge', '1.4.2+1')).toEqual({ ok: true });
+  });
+
+  test('flags the name field when bad', () => {
+    const r = validateNameAndVersion('../etc', '1.0.0+1');
+    expect(r.ok).toBe(false);
+    if (!r.ok) expect(r.field).toBe('name');
+  });
+
+  test('flags the vers field when bad', () => {
+    const r = validateNameAndVersion('homebridge', '../etc');
+    expect(r.ok).toBe(false);
+    if (!r.ok) expect(r.field).toBe('vers');
+  });
+
+  test('reports name first when both are bad (fail-fast)', () => {
+    const r = validateNameAndVersion('../etc', '1.0.0');
+    expect(r.ok).toBe(false);
+    if (!r.ok) expect(r.field).toBe('name');
+  });
+});

package/registry-server/src/validation.ts

@@ -0,0 +1,60 @@
+/**
+ * Input validation for every user-supplied identifier that ends up on the
+ * filesystem path. These two are the ONLY allowed shapes for module names
+ * and versions on this registry; both must be enforced at every public
+ * entry point in server.ts (publish, download, metadata, yank, unyank,
+ * sparse index) because `storage.packagePath()` / `storage.indexPath()`
+ * feed their inputs straight to `node:path.join`, which happily normalizes
+ * `..` segments and escapes the data directory.
+ *
+ * Keep these as pure regex predicates so there is no way to accidentally
+ * skip them — if the string passes, the path is guaranteed to contain
+ * only safe chars (lowercase alphanum + `-` / digits + `.` / `+`). No
+ * slashes, no `..`, no control chars.
+ */
+
+/** Module names: kebab-case lowercase, no leading/trailing hyphen. */
+const NAME_REGEX = /^[a-z0-9]+(-[a-z0-9]+)*$/;
+
+/**
+ * Module versions: optional Debian-style epoch (`N:`), then semver
+ * (major.minor.patch), then a literal `+`, then a numeric pkg revision.
+ * Examples: `1.4.2+1`, `1.0.0+12`, `2:1.0.0+1` (epoch bump).
+ * Rejects: `1.0.0+abc`, `1.0.0+1/../etc`, `1.0.0 +1`, empty, traversal.
+ */
+const VERSION_REGEX = /^(\d+:)?\d+\.\d+\.\d+\+\d+$/;
+
+/** Max bytes we'll accept for either identifier — far beyond any real use. */
+const MAX_IDENT_LENGTH = 128;
+
+export function isValidName(name: string): boolean {
+  return name.length > 0 && name.length <= MAX_IDENT_LENGTH && NAME_REGEX.test(name);
+}
+
+export function isValidVersion(vers: string): boolean {
+  return vers.length > 0 && vers.length <= MAX_IDENT_LENGTH && VERSION_REGEX.test(vers);
+}
+
+/** Throw-style version for handler call sites — returns the offending field name. */
+export function validateNameAndVersion(
+  name: string,
+  vers: string,
+): { ok: true } | { ok: false; field: 'name' | 'vers'; message: string } {
+  if (!isValidName(name)) {
+    return {
+      ok: false,
+      field: 'name',
+      message:
+        'Invalid module name — must be kebab-case lowercase, 1-128 chars (e.g. "my-module")',
+    };
+  }
+  if (!isValidVersion(vers)) {
+    return {
+      ok: false,
+      field: 'vers',
+      message:
+        'Invalid version — must match "<major>.<minor>.<patch>+<pkgrev>" (e.g. "1.4.2+1")',
+    };
+  }
+  return { ok: true };
+}

package/registry-server/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "strict": true,
+    "skipLibCheck": true,
+    "types": ["bun-types"],
+    "allowImportingTsExtensions": true,
+    "noEmit": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true
+  },
+  "include": ["src/**/*"]
+}

package/scripts/stage-libsignal.ts

@@ -0,0 +1,116 @@
+/**
+ * stage-libsignal.ts — put the libsignal JNI native (linux-aarch64) into the
+ * apt-repo staging pool, building it if we don't already have it.
+ *
+ * WHY THIS IS BUILT ON THE DEV MACHINE AND NOT IN THE E2E NETWORK
+ *
+ * The signal module needs a native that upstream does not publish for
+ * linux-aarch64 (see modules/signal/build/). Compiling it takes minutes,
+ * pulls hundreds of crates and a BoringSSL submodule, and needs crates.io and
+ * GitHub — none of which exist inside the sealed e2e network, and all of which
+ * would blow a 5-minute test budget.
+ *
+ * So the compile happens HERE, at `cele2e build-infra` time, on the host's own
+ * network, using the module's own recipe. The deploy then just installs a
+ * .deb, which is exactly what it does in production. The e2e network stays
+ * sealed and the deploy test stays fast, while the recipe is still exercised
+ * for real on every rebuild rather than rotting untested.
+ *
+ * Cached by version: the compile only re-runs when the version changes or the
+ * cached .deb is deleted.
+ *
+ * Graceful no-op when Docker is unavailable or the build fails — build-infra
+ * still completes, and the signal deploy test is what fails loudly (with an
+ * empty repo) rather than breaking the build for everyone. Mirrors how
+ * stageAptRepo treats a missing nfpm.
+ */
+
+import { spawnSync } from 'node:child_process';
+import { copyFileSync, existsSync, mkdirSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+
+const green = '\x1b[32m';
+const yellow = '\x1b[33m';
+const dim = '\x1b[2m';
+const reset = '\x1b[0m';
+
+/**
+ * Must match `modules/signal/manifest.yml`'s `libsignal_deb_url` default and
+ * `modules/signal/build/build.sh`. The build script verifies the pairing
+ * against the real signal-cli release, so a mismatch here fails the build
+ * loudly rather than producing a native that silently will not load.
+ */
+const LIBSIGNAL_VERSION = '0.96.3';
+const DEB_NAME = `libsignal-jni_${LIBSIGNAL_VERSION}_arm64.deb`;
+
+export function stageLibsignal(repoRoot: string, pkgDir: string): boolean {
+  const buildDir = join(repoRoot, 'modules', 'signal', 'build');
+  if (!existsSync(buildDir)) return false;
+
+  // Only aarch64 needs this native at all — an x86_64 host uses the one already
+  // bundled in the libsignal-client JAR. Building it there would mean an
+  // emulated compile of something nothing would install.
+  if (process.arch !== 'arm64') {
+    console.log(
+      `  ${'libsignal-jni'.padEnd(28)} ${dim}skip (host is ${process.arch}; JAR ships that native)${reset}`,
+    );
+    return true;
+  }
+
+  const pool = join(pkgDir, '.apt-repo-cache', 'pool');
+  const cache = join(pkgDir, '.libsignal-cache');
+  mkdirSync(pool, { recursive: true });
+  mkdirSync(cache, { recursive: true });
+
+  const cached = join(cache, DEB_NAME);
+
+  if (!existsSync(cached)) {
+    process.stdout.write(`  ${'libsignal-jni (build)'.padEnd(28)} `);
+    const t0 = Date.now();
+
+    const image = 'celilo-e2e/libsignal-builder';
+    const build = spawnSync(
+      'docker',
+      ['build', '--platform', 'linux/arm64', '-q', '-t', image, buildDir],
+      { stdio: 'pipe' },
+    );
+    if (build.status !== 0) {
+      console.log(`${yellow}skip${reset}  ${dim}(docker build failed)${reset}`);
+      return false;
+    }
+
+    // Named volumes keep the cargo registry and the checkout across runs, so a
+    // version bump is a few minutes rather than a fresh fetch of everything.
+    const run = spawnSync(
+      'docker',
+      [
+        'run',
+        '--rm',
+        '--platform',
+        'linux/arm64',
+        '-v',
+        `${cache}:/out`,
+        '-v',
+        'celilo-libsignal-work:/work',
+        '-v',
+        'celilo-libsignal-cargo:/root/.cargo/registry',
+        image,
+      ],
+      { stdio: 'pipe' },
+    );
+    if (run.status !== 0 || !existsSync(cached)) {
+      console.log(`${yellow}skip${reset}  ${dim}(build failed)${reset}`);
+      return false;
+    }
+    console.log(
+      `${green}✔${reset}  ${dim}${Math.round((Date.now() - t0) / 1000)}s${reset}`,
+    );
+  }
+
+  process.stdout.write(`  ${'libsignal-jni (stage)'.padEnd(28)} `);
+  for (const file of readdirSync(cache).filter((f) => f.endsWith('.deb'))) {
+    copyFileSync(join(cache, file), join(pool, file));
+  }
+  console.log(`${green}✔${reset}  ${dim}${DEB_NAME}${reset}`);
+  return true;
+}