@celilo/e2e 0.9.2 → 0.9.3

package/registry-server/src/bootstrap.ts

@@ -1,246 +0,0 @@
-/**
- * Bootstrap mode — optional behavior for the e2e stand-in registry.
- *
- * When BOOTSTRAP_MODULES_DIR is set, the server also serves modules whose
- * source lives in that directory. Each immediate subdirectory containing a
- * manifest.yml is treated as a module; its manifest's `id` and `version`
- * become the sparse index entry. On download, the directory is packaged
- * into a .netapp tarball on the fly (cached on disk).
- *
- * This preserves the existing packages/e2e/docker/Dockerfile.registry
- * behavior where tests can import any module from the repo without having
- * to explicitly publish each dependency first.
- *
- * In production, BOOTSTRAP_MODULES_DIR is unset and this whole layer is
- * inert — storage.ts is the only code that touches disk.
- */
-
-import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
-import { basename, dirname, join, relative } from 'node:path';
-import { createHash } from 'node:crypto';
-import type { IndexEntry } from './storage';
-
-export interface BootstrapEntry {
-  name: string;
-  version: string;
-  /** Source directory to package on demand (source mode). */
-  sourceDir?: string;
-  /** Pre-built .netapp to serve directly (upload mode). */
-  netappPath?: string;
-  /**
-   * Manifest's `description` field, when present. Used by the search
-   * endpoint to populate the module-list payload — see
-   * apps/celilo/designs/REGISTRY_BROWSE_UI.md (Phase 2 step 0).
-   */
-  description?: string;
-}
-
-/**
- * Scan a directory for modules. Each subdirectory is a module iff it contains
- * a manifest.yml. Manifest parsing is line-regex (not full YAML) so we don't
- * need a YAML dependency at runtime — this matches packages/e2e/src/registry-server.ts.
- */
-export function scanBootstrapDir(bootstrapDir: string): Map<string, BootstrapEntry> {
-  const entries = new Map<string, BootstrapEntry>();
-  if (!existsSync(bootstrapDir)) return entries;
-
-  for (const entry of readdirSync(bootstrapDir, { withFileTypes: true })) {
-    if (!entry.isDirectory()) continue;
-    const manifestPath = join(bootstrapDir, entry.name, 'manifest.yml');
-    if (!existsSync(manifestPath)) continue;
-    try {
-      const yaml = readFileSync(manifestPath, 'utf-8');
-      const idMatch = yaml.match(/^id:\s*['"]?([^\s'"#]+)['"]?/m);
-      const versionMatch = yaml.match(/^version:\s*['"]?([^\s'"#]+)['"]?/m);
-      // Description is the rest-of-line up to (but not including) the
-      // newline. The manifest schema allows multi-line scalars, but
-      // existing modules use the single-line form so the line-regex is
-      // sufficient. We strip surrounding quotes if present.
-      const descriptionMatch = yaml.match(/^description:\s*(.+?)\s*$/m);
-      const name = idMatch?.[1]?.trim() ?? entry.name;
-      const version = versionMatch?.[1]?.trim() ?? '0.0.0';
-      const description = descriptionMatch?.[1]?.trim().replace(/^['"]|['"]$/g, '');
-      entries.set(name, {
-        name,
-        version,
-        sourceDir: join(bootstrapDir, entry.name),
-        description,
-      });
-    } catch {
-      // skip malformed manifests — matches e2e server behavior
-    }
-  }
-  return entries;
-}
-
-/**
- * Scan a directory for pre-built .netapp files. Filename (minus suffix) is
- * the module name. Version defaults to the filename's version suffix or to
- * 0.0.0+1 — real version data is in the archive's manifest but parsing that
- * at startup is overkill for the e2e bootstrap use case.
- *
- * Used by the shared-infra stand-in: the e2e harness's publishModule helper
- * copies .netapps into BOOTSTRAP_UPLOADS_DIR with `docker cp`, and the
- * server picks them up without needing the full HTTP publish protocol.
- */
-export function scanUploadsDir(uploadsDir: string): Map<string, BootstrapEntry> {
-  const entries = new Map<string, BootstrapEntry>();
-  if (!existsSync(uploadsDir)) return entries;
-
-  for (const file of readdirSync(uploadsDir)) {
-    if (!file.endsWith('.netapp')) continue;
-    const name = file.slice(0, -7);
-    entries.set(name, {
-      name,
-      version: '0.0.0',
-      netappPath: join(uploadsDir, file),
-    });
-  }
-  return entries;
-}
-
-/** Render a bootstrap entry as an IndexEntry suitable for the sparse index. */
-export function bootstrapIndexEntry(entry: BootstrapEntry): IndexEntry {
-  const vers = entry.version.includes('+') ? entry.version : `${entry.version}+1`;
-  return {
-    name: entry.name,
-    vers,
-    deps: [],
-    cksum: 'bootstrap',
-    yanked: false,
-  };
-}
-
-/**
- * Walk a directory, skipping hidden entries, *.test.ts files, the
- * module's e2e/ tree (tests + their deps don't ship with the module),
- * and most of node_modules.
- *
- * `scripts/node_modules` (the module's hook runtime) is bundled IN FULL
- * (ISS-0046): hook scripts run on the target, where the package registry can
- * be unreachable, so the `.netapp` must carry @celilo/capabilities AND its
- * third-party deps (tldts, drizzle-orm, …). Any OTHER node_modules ships only
- * `@celilo/capabilities` — the framework SDK the module was authored against.
- *
- * This is the structural twin of the canonical `includeNodeModulesPath` rule in
- * apps/celilo/src/module/packaging/package-rules.ts. We can't import that at
- * runtime (this server ships in a standalone Docker image with no @celilo deps),
- * so the two are held in lockstep by bootstrap-packaging.test.ts, which asserts
- * `walkDir`'s output conforms to the canonical rule. `walkDir` is exported for
- * that test.
- */
-export function walkDir(dir: string, base = dir): string[] {
-  const results: string[] = [];
-  for (const entry of readdirSync(dir, { withFileTypes: true })) {
-    if (entry.name.startsWith('.')) continue;
-    if (entry.name.endsWith('.test.ts')) continue;
-
-    const fullPath = join(dir, entry.name);
-
-    // Module's e2e/ directory at the root is tests + their devDependencies.
-    // Detected by being a sibling of manifest.yml.
-    if (entry.name === 'e2e' && existsSync(join(dir, 'manifest.yml'))) {
-      continue;
-    }
-
-    if (entry.name === 'node_modules') {
-      if (basename(dir) === 'scripts') {
-        // The hook-script runtime closure — bundle it whole (incl. nested deps).
-        // walkAll keeps the hidden-entry / *.test.ts skips, so .bin shims drop.
-        results.push(...walkAll(fullPath, base));
-      } else {
-        // Other node_modules: collect only @celilo/capabilities.
-        const capDir = join(fullPath, '@celilo', 'capabilities');
-        if (existsSync(capDir)) {
-          results.push(...walkDir(capDir, base));
-        }
-      }
-      continue;
-    }
-
-    if (entry.isDirectory()) {
-      results.push(...walkDir(fullPath, base));
-    } else {
-      results.push(relative(base, fullPath));
-    }
-  }
-  return results;
-}
-
-/**
- * Collect every file under `dir` (recursing into nested node_modules too),
- * skipping only hidden entries (`.bin` symlink shims) and `*.test.ts`. Used to
- * bundle a module's full `scripts/node_modules` runtime closure.
- */
-function walkAll(dir: string, base: string): string[] {
-  const results: string[] = [];
-  for (const entry of readdirSync(dir, { withFileTypes: true })) {
-    if (entry.name.startsWith('.')) continue;
-    if (entry.name.endsWith('.test.ts')) continue;
-    const fullPath = join(dir, entry.name);
-    if (entry.isDirectory()) {
-      results.push(...walkAll(fullPath, base));
-    } else {
-      results.push(relative(base, fullPath));
-    }
-  }
-  return results;
-}
-
-/**
- * Package a module directory into a .netapp tarball. Cached by module name
- * in cacheDir; cache is keyed by name only (assume the source is immutable
- * for the lifetime of the server — fine for e2e).
- */
-export async function packageBootstrapModule(
-  entry: BootstrapEntry,
-  cacheDir: string,
-): Promise<string> {
-  // Upload-mode entries are already .netapp files — serve as-is.
-  if (entry.netappPath) return entry.netappPath;
-  const sourceDir = entry.sourceDir;
-  if (!sourceDir) {
-    throw new Error(`bootstrap entry ${entry.name} has neither sourceDir nor netappPath`);
-  }
-  mkdirSync(cacheDir, { recursive: true });
-  const cachePath = join(cacheDir, `${entry.name}.netapp`);
-  if (existsSync(cachePath)) return cachePath;
-
-  const filePaths = walkDir(sourceDir);
-  const fileHashes: Record<string, string> = {};
-  for (const relPath of filePaths) {
-    const data = readFileSync(join(sourceDir, relPath));
-    fileHashes[relPath] = createHash('sha256').update(data).digest('hex');
-  }
-
-  const checksumsJson = JSON.stringify(
-    { version: '1', generated: new Date().toISOString(), files: fileHashes },
-    null,
-    2,
-  );
-
-  // Source dir may be read-only (volume-mounted) — stage in /tmp first.
-  const stagingDir = join('/tmp', `celilo-bootstrap-${entry.name}-${Date.now()}`);
-  mkdirSync(stagingDir, { recursive: true });
-  try {
-    for (const relPath of filePaths) {
-      mkdirSync(dirname(join(stagingDir, relPath)), { recursive: true });
-      writeFileSync(join(stagingDir, relPath), readFileSync(join(sourceDir, relPath)));
-    }
-    writeFileSync(join(stagingDir, 'checksums.json'), checksumsJson);
-
-    const proc = Bun.spawn(['tar', '-czf', cachePath, '-C', stagingDir, '.'], {
-      stderr: 'inherit',
-    });
-    const exitCode = await proc.exited;
-    if (exitCode !== 0) throw new Error(`tar exited with code ${exitCode}`);
-  } finally {
-    try {
-      Bun.spawnSync(['rm', '-rf', stagingDir]);
-    } catch {
-      // best-effort cleanup
-    }
-  }
-
-  return cachePath;
-}

package/registry-server/src/index.ts

@@ -1,16 +0,0 @@
-export { RegistryStorage } from './storage';
-export type { IndexEntry, ModuleVersion } from './storage';
-export { TokenAuth } from './auth';
-export { startServer, startFromEnv } from './server';
-export {
-  scanBootstrapDir,
-  bootstrapIndexEntry,
-  packageBootstrapModule,
-} from './bootstrap';
-export type { BootstrapEntry } from './bootstrap';
-
-import { startFromEnv } from './server';
-
-if (import.meta.main) {
-  startFromEnv();
-}

package/registry-server/src/landing.ts

@@ -1,90 +0,0 @@
-/**
- * Landing-page response for `GET /` (and `/index.html`) on the registry.
- *
- * This is the polite-301 from the design doc:
- * (apps/celilo/designs/REGISTRY_BROWSE_UI.md, decision D2). The response
- * carries `301 Moved Permanently` with `Location:` pointing at the
- * celilo.computer site's `/modules/` browse UI — interactive browsers
- * follow the redirect. Clients that don't follow by default (curl
- * without `-L`, naive bots, bare-bones HTTP libraries) see the body,
- * which carries the install one-liner, a link to the browse UI, and
- * the API endpoints they probably actually wanted.
- *
- * Single endpoint, no Astro dep, no client JS, no asset pipeline.
- */
-
-/**
- * HTML-escape `<`, `>`, `&`, `"` so values interpolated into the body
- * can't inject markup. Operator-controlled values (publicUrl) are
- * trusted in practice, but defensive escaping costs nothing.
- */
-function escapeHtml(s: string): string {
-  return s
-    .replace(/&/g, '&amp;')
-    .replace(/</g, '&lt;')
-    .replace(/>/g, '&gt;')
-    .replace(/"/g, '&quot;');
-}
-
-/**
- * Build the polite-301 response.
- *
- * @param publicUrl  The registry's public URL (typically
- *                   `https://celilo.computer/registry`). The site's
- *                   origin is derived by stripping the path. The
- *                   `/modules/` browse URL on that origin is the
- *                   redirect target.
- */
-export function landingResponse(publicUrl: string): Response {
-  const u = new URL(publicUrl);
-  const siteOrigin = u.origin;
-  const browseUrl = `${siteOrigin}/modules/`;
-  const installUrl = `${siteOrigin}/install.sh`;
-  const apiBase = publicUrl.replace(/\/$/, '');
-
-  const body = `<!DOCTYPE html>
-<html lang="en">
-<head>
-<meta charset="UTF-8">
-<meta name="viewport" content="width=device-width, initial-scale=1">
-<title>Celilo Module Registry</title>
-<style>
-  body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
-         max-width: 640px; margin: 4em auto; padding: 0 1em;
-         line-height: 1.6; color: #222; }
-  h1 { margin-bottom: 0.2em; }
-  .lead { color: #666; margin-top: 0; }
-  pre { background: #f5f5f5; padding: 0.8em 1em; border-radius: 6px;
-        overflow-x: auto; font-size: 0.9em; }
-  a { color: #0d72b9; }
-  small { color: #888; }
-</style>
-</head>
-<body>
-<h1>Celilo Module Registry</h1>
-<p class="lead">Home-lab orchestration modules — published, versioned, install via the Celilo CLI.</p>
-
-<p>If you're a person, head to <a href="${escapeHtml(browseUrl)}">browse the modules →</a></p>
-
-<p>Install the Celilo CLI:</p>
-<pre><code>curl -fsSL ${escapeHtml(installUrl)} | bash</code></pre>
-
-<p>If you're a tool, the JSON API is here:</p>
-<pre><code>GET ${escapeHtml(apiBase)}/api/v1/modules
-GET ${escapeHtml(apiBase)}/api/v1/modules/{name}
-GET ${escapeHtml(apiBase)}/api/v1/modules/{name}/{version}/download</code></pre>
-
-<p><small>This URL is a 301 to <a href="${escapeHtml(browseUrl)}">${escapeHtml(browseUrl)}</a>. Most browsers followed it before you saw this.</small></p>
-</body>
-</html>
-`;
-
-  return new Response(body, {
-    status: 301,
-    headers: {
-      Location: browseUrl,
-      'Content-Type': 'text/html; charset=utf-8',
-      'Cache-Control': 'public, max-age=300',
-    },
-  });
-}

package/registry-server/src/rate-limit.test.ts

@@ -1,62 +0,0 @@
-import { describe, expect, test } from 'bun:test';
-import { clientIp, createRateLimiter } from './rate-limit';
-
-describe('createRateLimiter', () => {
-  test('allows up to max, then rejects', () => {
-    const limiter = createRateLimiter({ max: 3, windowMs: 60_000, now: () => 0 });
-    expect(limiter.take('1.1.1.1').ok).toBe(true);
-    expect(limiter.take('1.1.1.1').ok).toBe(true);
-    expect(limiter.take('1.1.1.1').ok).toBe(true);
-    const over = limiter.take('1.1.1.1');
-    expect(over.ok).toBe(false);
-    if (!over.ok) expect(over.retryAfterSec).toBeGreaterThan(0);
-  });
-
-  test('distinct IPs are tracked separately', () => {
-    const limiter = createRateLimiter({ max: 1, windowMs: 60_000, now: () => 0 });
-    expect(limiter.take('1.1.1.1').ok).toBe(true);
-    expect(limiter.take('2.2.2.2').ok).toBe(true); // different IP, fresh allowance
-    expect(limiter.take('1.1.1.1').ok).toBe(false);
-  });
-
-  test('bucket resets after window elapses', () => {
-    let t = 0;
-    const limiter = createRateLimiter({ max: 1, windowMs: 1000, now: () => t });
-    expect(limiter.take('1.1.1.1').ok).toBe(true);
-    expect(limiter.take('1.1.1.1').ok).toBe(false);
-    t = 1001;
-    expect(limiter.take('1.1.1.1').ok).toBe(true);
-  });
-
-  test('retryAfterSec reflects remaining window', () => {
-    const limiter = createRateLimiter({ max: 1, windowMs: 60_000, now: () => 0 });
-    limiter.take('1.1.1.1');
-    const r = limiter.take('1.1.1.1');
-    expect(r.ok).toBe(false);
-    if (!r.ok) {
-      expect(r.retryAfterSec).toBeLessThanOrEqual(60);
-      expect(r.retryAfterSec).toBeGreaterThan(0);
-    }
-  });
-});
-
-describe('clientIp', () => {
-  function req(headers: Record<string, string>): Request {
-    return new Request('http://x/', { headers });
-  }
-  const stubServer = {
-    requestIP: () => ({ address: '10.0.0.1' }),
-  };
-
-  test('uses X-Forwarded-For first entry when present', () => {
-    expect(clientIp(req({ 'x-forwarded-for': '1.1.1.1, 2.2.2.2' }), stubServer)).toBe('1.1.1.1');
-  });
-
-  test('falls back to Bun.Server.requestIP when XFF absent', () => {
-    expect(clientIp(req({}), stubServer)).toBe('10.0.0.1');
-  });
-
-  test('returns "unknown" when neither available', () => {
-    expect(clientIp(req({}), { requestIP: () => null })).toBe('unknown');
-  });
-});

package/registry-server/src/rate-limit.ts

@@ -1,83 +0,0 @@
-/**
- * Simple in-memory per-IP rate limiter for the auth-gated endpoints
- * (publish, yank, unyank). The public-read endpoints (download, index,
- * search) are left unlimited — they're cacheable and the body-size cap
- * on publish already covers the main DoS vector.
- *
- * We implement this in-process rather than in Caddy because the stock
- * apt-installed Caddy doesn't include the community `caddy-ratelimit`
- * plugin. If we ever switch the caddy module to `xcaddy build` with the
- * plugin, we can move this upstream and delete this file.
- *
- * The limiter is a fixed window counter: each IP gets a fresh allowance
- * every `windowMs`. No burst smoothing, which is fine for the tiny
- * traffic a registry actually sees — we just want to keep a bad actor
- * from flooding the disk via repeated publish attempts.
- */
-
-interface Bucket {
-  count: number;
-  /** Epoch ms when this bucket resets to 0 and `windowMs` restarts. */
-  resetAt: number;
-}
-
-export interface RateLimiter {
-  /**
-   * Record one hit from `ip`. Returns {ok: true, remaining} if under the
-   * limit; {ok: false, retryAfterSec} if the window is exhausted.
-   */
-  take(ip: string): { ok: true; remaining: number } | { ok: false; retryAfterSec: number };
-}
-
-export function createRateLimiter(opts: {
-  /** Max events per IP per window. */
-  max: number;
-  /** Window length in ms. */
-  windowMs: number;
-  /** Inject a clock for tests; defaults to Date.now. */
-  now?: () => number;
-}): RateLimiter {
-  const buckets = new Map<string, Bucket>();
-  const now = opts.now ?? (() => Date.now());
-
-  // Opportunistic cleanup: if the map grows past a size threshold, drop
-  // expired entries. Keeps memory bounded even under churny attackers
-  // hitting from rotating IPs.
-  const CLEANUP_AT = 1000;
-
-  return {
-    take(ip: string) {
-      const t = now();
-      let bucket = buckets.get(ip);
-      if (!bucket || t >= bucket.resetAt) {
-        bucket = { count: 0, resetAt: t + opts.windowMs };
-        buckets.set(ip, bucket);
-      }
-      bucket.count += 1;
-      if (bucket.count > opts.max) {
-        return { ok: false, retryAfterSec: Math.max(1, Math.ceil((bucket.resetAt - t) / 1000)) };
-      }
-      if (buckets.size > CLEANUP_AT) {
-        for (const [k, v] of buckets) if (t >= v.resetAt) buckets.delete(k);
-      }
-      return { ok: true, remaining: opts.max - bucket.count };
-    },
-  };
-}
-
-/**
- * Extract the caller's IP for rate-limit keying. Prefers the first entry
- * of X-Forwarded-For (trusted-proxy assumption — we sit behind Caddy);
- * falls back to the raw socket peer via Bun.Server.requestIP.
- */
-export function clientIp(
-  req: Request,
-  server: { requestIP(r: Request): { address: string } | null },
-): string {
-  const xff = req.headers.get('x-forwarded-for');
-  if (xff) {
-    const first = xff.split(',')[0]?.trim();
-    if (first) return first;
-  }
-  return server.requestIP(req)?.address ?? 'unknown';
-}

package/registry-server/src/scoped-token-store.test.ts

@@ -1,93 +0,0 @@
-import { describe, expect, test } from 'bun:test';
-import { hashToken } from './auth';
-import {
-  type ScopedTokenEntry,
-  type ScopedTokenPersistence,
-  ScopedTokenStore,
-} from './scoped-token-store';
-
-/** In-memory persistence so the store is testable without the filesystem. */
-function memoryPersistence(seed: ScopedTokenEntry[] = []): ScopedTokenPersistence & {
-  saved: ScopedTokenEntry[];
-} {
-  const state = { saved: [...seed] };
-  return {
-    saved: state.saved,
-    load() {
-      return [...state.saved];
-    },
-    save(entries) {
-      state.saved.length = 0;
-      state.saved.push(...entries);
-    },
-  };
-}
-
-/** Deterministic token generator (avoids randomness in assertions). */
-function seqGen() {
-  let n = 0;
-  return () => `tok-${++n}`;
-}
-
-describe('ScopedTokenStore.mint (ISS-0140)', () => {
-  test('mints a scoped token and persists only the hash', () => {
-    const p = memoryPersistence();
-    const store = new ScopedTokenStore(p, seqGen(), () => '2026-06-20T00:00:00Z');
-    const { token, entry, revokedHashes } = store.mint('celilo/lunacycle', 'lunacycle');
-
-    expect(token).toBe('tok-1');
-    expect(entry.scope).toBe('lunacycle');
-    expect(entry.repo).toBe('celilo/lunacycle');
-    expect(entry.hash).toBe(hashToken('tok-1'));
-    expect(revokedHashes).toEqual([]);
-    // Persisted form holds the hash, never the cleartext token.
-    expect(JSON.stringify(p.saved)).not.toContain('tok-1');
-    expect(p.saved).toHaveLength(1);
-  });
-
-  test('re-minting for the same repo rotates the token (reconcile)', () => {
-    const p = memoryPersistence();
-    const store = new ScopedTokenStore(p, seqGen(), () => 't');
-    const first = store.mint('celilo/lunacycle', 'lunacycle');
-    const second = store.mint('celilo/lunacycle', 'lunacycle');
-
-    expect(second.token).toBe('tok-2');
-    expect(second.revokedHashes).toEqual([first.entry.hash]);
-    // Still exactly one active token for the repo.
-    expect(store.list().filter((e) => e.repo === 'celilo/lunacycle')).toHaveLength(1);
-    expect(store.list()).toHaveLength(1);
-  });
-
-  test('different repos coexist independently', () => {
-    const store = new ScopedTokenStore(memoryPersistence(), seqGen(), () => 't');
-    store.mint('celilo/lunacycle', 'lunacycle');
-    store.mint('celilo/caddy', 'caddy');
-    expect(store.list()).toHaveLength(2);
-  });
-});
-
-describe('ScopedTokenStore.revoke', () => {
-  test('removes all tokens for a repo and returns their hashes', () => {
-    const store = new ScopedTokenStore(memoryPersistence(), seqGen(), () => 't');
-    const { entry } = store.mint('celilo/lunacycle', 'lunacycle');
-    const removed = store.revoke('celilo/lunacycle');
-    expect(removed).toEqual([entry.hash]);
-    expect(store.list()).toHaveLength(0);
-  });
-
-  test('revoking an unknown repo is a no-op', () => {
-    const store = new ScopedTokenStore(memoryPersistence(), seqGen(), () => 't');
-    expect(store.revoke('celilo/nope')).toEqual([]);
-  });
-});
-
-describe('ScopedTokenStore startup load', () => {
-  test('loads persisted entries on construction', () => {
-    const seed: ScopedTokenEntry[] = [
-      { hash: hashToken('x'), scope: 'caddy', repo: 'celilo/caddy', mintedAt: 't' },
-    ];
-    const store = new ScopedTokenStore(memoryPersistence(seed), seqGen(), () => 't');
-    expect(store.list()).toHaveLength(1);
-    expect(store.list()[0]?.repo).toBe('celilo/caddy');
-  });
-});