@celilo/e2e 0.9.2-alpha.0 → 0.9.3

package/registry-server/src/server.ts

@@ -1,572 +0,0 @@
-/**
- * Celilo Module Registry Server
- *
- * Implements the Cargo sparse registry protocol:
- *   GET  /index/config.json                              → sparse protocol config
- *   GET  /index/{path}                                   → sparse index file (NDJSON)
- *   GET  /api/v1/modules                                 → search modules
- *   GET  /api/v1/modules/{name}                          → module metadata
- *   GET  /api/v1/modules/{name}/{ver}/download           → download .netapp
- *   PUT  /api/v1/modules/new                             → publish (token auth)
- *   DELETE /api/v1/modules/{name}/{ver}/yank             → yank
- *   PUT  /api/v1/modules/{name}/{ver}/unyank             → unyank
- *
- * All routes can be prefixed with PATH_PREFIX (e.g. "/registry").
- *
- * When BOOTSTRAP_MODULES_DIR is set, modules discovered there are exposed
- * through the same endpoints — see bootstrap.ts. Bootstrap entries are
- * shadowed by real publishes (same name): the stored package wins so tests
- * can publish-then-import against a live registry.
- */
-
-import { join } from 'node:path';
-import { TokenAuth } from './auth';
-import {
-  type BootstrapEntry,
-  bootstrapIndexEntry,
-  packageBootstrapModule,
-  scanBootstrapDir,
-  scanUploadsDir,
-} from './bootstrap';
-import { landingResponse } from './landing';
-import { type RateLimiter, clientIp, createRateLimiter } from './rate-limit';
-import { ScopedTokenStore, fileScopedTokenPersistence } from './scoped-token-store';
-import { type IndexEntry, RegistryStorage } from './storage';
-import { isValidName, isValidVersion, validateNameAndVersion } from './validation';
-
-/** Max total publish body size. Real modules are a few MB; 100MB is generous. */
-const MAX_PUBLISH_BYTES = 100 * 1024 * 1024;
-
-/** Max JSON metadata blob size inside a publish body. Real metadata is ~hundreds of bytes. */
-const MAX_META_BYTES = 64 * 1024;
-
-/**
- * RFC 6266-safe Content-Disposition value. Strips everything but
- * `[A-Za-z0-9_.+-]` for the quoted form (defense-in-depth: even if the
- * upstream name/version validator is bypassed, the header cannot inject
- * CRLF or break out of the quoted-string), and includes the original
- * UTF-8 value via `filename*=UTF-8''...` for compliant clients.
- */
-function contentDispositionAttachment(filename: string): string {
-  const quotedSafe = filename.replace(/[^A-Za-z0-9_.+\-]/g, '_');
-  return `attachment; filename="${quotedSafe}"; filename*=UTF-8''${encodeURIComponent(filename)}`;
-}
-
-interface ServerOptions {
-  dataDir: string;
-  port: number;
-  pathPrefix: string;
-  publicUrl: string;
-  /** Scanned for module source dirs (with manifest.yml), packaged on demand. */
-  bootstrapModulesDir?: string;
-  /** Scanned for pre-built .netapp files, served directly. */
-  bootstrapUploadsDir?: string;
-  bootstrapCacheDir?: string;
-  auth: TokenAuth;
-  /**
-   * Optional override for the write-endpoint rate limiter; defaults to
-   * 30 events per IP per minute, which is generous for a registry (real
-   * publishes happen on the order of once per feature) but tight enough
-   * to slow a bad actor to a crawl.
-   */
-  rateLimiter?: RateLimiter;
-  /**
-   * Store of minted, per-repo package-scoped tokens (build-bus Phase 3,
-   * ISS-0140). Defaults to a JSON file in `dataDir`. Its persisted hashes are
-   * loaded into `auth` at startup so minted tokens survive a restart.
-   */
-  scopedTokenStore?: ScopedTokenStore;
-}
-
-export function startServer(options: ServerOptions): ReturnType<typeof Bun.serve> {
-  const { dataDir, port, pathPrefix, publicUrl, auth } = options;
-  const storage = new RegistryStorage(dataDir);
-  const scopedTokenStore =
-    options.scopedTokenStore ??
-    new ScopedTokenStore(fileScopedTokenPersistence(join(dataDir, 'scoped-tokens.json')));
-  // Load persisted minted tokens into the in-memory authorizer.
-  for (const entry of scopedTokenStore.list()) {
-    auth.addHashed(entry.hash, entry.scope);
-  }
-  const bootstrapCacheDir = options.bootstrapCacheDir ?? join(dataDir, '.bootstrap-cache');
-  const rateLimiter = options.rateLimiter ?? createRateLimiter({ max: 30, windowMs: 60_000 });
-
-  /**
-   * Rescan every request so .netapps dropped into bootstrapUploadsDir at
-   * runtime (via `docker cp` from the e2e harness) are picked up without
-   * restarting the server. Source dirs are less volatile so we only rescan
-   * them when uploads also get rescanned — the cost is a readdir or two per
-   * request, negligible in the e2e fixture context where this runs.
-   *
-   * In production neither env var is set and this function returns an empty
-   * map without touching the disk.
-   */
-  function resolveBootstrap(): Map<string, BootstrapEntry> {
-    const merged = new Map<string, BootstrapEntry>();
-    if (options.bootstrapModulesDir) {
-      for (const [name, entry] of scanBootstrapDir(options.bootstrapModulesDir)) {
-        merged.set(name, entry);
-      }
-    }
-    if (options.bootstrapUploadsDir) {
-      // Uploads shadow source-dir entries for the same name.
-      for (const [name, entry] of scanUploadsDir(options.bootstrapUploadsDir)) {
-        merged.set(name, entry);
-      }
-    }
-    return merged;
-  }
-
-  const sparseConfig = {
-    dl: `${publicUrl}/api/v1/modules/{name}/{version}/download`,
-    api: publicUrl,
-  };
-
-  function err(detail: string, status = 400): Response {
-    return Response.json({ errors: [{ detail }] }, { status });
-  }
-
-  function notFound(): Response {
-    return err('Not found', 404);
-  }
-
-  function unauthorized(): Response {
-    return err('Unauthorized — provide a valid publish token in the Authorization header', 401);
-  }
-
-  /** True when the request's token may publish/yank `pkg` (admin or exact scope). */
-  function authorizePackage(req: Request, pkg: string): boolean {
-    return auth.authorize(req.headers.get('Authorization') ?? '', pkg);
-  }
-
-  /** True when the request carries an admin (`*`-scoped) token — required to mint. */
-  function authorizeAdmin(req: Request): boolean {
-    return auth.isAdmin(req.headers.get('Authorization') ?? '');
-  }
-
-  /**
-   * Rate-limit check for write endpoints. Returns a 429 response when
-   * exceeded (with Retry-After), null when allowed.
-   */
-  function rateLimitOrNull(
-    req: Request,
-    srv: { requestIP(r: Request): { address: string } | null },
-  ): Response | null {
-    const ip = clientIp(req, srv);
-    const result = rateLimiter.take(ip);
-    if (result.ok) return null;
-    return new Response(
-      JSON.stringify({ errors: [{ detail: 'Too many write requests — try again later' }] }),
-      {
-        status: 429,
-        headers: {
-          'Content-Type': 'application/json',
-          'Retry-After': String(result.retryAfterSec),
-        },
-      },
-    );
-  }
-
-  /**
-   * Resolve a module name to (a) its published entries and (b) whether a
-   * bootstrap fallback exists. Real publishes take precedence — once a name
-   * has any published version, bootstrap for that name is ignored, so tests
-   * that publish namecheap@1.0.0+1 don't keep picking up the repo's source.
-   *
-   * Caller MUST have validated `name` first. We also guard here as a cheap
-   * backstop since the function is reachable from several handlers.
-   */
-  function resolveEntries(name: string): IndexEntry[] {
-    if (!isValidName(name)) return [];
-    const published = storage.readIndex(name);
-    if (published.length > 0) return published;
-    const b = resolveBootstrap().get(name);
-    return b ? [bootstrapIndexEntry(b)] : [];
-  }
-
-  async function serveBootstrapDownload(entry: BootstrapEntry): Promise<Response> {
-    const netappPath = await packageBootstrapModule(entry, bootstrapCacheDir);
-    return new Response(Bun.file(netappPath), {
-      headers: {
-        'Content-Type': 'application/octet-stream',
-        'Content-Disposition': contentDispositionAttachment(`${entry.name}.netapp`),
-      },
-    });
-  }
-
-  async function handlePublish(req: Request): Promise<Response> {
-    // Refuse to even buffer oversized publishes. Content-Length can be spoofed
-    // but a hostile client that lies to get past this still has to stream
-    // through Caddy's per-connection read limits; defense-in-depth.
-    const declaredLength = Number(req.headers.get('Content-Length'));
-    if (Number.isFinite(declaredLength) && declaredLength > MAX_PUBLISH_BYTES) {
-      return err(
-        `Publish body too large: ${declaredLength} bytes exceeds ${MAX_PUBLISH_BYTES}`,
-        413,
-      );
-    }
-
-    const buf = Buffer.from(await req.arrayBuffer());
-    if (buf.length > MAX_PUBLISH_BYTES) {
-      return err(`Publish body too large: ${buf.length} bytes exceeds ${MAX_PUBLISH_BYTES}`, 413);
-    }
-    if (buf.length < 8) return err('Request body too short');
-
-    const metaLen = buf.readUInt32LE(0);
-    // Cap the JSON metadata independently of the overall body. A malicious
-    // payload could set metaLen close to MAX_PUBLISH_BYTES to force
-    // multi-MB JSON.parse; the sparse protocol's metadata is always tiny
-    // in practice (a few hundred bytes).
-    if (metaLen > MAX_META_BYTES) {
-      return err(`Metadata too large: ${metaLen} bytes exceeds ${MAX_META_BYTES}`, 413);
-    }
-    if (buf.length < 4 + metaLen + 4) return err('Request body truncated (metadata)');
-
-    let meta: Record<string, unknown>;
-    try {
-      meta = JSON.parse(buf.subarray(4, 4 + metaLen).toString('utf-8')) as Record<string, unknown>;
-    } catch {
-      return err('Invalid JSON metadata');
-    }
-
-    const name = String(meta.name ?? '');
-    const vers = String(meta.vers ?? '');
-    // Description is optional, comes from the publishing client (the
-    // celilo CLI reads manifest.yml#description and includes it). When
-    // not provided we leave it undefined; the search endpoint falls
-    // back to bootstrap data or empty.
-    const description = typeof meta.description === 'string' ? meta.description : undefined;
-
-    const validation = validateNameAndVersion(name, vers);
-    if (!validation.ok) return err(validation.message);
-
-    // Scope check AFTER the name is known: a package-scoped token may publish
-    // only its own package; an admin token publishes anything (ISS-0140).
-    if (!authorizePackage(req, name)) return unauthorized();
-
-    const fileOffset = 4 + metaLen;
-    const fileLen = buf.readUInt32LE(fileOffset);
-    if (buf.length < fileOffset + 4 + fileLen) return err('Request body truncated (file)');
-
-    const fileData = buf.subarray(fileOffset + 4, fileOffset + 4 + fileLen);
-
-    if (storage.packageExists(name, vers)) {
-      return err(`Version ${name}@${vers} already exists — versions are immutable`, 409);
-    }
-
-    const cksum = storage.storePackage(name, vers, fileData);
-    storage.appendIndex({
-      name,
-      vers,
-      deps: [],
-      cksum: `sha256:${cksum}`,
-      yanked: false,
-      description,
-    });
-
-    console.log(`[registry] published ${name}@${vers} (${fileLen} bytes, sha256:${cksum})`);
-    return Response.json({ ok: true, name, vers });
-  }
-
-  function setYanked(name: string, version: string, yanked: boolean): Response {
-    const entries = storage.readIndex(name);
-    const entry = entries.find((e) => e.vers === version);
-    if (!entry) return notFound();
-    entry.yanked = yanked;
-    storage.updateIndex(name, entries);
-    return Response.json({ ok: true });
-  }
-
-  /**
-   * Mint a per-repo, package-scoped publish token (ISS-0140). Admin-only. The
-   * `registry_publish` capability calls this on behalf of `source_forge.
-   * registerRepo`. Reconciled: re-minting for the same repo rotates the token.
-   * The raw token is returned ONCE — the caller sets it as the repo's Actions
-   * secret; the server keeps only its hash.
-   */
-  async function handleMintToken(req: Request): Promise<Response> {
-    if (!authorizeAdmin(req)) return unauthorized();
-    let body: { repo?: unknown; scope?: unknown };
-    try {
-      body = (await req.json()) as { repo?: unknown; scope?: unknown };
-    } catch {
-      return err('Invalid JSON body');
-    }
-    const repo = typeof body.repo === 'string' ? body.repo.trim() : '';
-    const scope = typeof body.scope === 'string' ? body.scope.trim() : '';
-    if (!repo) return err('repo is required');
-    if (!scope || !isValidName(scope)) return err('scope must be a valid package name');
-
-    const { token, entry, revokedHashes } = scopedTokenStore.mint(repo, scope);
-    for (const h of revokedHashes) auth.removeHashed(h);
-    auth.addHashed(entry.hash, entry.scope);
-    console.log(`[registry] minted scoped token for ${repo} → ${scope}`);
-    return Response.json({ ok: true, token, scope, repo });
-  }
-
-  /** Revoke all scoped tokens for a repo (ISS-0140). Admin-only. */
-  async function handleRevokeToken(req: Request): Promise<Response> {
-    if (!authorizeAdmin(req)) return unauthorized();
-    let body: { repo?: unknown };
-    try {
-      body = (await req.json()) as { repo?: unknown };
-    } catch {
-      return err('Invalid JSON body');
-    }
-    const repo = typeof body.repo === 'string' ? body.repo.trim() : '';
-    if (!repo) return err('repo is required');
-
-    const removed = scopedTokenStore.revoke(repo);
-    for (const h of removed) auth.removeHashed(h);
-    console.log(`[registry] revoked ${removed.length} scoped token(s) for ${repo}`);
-    return Response.json({ ok: true, repo, revoked: removed.length });
-  }
-
-  const server = Bun.serve({
-    port,
-    async fetch(req, srv) {
-      const url = new URL(req.url);
-      const path = url.pathname;
-      const method = req.method;
-
-      // Strip the configured prefix (if any) before matching.
-      if (pathPrefix && !path.startsWith(pathPrefix)) return notFound();
-      const suffix = pathPrefix ? path.slice(pathPrefix.length) : path;
-
-      // Polite redirect for the bare registry root: serve a 301 →
-      // /modules/ on the celilo.computer site, with a minimal HTML
-      // body for clients that don't follow redirects (curl without
-      // -L, bots, bare HTTP libraries). Matches three forms so it
-      // works whether PATH_PREFIX is set or empty:
-      //   - suffix=''        → request was exactly the prefix, no slash
-      //   - suffix='/'       → request was prefix + '/', or just '/'
-      //   - suffix='/index.html'
-      // See apps/celilo/designs/REGISTRY_BROWSE_UI.md (decision D2).
-      if (method === 'GET' && (suffix === '' || suffix === '/' || suffix === '/index.html')) {
-        return landingResponse(publicUrl);
-      }
-
-      // Sparse config
-      if (method === 'GET' && suffix === '/index/config.json') {
-        return Response.json(sparseConfig);
-      }
-
-      // Sparse index files — the module name is the last path segment.
-      // Reject anything that isn't a valid module name so the storage layer
-      // can never see a path-traversal string via the index endpoint.
-      if (method === 'GET' && suffix.startsWith('/index/')) {
-        const name = decodeURIComponent(suffix.slice(suffix.lastIndexOf('/') + 1));
-        if (!isValidName(name)) return notFound();
-        const entries = resolveEntries(name);
-        if (entries.length === 0) return notFound();
-        const body = `${entries.map((e) => JSON.stringify(e)).join('\n')}\n`;
-        return new Response(body, {
-          headers: { 'Content-Type': 'text/plain; charset=utf-8' },
-        });
-      }
-
-      const apiBase = '/api/v1/modules';
-
-      // Search
-      if (method === 'GET' && suffix === apiBase) {
-        const q = url.searchParams.get('q')?.toLowerCase() ?? '';
-        const perPage = Math.min(Number(url.searchParams.get('per_page') ?? 25), 100);
-        // Sort options: 'name' (default, alphabetical) or 'downloads'
-        // (most-downloaded first, ties broken alphabetically). Anything
-        // unrecognized falls back to 'name'.
-        const sort = url.searchParams.get('sort') === 'downloads' ? 'downloads' : 'name';
-
-        const publishedNames = new Set(storage.listModules().map((m) => m.name));
-        const bootstrap = resolveBootstrap();
-        const allNames = new Set([...publishedNames, ...bootstrap.keys()]);
-        const filtered = [...allNames].filter((n) => !q || n.includes(q));
-
-        // Build the per-module records first so sorting can read the
-        // download counts. Empty input → empty output, no work done.
-        const records = filtered.map((name) => {
-          const entries = resolveEntries(name);
-          const latest = entries.filter((v) => !v.yanked).at(-1) ?? entries.at(-1);
-          // Description preference order:
-          //   1. Latest published version's stored description (best —
-          //      reflects the captured-at-publish-time manifest).
-          //   2. Bootstrap entry's manifest description (when serving
-          //      a module from a source directory the registry hasn't
-          //      formally received via publish).
-          //   3. Empty string (older publishes that pre-date the
-          //      description capture and haven't been republished).
-          const bootstrapEntry = bootstrap.get(name);
-          const description = latest?.description ?? bootstrapEntry?.description ?? '';
-          return {
-            name,
-            max_version: latest?.vers ?? '0.0.0',
-            description,
-            total_downloads: storage.getDownloads(name),
-          };
-        });
-
-        if (sort === 'downloads') {
-          records.sort((a, b) => {
-            if (b.total_downloads !== a.total_downloads) {
-              return b.total_downloads - a.total_downloads;
-            }
-            return a.name.localeCompare(b.name);
-          });
-        } else {
-          records.sort((a, b) => a.name.localeCompare(b.name));
-        }
-
-        return Response.json({
-          modules: records.slice(0, perPage),
-          total: records.length,
-        });
-      }
-
-      // Publish — rate-limit before even checking auth so a bad-token flood
-      // can't crowd out legitimate traffic on the shared buckets. The scope
-      // check happens inside handlePublish, once the package name is known.
-      if (method === 'PUT' && suffix === `${apiBase}/new`) {
-        const rl = rateLimitOrNull(req, srv);
-        if (rl) return rl;
-        return handlePublish(req);
-      }
-
-      // Mint / revoke per-repo scoped publish tokens (admin-only — ISS-0140).
-      if (method === 'POST' && suffix === `${apiBase}/tokens/mint`) {
-        const rl = rateLimitOrNull(req, srv);
-        if (rl) return rl;
-        return handleMintToken(req);
-      }
-      if (method === 'POST' && suffix === `${apiBase}/tokens/revoke`) {
-        const rl = rateLimitOrNull(req, srv);
-        if (rl) return rl;
-        return handleRevokeToken(req);
-      }
-
-      if (suffix.startsWith(`${apiBase}/`)) {
-        const modulePath = suffix.slice(apiBase.length + 1);
-        // URL-decode path segments — module versions use a literal '+' to
-        // separate semver from pkg revision (e.g. 1.0.0+1), and clients
-        // URL-encode that as %2B. Without decoding, storage lookups miss
-        // because they'd use the raw escape string.
-        const parts = modulePath.split('/').map((p) => decodeURIComponent(p));
-
-        // Module metadata
-        if (method === 'GET' && parts.length === 1 && parts[0]) {
-          const name = parts[0];
-          if (!isValidName(name)) return notFound();
-          const entries = resolveEntries(name);
-          if (entries.length === 0) return notFound();
-          // Description: latest non-yanked version's stored description,
-          // falling back to the bootstrap entry's manifest description
-          // for source-mode modules. Same precedence as the search
-          // endpoint — see notes there.
-          const latest = entries.filter((v) => !v.yanked).at(-1) ?? entries.at(-1);
-          const bootstrapEntry = resolveBootstrap().get(name);
-          const description = latest?.description ?? bootstrapEntry?.description ?? '';
-          return Response.json({
-            name,
-            description,
-            total_downloads: storage.getDownloads(name),
-            versions: entries.map((v) => ({
-              num: v.vers,
-              yanked: v.yanked,
-              created_at: new Date(0).toISOString(),
-            })),
-          });
-        }
-
-        // Download
-        if (method === 'GET' && parts.length === 3 && parts[2] === 'download') {
-          const [name, version] = parts;
-          if (!isValidName(name) || !isValidVersion(version)) return notFound();
-          const data = storage.readPackage(name, version);
-          if (data) {
-            // Bump the per-module counter on every successful download.
-            // Liberal counting: no User-Agent filter, no de-duplication.
-            // CI/crawler traffic inflates the number but the simplicity
-            // is worth it — see REGISTRY_BROWSE_UI.md (Phase 3).
-            storage.incrementDownloads(name);
-            return new Response(new Uint8Array(data), {
-              headers: {
-                'Content-Type': 'application/octet-stream',
-                'Content-Disposition': contentDispositionAttachment(`${name}-${version}.netapp`),
-              },
-            });
-          }
-          // Bootstrap fallback — published packages take precedence above.
-          const b = resolveBootstrap().get(name);
-          if (b) {
-            storage.incrementDownloads(name);
-            return serveBootstrapDownload(b);
-          }
-          return notFound();
-        }
-
-        // Yank
-        if (method === 'DELETE' && parts.length === 3 && parts[2] === 'yank') {
-          const rl = rateLimitOrNull(req, srv);
-          if (rl) return rl;
-          const [name, version] = parts;
-          if (!isValidName(name) || !isValidVersion(version)) return notFound();
-          if (!authorizePackage(req, name)) return unauthorized();
-          return setYanked(name, version, true);
-        }
-
-        // Unyank
-        if (method === 'PUT' && parts.length === 3 && parts[2] === 'unyank') {
-          const rl = rateLimitOrNull(req, srv);
-          if (rl) return rl;
-          const [name, version] = parts;
-          if (!isValidName(name) || !isValidVersion(version)) return notFound();
-          if (!authorizePackage(req, name)) return unauthorized();
-          return setYanked(name, version, false);
-        }
-      }
-
-      return notFound();
-    },
-  });
-
-  return server;
-}
-
-/** Env-var entry point used by bin/celilo-registry-server and the Dockerfile. */
-export function startFromEnv(): void {
-  const dataDir = process.env.DATA_DIR ?? '/var/lib/celilo-registry';
-  const port = Number(process.env.PORT ?? 3000);
-  const pathPrefix = (process.env.PATH_PREFIX ?? '').replace(/\/$/, '');
-  const publicUrl =
-    process.env.PUBLIC_URL ?? `https://${process.env.DOMAIN ?? 'celilo.computer'}${pathPrefix}`;
-  const bootstrapModulesDir = process.env.BOOTSTRAP_MODULES_DIR || undefined;
-  const bootstrapUploadsDir = process.env.BOOTSTRAP_UPLOADS_DIR || undefined;
-  const bootstrapCacheDir = process.env.BOOTSTRAP_CACHE_DIR || undefined;
-  const auth = TokenAuth.fromEnv();
-
-  const server = startServer({
-    dataDir,
-    port,
-    pathPrefix,
-    publicUrl,
-    bootstrapModulesDir,
-    bootstrapUploadsDir,
-    bootstrapCacheDir,
-    auth,
-  });
-
-  const bootstrapBits = [
-    bootstrapModulesDir ? `modulesDir=${bootstrapModulesDir}` : '',
-    bootstrapUploadsDir ? `uploadsDir=${bootstrapUploadsDir}` : '',
-  ]
-    .filter(Boolean)
-    .join(', ');
-  console.log(
-    `[registry] listening on :${server.port} (prefix=${pathPrefix || '(none)'}, publicUrl=${publicUrl}${
-      bootstrapBits ? `, bootstrap=[${bootstrapBits}]` : ''
-    }, auth=${auth.hasTokens() ? 'enabled' : 'disabled (publish will always 401)'})`,
-  );
-}
-
-// Invoke when run as an entry point (`bun run src/server.ts`).
-if (import.meta.main) {
-  startFromEnv();
-}