@camstack/server 0.1.3

package/src/main.ts

@@ -0,0 +1,1049 @@
+/* eslint-disable @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument -- pre-existing lint debt across this 800+ line bootstrap module. The flagged sites cross typed boundaries (Fastify request typing, AddonRouteRegistry, AuthService inherited methods) where the projectService context can't trace inheritance chains. Tracked separately; do not amend in unrelated edits. */
+import { fastifyTRPCPlugin } from "@trpc/server/adapters/fastify";
+import { applyWSSHandler } from "@trpc/server/adapters/ws";
+import type { CreateWSSContextFnOptions } from "@trpc/server/adapters/ws";
+import fastifyStatic from "@fastify/static";
+import fastifyCookie from "@fastify/cookie";
+import { WebSocketServer } from "ws";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { execSync } from "node:child_process";
+import { LoggingService } from "./core/logging/logging.service";
+import { EventBusService } from "./core/events/event-bus.service";
+import { ConfigService } from "./core/config/config.service";
+import { AuthService } from "./core/auth/auth.service";
+
+// Boot-time capability declaration runs over the auto-generated
+// `ALL_CAPABILITY_DEFINITIONS` array — every `*.cap.ts` file that ships
+// with `@camstack/types` is included automatically. Adding a new cap
+// requires no edit to this file: drop the `*.cap.ts` and re-run
+// `npx tsx scripts/generate-capability-router-types.ts`. The previous
+// hand-curated list silently dropped caps (zones, zone-rules,
+// zone-analytics, audio-metrics — see git for the regression).
+import { ALL_CAPABILITY_DEFINITIONS } from "@camstack/types";
+import { StreamProbeService } from "./core/streaming/stream-probe.service";
+import { FeatureService } from "./core/feature/feature.service";
+import { AgentRegistryService } from "./core/agent/agent-registry.service";
+import { MoleculerService } from "./core/moleculer/moleculer.service";
+import { AddonRegistryService } from "./core/addon/addon-registry.service";
+import { AddonPackageService } from "./core/addon/addon-package.service";
+import { ReplEngineService } from "./core/repl/repl-engine.service";
+import { NetworkQualityService } from "./core/network/network-quality.service";
+import { StorageService } from "./core/storage/storage.service";
+import { AddonBridgeService } from "./core/addon-bridge/addon-bridge.service";
+import { AddonPagesService } from "./core/addon-pages/addon-pages.service";
+import { AddonWidgetsService } from "./core/addon-widgets/addon-widgets.service";
+import { buildAppRouter } from "./api/trpc/trpc.router";
+import { buildCoreCapService } from "./api/trpc/core-cap-bridge";
+import {
+  createTrpcContext,
+  createWsTrpcContext,
+} from "./api/trpc/trpc.context";
+import { registerAddonUploadRoute } from "./api/addon-upload";
+import { registerAuthWhoamiRoute } from "./api/auth-whoami";
+import { buildSessionCookie, clearSessionCookie, SESSION_COOKIE, shouldRedirectToLogin, loginRedirectUrl } from "./auth/session-cookie.js";
+import { registerHealthRoutes } from "./api/health/health.routes";
+import { registerOauth2Routes } from "./api/oauth2/oauth2-routes.js";
+import {
+  AddonRouteRegistry,
+} from "@camstack/core";
+import type { FastifyRequest, FastifyReply } from "fastify";
+import { loadBootstrapConfig, setupInfra } from "./boot/boot-config";
+import { bootManual } from "./manual-boot";
+
+import { PostBootService } from "./boot/post-boot.service";
+
+// ---- Process-level error handlers ----
+
+process.on("uncaughtException", (err) => {
+  console.error(
+    "[uncaughtException] Unhandled exception — server will continue:",
+    err,
+  );
+});
+
+process.on("unhandledRejection", (reason) => {
+  console.error(
+    "[unhandledRejection] Unhandled promise rejection — server will continue:",
+    reason,
+  );
+});
+
+// ---- Graceful shutdown ----
+
+// Kill all child processes immediately on signal, then let Fastify clean up.
+// This is critical because tsx watch (dev) force-kills the main process after a short
+// timeout — if we wait for Fastify OnModuleDestroy, children become orphans.
+let shutdownStarted = false;
+function immediateChildCleanup(signal: string) {
+  if (shutdownStarted) return;
+  shutdownStarted = true;
+
+  console.log(`[shutdown] Received ${signal} — killing child processes immediately…`);
+
+  // Synchronously kill all children of this process via the OS.
+  // This is fast and ensures no orphans even if Fastify shutdown is slow.
+  try {
+    const myPid = process.pid;
+    const isMac = process.platform === "darwin";
+    // pkill sends SIGTERM to all processes whose parent is our PID
+    const cmd = isMac
+      ? `pkill -TERM -P ${myPid} 2>/dev/null; true`
+      : `kill -- -${myPid} 2>/dev/null; true`;
+    execSync(cmd, { timeout: 3000 });
+    console.log("[shutdown] Child processes signalled");
+  } catch {
+    // Best effort — children may have already exited
+  }
+
+  // Safety: force exit after 10s if Fastify shutdown stalls
+  const timer = setTimeout(() => {
+    console.error("[shutdown] Graceful shutdown timed out after 10s — forcing exit");
+    process.exit(1);
+  }, 10_000);
+  timer.unref();
+}
+
+process.on("SIGTERM", () => immediateChildCleanup("SIGTERM"));
+process.on("SIGINT", () => immediateChildCleanup("SIGINT"));
+
+// ---- Orphan cleanup ----
+
+/**
+ * Kill leftover camstack processes from a previous crash.
+ * Targets: coreml_inference.py (Python engines).
+ * Only kills processes whose parent is PID 1 (orphaned) to avoid killing children of a live server.
+ */
+function cleanupOrphanProcesses(): void {
+  const isMac = process.platform === "darwin";
+  const isLinux = process.platform === "linux";
+  if (!isMac && !isLinux) return;
+
+  let killed = 0;
+
+  try {
+    // Find orphaned coreml_inference.py processes (ppid=1 means orphaned)
+    const cmd = isMac
+      ? "ps -eo pid,ppid,command | grep -E 'coreml_inference\\.py' | grep -v grep"
+      : "ps -eo pid,ppid,cmd | grep -E 'coreml_inference\\.py' | grep -v grep";
+
+    const output = execSync(cmd, { encoding: "utf8", timeout: 5000 }).trim();
+    if (!output) return;
+
+    for (const line of output.split("\n")) {
+      const parts = line.trim().split(/\s+/);
+      const pid = parseInt(parts[0]!, 10);
+      const ppid = parseInt(parts[1]!, 10);
+
+      // Only kill orphans (ppid=1) — never kill children of a running server
+      if (ppid !== 1 || isNaN(pid) || pid === process.pid) continue;
+
+      try {
+        process.kill(pid, "SIGTERM");
+        killed++;
+      } catch {
+        // Process may have already exited
+      }
+    }
+  } catch {
+    // grep returns exit code 1 when no matches — that's fine
+  }
+
+  if (killed > 0) {
+    console.log(`[cleanup] Killed ${killed} orphaned camstack process(es) from a previous run`);
+  }
+}
+
+// ---- Bootstrap ----
+
+async function bootstrap() {
+  // Clean up orphaned processes from previous crashes before starting
+  cleanupOrphanProcesses();
+
+  // SPA fallback — set later when admin UI is resolved, used by addon route catch-all
+  let spaIndexHtml: string | null = null;
+
+  // --- Phase 1 + 2: Load config, setup storage locations, TLS ---
+  const configPath =
+    process.env.CONFIG_PATH ??
+    path.join(process.cwd(), "camstack-data", "config.yaml");
+  const bootstrapConfig = loadBootstrapConfig(configPath);
+  const infra = await setupInfra(configPath, bootstrapConfig);
+
+  const { dataPath, tlsOptions } = infra;
+  const port = infra.bootstrapConfig.server.port;
+  const host = infra.bootstrapConfig.server.host;
+
+  // --- Phase 3: Create app + tRPC register + listen ---
+  const fastifyOpts: Record<string, unknown> = tlsOptions
+    ? { https: tlsOptions }
+    : {};
+  const app = await bootManual({ infra, fastifyOpts });
+  app.enableShutdownHooks();
+  app.enableCors();
+
+  const fastify = app.getHttpAdapter().getInstance();
+  await fastify.register(fastifyCookie);
+
+  // Make LocationManager available to StorageService before lifecycle hooks run
+  const storageService = app.get(StorageService);
+  storageService.setLocationManager(infra.locationManager);
+
+  // ConfigService — SettingsStore is wired later by the settings-store capability consumer
+  const config = app.get(ConfigService);
+
+  // Register addon upload route (multipart — must be registered before tRPC).
+  // We pass AddonRegistryService so the handler can resolve the
+  // `user-management` cap singleton at request time (the only working
+  // path to validate `cst_*` scoped tokens — see addon-upload.ts).
+  try {
+    const uploadAuthService = app.get(AuthService);
+    const uploadAddonBridge = app.get(AddonBridgeService);
+    const uploadMoleculer = app.get(MoleculerService);
+    const uploadAddonRegistry = app.get(AddonRegistryService);
+    const uploadLogger = app.get(LoggingService).createLogger("addon-upload");
+    await registerAddonUploadRoute(
+      fastify,
+      uploadAddonBridge,
+      uploadAuthService,
+      uploadMoleculer,
+      uploadAddonRegistry,
+      uploadLogger,
+    );
+    console.log(
+      "[bootstrap] Addon upload route registered at POST /api/addons/upload",
+    );
+    // Companion endpoint: /api/auth/whoami — validates JWT or cst_*
+    // scoped tokens, returns the resolved identity + scope summary.
+    // Mirrors the addon-upload auth chain so the CLI can ping for
+    // token-still-valid without consuming a real cap.
+    await registerAuthWhoamiRoute(
+      fastify,
+      uploadAuthService,
+      uploadAddonRegistry,
+    );
+    console.log(
+      "[bootstrap] Auth whoami route registered at GET /api/auth/whoami",
+    );
+  } catch (err) {
+    console.warn("[bootstrap] Failed to register addon upload route:", err);
+  }
+
+  // Register tRPC plugin on Fastify BEFORE listen.
+  // If registration fails, start in degraded mode: serve a health warning endpoint.
+  // Instantiate new core services
+  const loggingService = app.get(LoggingService);
+  const addonRouteRegistry = new AddonRouteRegistry();
+
+  // Use Fastify-managed notification/toast wrappers (globally provided by NotificationModule)
+  const { NotificationServiceWrapper } =
+    await import("./core/notification/notification-wrapper.service");
+  const { ToastServiceWrapper } =
+    await import("./core/notification/toast-wrapper.service");
+  const notificationWrapper = app.get(NotificationServiceWrapper);
+  const toastWrapper = app.get(ToastServiceWrapper);
+  // Expose the underlying core services for the tRPC router
+  const notificationService = notificationWrapper.service;
+  const toastService = toastWrapper.service;
+
+  // Wire AddonRouteRegistry and NotificationService
+  const addonRegistry = app.get(AddonRegistryService);
+  addonRegistry.setAddonRouteRegistry(addonRouteRegistry);
+
+  // ── Configure the CapabilityRegistry (created in AddonRegistryService constructor) ──
+  const capabilityRegistry = addonRegistry.getCapabilityRegistry();
+  capabilityRegistry.setConfigManager(config);
+
+  // Declare every shipped capability BEFORE app.init() so addon
+  // registerProvider() calls (in-process or via the Moleculer bridge
+  // from forked workers / agents) find their definition during
+  // onModuleInit. The list comes from the codegen — see the import
+  // comment above for the rationale.
+  for (const capDef of ALL_CAPABILITY_DEFINITIONS) {
+    capabilityRegistry.declareCapability(capDef);
+  }
+
+  // Hub-internal `sso-bridge` provider — gives auth-provider addons
+  // (OIDC, SAML, magic-link, …) a typed gateway to mint an HMAC-signed
+  // bridge token before redirecting to `/api/auth/sso/finish`. Without
+  // this, the finish endpoint would have to trust unsigned query params
+  // (anyone could craft `?isAdmin=1`). Backed by AuthService.
+  const authServiceForBridge = app.get(AuthService);
+  capabilityRegistry.registerProvider('sso-bridge', '$hub', {
+    signBridgeToken: async ({ claims, ttlSec }: { claims: { userId: string; username: string; isAdmin: boolean; provider: string; email?: string; displayName?: string }; ttlSec?: number }) => {
+      const token = authServiceForBridge.signSsoBridgeToken(claims, ttlSec ?? 300);
+      return { token };
+    },
+    verifyBridgeToken: async ({ token }: { token: string }) => {
+      return authServiceForBridge.verifySsoBridgeToken(token);
+    },
+  });
+
+  // Wire registry into NotificationService for proxy-based output resolution
+  notificationService.setRegistry(capabilityRegistry);
+
+  // Hub metrics and sub-process info now come from Moleculer service discovery
+  // and the distributed metrics-provider capability — no manual wiring needed.
+
+  let trpcRegistered = false;
+  let appRouter: ReturnType<typeof buildAppRouter> | null = null;
+
+  // Run app.init() FIRST so onModuleInit hooks complete (PipelineWiring, AddonBridge, etc.)
+  // before we build the tRPC router which depends on their results.
+  await app.init();
+  console.log("[bootstrap] app.init() complete — all onModuleInit hooks have run");
+
+  // Mark registry as ready — providers from app.init() are now registered
+  capabilityRegistry.ready();
+
+  // Register log-receiver service for agent log forwarding (after app.init
+  // so Moleculer re-advertises the service list to the network)
+  const moleculer = app.get(MoleculerService);
+  moleculer.registerLogReceiver();
+
+  // ── Health routes (hub self + agent forwarding) ──────────────────
+  // Registered after app.init() so the AgentRegistryService is wired and
+  // the Moleculer broker is ready to forward `$agent.health` calls.
+  try {
+    const hubVersion = readHubVersion();
+    registerHealthRoutes(fastify, {
+      moleculer,
+      agentRegistry: app.get(AgentRegistryService),
+      hubVersion,
+    });
+    console.log(`[bootstrap] Health routes registered (hub v${hubVersion})`);
+  } catch (err) {
+    console.warn("[bootstrap] Failed to register health routes:", err);
+  }
+
+  // Seed the device-name cache the log formatter consults so logs
+  // emitted before the first DeviceRegistered already resolve names.
+  // Subsequent registers/unregisters flow through the event-bus
+  // subscription installed in `LoggingService.attachDeviceNameStream`.
+  try {
+    const dm = capabilityRegistry.getSingleton('device-manager') as {
+      listAll?: (input: { addonId?: string }) => Promise<readonly { id: number; name: string }[]>
+    } | null
+    if (dm?.listAll) {
+      const devices = await dm.listAll({})
+      loggingService.setDeviceNames(devices.map((d) => ({ id: d.id, name: d.name })))
+      loggingService.createLogger('logging').info('device-name cache seeded from device-manager', {
+        meta: { count: devices.length, ids: devices.map((d) => d.id) },
+      })
+    } else {
+      loggingService.createLogger('logging').warn('device-name cache seed skipped — device-manager not available')
+    }
+  } catch (err) {
+    console.warn('[bootstrap] device-name cache seed skipped:', err instanceof Error ? err.message : err)
+  }
+
+  try {
+    const authService = app.get(AuthService);
+
+    appRouter = buildAppRouter({
+      authService,
+      configService: config,
+      featureService: app.get(FeatureService),
+
+      loggingService,
+      eventBus: app.get(EventBusService),
+      agentRegistry: app.get(AgentRegistryService),
+      moleculer: app.get(MoleculerService),
+      addonRegistry,
+      replEngine: app.get(ReplEngineService),
+      networkQualityService: app.get(NetworkQualityService),
+      addonBridge: app.get(AddonBridgeService),
+      addonPackageService: app.get(AddonPackageService),
+      notificationService,
+      toastService,
+      capabilityRegistry,
+      streamProbe: app.get(StreamProbeService),
+    });
+
+    await fastify.register(fastifyTRPCPlugin, {
+      prefix: "/trpc",
+      trpcOptions: {
+        router: appRouter,
+          createContext: ({ req }: { req: FastifyRequest }) =>
+          createTrpcContext(req, authService, addonRegistry),
+        onError: ({ path, error }: { path: string | undefined; error: { code: string; message: string; cause?: unknown } }) => {
+          const trpcLogger = app.get(LoggingService).createLogger("tRPC");
+          trpcLogger.warn('tRPC error', { meta: { code: error.code, path: path ?? '?', message: error.message } });
+          if (error.cause) trpcLogger.warn('tRPC error cause', { meta: { cause: error.cause instanceof Error ? error.cause.message : JSON.stringify(error.cause) } });
+        },
+      },
+    });
+    trpcRegistered = true;
+
+    // Mount the `$core-caps` Moleculer service so forked addons /
+    // remote agents can reach the hub's core (non-addon) tRPC routers
+    // through `ctx.api.<coreCap>`. Without it those calls hang in
+    // `brokerTransportLink`'s unbounded discovery wait.
+    try {
+      moleculer.registerCoreCapService(buildCoreCapService(appRouter));
+      console.log("[bootstrap] core-cap mesh bridge registered ($core-caps)");
+    } catch (err) {
+      console.warn("[bootstrap] Failed to register core-cap mesh bridge:", err);
+    }
+
+    // Register addon-pages static file endpoint
+    const addonPagesService = app.get(AddonPagesService);
+    fastify.get<{ Params: { addonId: string; "*": string } }>(
+      "/api/addon-pages/:addonId/*",
+      async (request, reply) => {
+        const { addonId } = request.params;
+        const filePath = request.params["*"];
+        if (!filePath) {
+          return reply.status(400).send("Missing file path");
+        }
+        const resolved = addonPagesService.resolveBundle(addonId, filePath);
+        if (!resolved) {
+          return reply.status(404).send("Not found");
+        }
+        const ext = path.extname(resolved).toLowerCase();
+        const contentType =
+          ext === ".js" || ext === ".mjs"
+            ? "application/javascript"
+            : ext === ".css"
+              ? "text/css"
+              : ext === ".json"
+                ? "application/json"
+                : ext === ".html"
+                  ? "text/html"
+                  : "application/octet-stream";
+        const stream = fs.createReadStream(resolved);
+        return reply.type(contentType).send(stream);
+      },
+    );
+
+    // Register addon-widgets static file endpoint — same shape as
+    // addon-pages but reads bundles from each addon's
+    // `dist/widgets.mjs` (declared per-widget in `addon-widgets-source`
+    // metadata). Path-traversal protection + cap-registration check
+    // both live in `AddonWidgetsService.resolveBundle`.
+    const addonWidgetsService = app.get(AddonWidgetsService);
+    fastify.get<{ Params: { addonId: string; "*": string } }>(
+      "/api/addon-widgets/:addonId/*",
+      async (request, reply) => {
+        const { addonId } = request.params;
+        const filePath = request.params["*"];
+        if (!filePath) {
+          return reply.status(400).send("Missing file path");
+        }
+        const resolved = addonWidgetsService.resolveBundle(addonId, filePath);
+        if (!resolved) {
+          return reply.status(404).send("Not found");
+        }
+        const ext = path.extname(resolved).toLowerCase();
+        const contentType =
+          ext === ".js" || ext === ".mjs"
+            ? "application/javascript"
+            : ext === ".css"
+              ? "text/css"
+              : ext === ".json"
+                ? "application/json"
+                : ext === ".html"
+                  ? "text/html"
+                  : "application/octet-stream";
+        const stream = fs.createReadStream(resolved);
+        return reply.type(contentType).send(stream);
+      },
+    );
+
+    // Serve static assets (e.g. SVG icons) from an addon's package directory
+    fastify.get<{ Params: { addonId: string; "*": string } }>(
+      "/api/addon-assets/:addonId/*",
+      async (request, reply) => {
+        const { addonId } = request.params;
+        const assetPath = request.params["*"] ?? "";
+
+        const packageDir = addonRegistry.getAddonPackageDir(addonId);
+        if (!packageDir) {
+          return reply.code(404).send({ error: "Addon not found" });
+        }
+
+        const resolvedPackageDir = path.resolve(packageDir);
+        const filePath = path.resolve(resolvedPackageDir, assetPath);
+
+        // Security: prevent path traversal attacks
+        if (
+          !filePath.startsWith(resolvedPackageDir + path.sep) &&
+          filePath !== resolvedPackageDir
+        ) {
+          return reply.code(403).send({ error: "Access denied" });
+        }
+
+        if (!fs.existsSync(filePath)) {
+          return reply.code(404).send({ error: "Asset not found" });
+        }
+
+        const ext = path.extname(filePath).toLowerCase();
+        const contentTypes: Record<string, string> = {
+          ".svg": "image/svg+xml",
+          ".png": "image/png",
+          ".jpg": "image/jpeg",
+          ".jpeg": "image/jpeg",
+          ".json": "application/json",
+          ".webp": "image/webp",
+        };
+        const contentType = contentTypes[ext] ?? "application/octet-stream";
+
+        reply.header("content-type", contentType);
+        reply.header("cache-control", "public, max-age=86400");
+        return reply.send(fs.readFileSync(filePath));
+      },
+    );
+
+    // ── SSO finish endpoint ─────────────────────────────────────────
+    // OIDC / SAML / external auth providers redirect here after their
+    // own callback handler validated the IdP response. We trust the
+    // claims because:
+    //   1. The path is server-internal — providers run in-process and
+    //      issue the redirect with their own validated state.
+    //   2. The query string is consumed once and immediately swapped
+    //      for a JWT in the URL fragment (#token=...) which never hits
+    //      the server logs and isn't replayable.
+    //
+    // The provider prefixes the redirect with `provider=auth-oidc` so
+    // we can attribute the session. The mint is currently best-effort
+    // — a future hardening should require the provider to also pass a
+    // nonce signed with the auth.jwtSecret so we can verify the redirect
+    // came from the addon's own callback handler.
+    fastify.get<{
+      Querystring: {
+        bridge?: string
+      }
+    }>(
+      "/api/auth/sso/finish",
+      async (request, reply) => {
+        // The auth-provider addon (OIDC, SAML, …) mints an HMAC-signed
+        // bridge token via the `sso-bridge` cap and 302s to here with
+        // `?bridge=<jwt>`. We verify the signature with the same JWT
+        // secret used elsewhere — only then do we trust the claims
+        // (including the `isAdmin` flag). This closes the prior gap
+        // where any client could call `?isAdmin=1` and become admin.
+        const bridge = request.query.bridge
+        if (!bridge) {
+          return reply.status(400).send({ error: "Missing bridge token" })
+        }
+        const ssoAuth = app.get(AuthService)
+        const claims = ssoAuth.verifySsoBridgeToken(bridge)
+        if (!claims) {
+          return reply.status(401).send({ error: "Invalid or expired bridge token" })
+        }
+
+        try {
+          const token = ssoAuth.signToken({
+            userId: claims.userId,
+            username: claims.username,
+            isAdmin: claims.isAdmin,
+            allowedProviders: '*',
+            allowedDevices: {},
+          })
+          // Redirect to the admin UI with the token in the URL fragment.
+          // Fragments don't hit the server log + don't get sent on
+          // subsequent requests — the SPA's auth-context picks the
+          // token up on mount and stores it in localStorage.
+          reply.code(302)
+          reply.header(
+            "Location",
+            `/admin/login#token=${encodeURIComponent(token)}&provider=${encodeURIComponent(claims.provider)}`,
+          )
+          return reply.send("")
+        } catch (err) {
+          return reply.status(500).send({
+            error: "SSO finish failed",
+            message: err instanceof Error ? err.message : String(err),
+          })
+        }
+      },
+    )
+
+    // ── Backup archive download (Phase 4 / Task 24) ────────────────
+    // Streams an archive at `<locationId>/<archiveId>` through the
+    // storage cap's chunked-download protocol straight to an HTTP
+    // response. The admin UI uses an authenticated `fetch` + Blob to
+    // fetch and trigger a save dialog (no cookie auth on this server,
+    // so we can't rely on `window.location.assign`).
+    fastify.get<{ Params: { locationId: string; archiveId: string } }>(
+      "/api/backup/download/:locationId/:archiveId",
+      async (request, reply) => {
+        const authHeader = request.headers.authorization;
+        if (!authHeader) {
+          return reply.status(401).send({ error: "Unauthorized" });
+        }
+        try {
+          const token = authHeader.replace("Bearer ", "");
+          const downloadAuth = app.get(AuthService);
+          const payload = downloadAuth.verifyToken(token);
+          if (!payload.isAdmin) {
+            return reply.status(403).send({ error: "Admin required" });
+          }
+        } catch {
+          return reply.status(401).send({ error: "Invalid token" });
+        }
+
+        const { locationId, archiveId } = request.params;
+        const backupSingleton = capabilityRegistry.getSingleton<{
+          listArchives: (input: { destinationId: string }) => Promise<readonly { id: string; filename: string; sizeBytes: number; label?: string }[]>
+        }>("backup");
+        if (!backupSingleton?.listArchives) {
+          return reply.status(503).send({ error: "Backup orchestrator unavailable" });
+        }
+        const archives = await backupSingleton.listArchives({ destinationId: locationId });
+        const archive = archives.find((a) => a.id === archiveId);
+        if (!archive) {
+          return reply.status(404).send({ error: "Archive not found" });
+        }
+
+        const storage = capabilityRegistry.getSingleton<{
+          beginDownload: (input: { location: string; relativePath: string }) => Promise<{ downloadId: string; sizeBytes: number }>
+          readChunk: (input: { downloadId: string; offset: number; length: number }) => Promise<Uint8Array>
+          endDownload: (input: { downloadId: string }) => Promise<void>
+        }>("storage");
+        if (!storage?.beginDownload) {
+          return reply.status(503).send({ error: "Storage cap unavailable" });
+        }
+
+        const downloadName = `${archive.label ?? archive.id}.tar.gz`;
+        // Sanitize: strip newlines and double-quotes which would break
+        // the Content-Disposition header. Whitespace is fine.
+        const safeName = downloadName.replace(/[\r\n"]/g, "_");
+        reply.header("content-type", "application/gzip");
+        reply.header("content-length", String(archive.sizeBytes));
+        reply.header(
+          "content-disposition",
+          `attachment; filename="${safeName}"`,
+        );
+
+        const { downloadId, sizeBytes } = await storage.beginDownload({
+          location: locationId,
+          relativePath: archive.filename,
+        });
+        const CHUNK = 8 * 1024 * 1024;
+        try {
+          let offset = 0;
+          // Stream the chunked response. Fastify's `reply.raw` is the
+          // Node.js writable; we write each chunk and `end()` once
+          // we've drained the source. Per-write back-pressure is
+          // handled inside the runtime — buffered writes pile up but
+          // never beyond the OS socket buffer.
+          while (offset < sizeBytes) {
+            const len = Math.min(CHUNK, sizeBytes - offset);
+            const chunk = await storage.readChunk({ downloadId, offset, length: len });
+            if (chunk.byteLength === 0) {
+              throw new Error(
+                `backup download: empty chunk at offset ${offset}/${sizeBytes}`,
+              );
+            }
+            reply.raw.write(chunk);
+            offset += chunk.byteLength;
+          }
+          reply.raw.end();
+        } catch (err) {
+          console.error("[backup-download] stream failed:", err);
+          // Headers may already be flushed — abort the socket if so.
+          if (!reply.raw.headersSent) {
+            return reply.status(500).send({ error: "Download failed" });
+          }
+          reply.raw.destroy(err instanceof Error ? err : new Error(String(err)));
+        } finally {
+          try {
+            await storage.endDownload({ downloadId });
+          } catch {
+            /* best-effort */
+          }
+        }
+        return reply;
+      },
+    );
+
+    // POST /api/auth/session — upgrade a tRPC-issued JWT to a browser cookie.
+    fastify.post<{ Body: { token?: string } }>('/api/auth/session', async (request, reply) => {
+      const token = request.body?.token
+      if (!token) return reply.status(400).send({ error: 'token required' })
+      let ttlSec: number
+      try {
+        const payload = authService.verifyToken(token)   // throws on invalid/expired
+        const expSec = typeof payload.exp === 'number' ? payload.exp : 0
+        ttlSec = Math.max(0, expSec - Math.floor(Date.now() / 1000))
+      } catch {
+        return reply.status(401).send({ error: 'invalid token' })
+      }
+      const c = buildSessionCookie(token, ttlSec)
+      reply.setCookie(c.name, c.value, c.options)
+      return reply.send({ ok: true })
+    })
+
+    // DELETE /api/auth/session — clear the cookie on logout.
+    fastify.delete('/api/auth/session', async (_request, reply) => {
+      const c = clearSessionCookie()
+      reply.setCookie(c.name, c.value, c.options)
+      return reply.send({ ok: true })
+    })
+
+    // Addon HTTP API route catch-all: /addon/:addonId/*
+    // Only handles non-GET or routes that actually exist in the addon route registry.
+    // GET requests that don't match an addon route are SPA pages — handled by the /* fallback.
+    fastify.all<{
+      Params: { addonId: string; "*": string }
+      Querystring: Record<string, string>
+      Headers: Record<string, string>
+    }>("/addon/:addonId/*", async (request, reply) => {
+      const { addonId } = request.params;
+      const subPath = request.params["*"] ?? "";
+      const method = request.method;
+      const fullPath = `/addon/${addonId}/${subPath}`;
+      const query: Record<string, string> = request.query ?? {};
+      const headers: Record<string, string> = {};
+      for (const [k, v] of Object.entries(request.headers)) {
+        if (typeof v === "string") headers[k] = v;
+        else if (Array.isArray(v)) headers[k] = v.join(",");
+      }
+
+      const match = addonRouteRegistry.matchRoute(method, fullPath);
+      if (!match) {
+        if (method === "GET" && spaIndexHtml) {
+          return reply.type("text/html").send(fs.createReadStream(spaIndexHtml));
+        }
+        return reply.status(404).send({ error: "Not found" });
+      }
+
+      // Auth check based on route.access
+      if (match.route.access !== "public") {
+        const authHeader = request.headers.authorization;
+        // Browser navigation has no Authorization header — fall back to the
+        // session cookie, and if that is missing bounce to the login page.
+        const cookieToken = request.cookies?.[SESSION_COOKIE];
+        if (!authHeader && !cookieToken) {
+          if (shouldRedirectToLogin(request.method, request.headers.accept)) {
+            const qs = request.url.includes('?') ? request.url.slice(request.url.indexOf('?')) : '';
+            return reply.redirect(loginRedirectUrl(fullPath + qs));
+          }
+          return reply.status(401).send({ error: "Unauthorized" });
+        }
+        const token = authHeader ? authHeader.replace("Bearer ", "") : cookieToken!;
+        if (token.startsWith("cst_")) {
+          const userMgmt = capabilityRegistry?.getSingleton('user-management');
+          if (!userMgmt) return reply.status(503).send({ error: "User management not available" });
+          const scopedToken = await userMgmt.validateScopedToken({ token });
+          if (!scopedToken) {
+            return reply.status(401).send({ error: "Invalid token" });
+          }
+          // v2 model: scoped tokens grant by cap-category/cap-name/addon.
+          // For addon-route REST endpoints we match the `addon` scope
+          // type only — generic route-prefix scopes were dropped with
+          // the caps-only refactor.
+          const scopeMatch = scopedToken.scopes.some((scope) => {
+            return scope.type === 'addon' && scope.target === addonId;
+          });
+          if (!scopeMatch) {
+            return reply.status(403).send({ error: "Token scope mismatch" });
+          }
+          // Build addon request with scoped token context
+          const addonRequest = {
+            params: match.params,
+            query,
+            body: request.body,
+            headers,
+            scopedToken: {
+              id: scopedToken.id,
+              userId: scopedToken.userId,
+              scopes: scopedToken.scopes,
+            },
+          };
+          const addonReply = buildAddonReply(reply);
+          return match.route.handler(addonRequest, addonReply);
+        } else {
+          try {
+            const payload = authService.verifyToken(token);
+            if (match.route.access === "admin" && !payload.isAdmin) {
+              return reply.status(403).send({ error: "Admin required" });
+            }
+            const addonRequest = {
+              params: match.params,
+              query,
+              body: request.body,
+              headers,
+              user: {
+                id: payload.userId ?? "unknown",
+                username: payload.username ?? "unknown",
+                isAdmin: payload.isAdmin,
+              },
+            };
+            const addonReply = buildAddonReply(reply);
+            return match.route.handler(addonRequest, addonReply);
+          } catch {
+            return reply.status(401).send({ error: "Invalid token" });
+          }
+        }
+      }
+
+      // Public route — no auth required
+      const addonRequest = {
+        params: match.params,
+        query,
+        body: request.body,
+        headers,
+      };
+      const addonReply = buildAddonReply(reply);
+      return match.route.handler(addonRequest, addonReply);
+    });
+
+    // ── OAuth2 authorization endpoints ─────────────────────────────────────
+    // Mounted under /api/oauth2/* so they sit inside the universal "/api/"
+    // namespace already excluded from the SPA catch-all everywhere.
+    // getRegistry is a closure so it always resolves the live registry at
+    // request time (safe even if called before ready()).
+    registerOauth2Routes(fastify, {
+      getRegistry: () => capabilityRegistry,
+      verifyToken: (t) => authService.verifyToken(t),
+      publicHubUrl: () => process.env.CAMSTACK_PUBLIC_ORIGIN ?? `https://localhost:${port}`,
+    });
+    console.log('[bootstrap] OAuth2 routes registered at /api/oauth2/*');
+
+    // Attach tRPC WebSocket handler using noServer mode to avoid
+    // Fastify intercepting the upgrade request with a 400 response.
+    const wss = new WebSocketServer({ noServer: true });
+    applyWSSHandler({
+      wss,
+      router: appRouter,
+      createContext: (opts: CreateWSSContextFnOptions) => createWsTrpcContext(opts, authService, addonRegistry),
+      onError: ({ path, error }: { path: string | undefined; error: { code: string; message: string; cause?: unknown } }) => {
+        const trpcLogger = app.get(LoggingService).createLogger("tRPC:ws");
+        trpcLogger.warn('tRPC error', { meta: { code: error.code, path: path ?? '?', message: error.message } });
+        if (error.cause) trpcLogger.warn('tRPC error cause', { meta: { cause: error.cause instanceof Error ? error.cause.message : JSON.stringify(error.cause) } });
+      },
+    });
+
+    // Manually handle HTTP upgrade for /trpc path.
+    // Must use 'upgrade' on the raw Node HTTP server BEFORE Fastify processes it.
+    const httpServer = fastify.server;
+    // Remove any existing upgrade listeners that might conflict
+    const existingUpgradeListeners = httpServer.listeners("upgrade");
+    httpServer.removeAllListeners("upgrade");
+
+    httpServer.on("upgrade", (request, socket, head) => {
+      const pathname = (request.url ?? "").split("?")[0];
+      if (pathname === "/trpc") {
+        wss.handleUpgrade(request, socket, head, (ws) => {
+          wss.emit("connection", ws, request);
+        });
+      } else {
+        // Re-emit for other upgrade handlers (agent WS, etc.)
+        // `EventEmitter.listeners()` returns `Function[]` with no call
+        // signature — use Reflect.apply to invoke without a cast.
+        for (const listener of existingUpgradeListeners) {
+          Reflect.apply(listener, httpServer, [request, socket, head]);
+        }
+      }
+    });
+  } catch (err) {
+    console.error(
+      "[bootstrap] tRPC registration failed — starting in degraded mode:",
+      err,
+    );
+    // Register a fallback health endpoint that signals degraded state
+    fastify.get("/trpc/health", async (_req: FastifyRequest, reply: FastifyReply) => {
+      reply.status(503).send({
+        status: "degraded",
+        message: "tRPC router failed to initialize. Check server logs.",
+      });
+    });
+  }
+
+  // Wire the app router into AddonRegistry so addons get context.api
+  if (appRouter) {
+    await addonRegistry.setAppRouter(appRouter);
+    console.log("[bootstrap] AddonRegistry wired with tRPC direct caller");
+  }
+
+  // ScopedTokenManager and admin user creation handled by local-auth addon.
+
+  // Serve admin UI static files from the admin-ui singleton capability.
+  // Always enabled — in dev mode Vite runs on its own port and doesn't interfere.
+  //
+  // The admin-ui addon runs in its own dedicated runner subprocess and
+  // finishes registering 1-3s after bootstrap; poll briefly for it
+  // before giving up to avoid the 'admin-ui capability not registered —
+  // no static file serving' warn that left the SPA unserved until next
+  // restart.
+  try {
+    const addonRegistry = app.get(AddonRegistryService);
+    const capRegistry = addonRegistry.getCapabilityRegistry();
+    // The admin-ui addon runs in its own dedicated runner subprocess
+    // (one addon, one process — base-layer D2), so `getSingleton`
+    // returns a RemoteProxy whose method calls cross the broker — every
+    // call is async. The canonical cap shape returns `{staticDir}`/
+    // `{version}` objects so plain (in-process) and remote-proxied
+    // providers share the same Promise<object> signature.
+    type AdminUI = {
+      getStaticDir(): Promise<{ readonly staticDir: string }>;
+      getVersion(): Promise<{ readonly version: string }>;
+    };
+    let adminUI = capRegistry?.getSingleton<AdminUI>("admin-ui");
+    // CAMSTACK_SKIP_ADMIN_UI_WAIT — bypass the 60s poll. Used by the
+    // e2e harness, which doesn't need the SPA served and spawns hubs
+    // with strict boot timeouts. Production keeps the poll so cold
+    // boots wait for the forked admin-ui group to register.
+    const skipAdminUIWait = process.env['CAMSTACK_SKIP_ADMIN_UI_WAIT'] === '1';
+    if (!adminUI && capRegistry && !skipAdminUIWait) {
+      // Forked-addon spawn + Moleculer registration + tRPC hydration
+      // can take ~15-20s on cold boot, especially when several runners
+      // are spawning in parallel. The previous 10s window was racing
+      // the admin-ui runner's boot — fastify-static didn't register and
+      // every `GET /` returned 404. Bump to 60s; we only pay this
+      // wait once at boot.
+      const ADMIN_UI_WAIT_MS = 60_000;
+      const POLL_MS = 200;
+      const deadline = Date.now() + ADMIN_UI_WAIT_MS;
+      while (!adminUI && Date.now() < deadline) {
+        await new Promise<void>(r => setTimeout(r, POLL_MS));
+        adminUI = capRegistry.getSingleton<AdminUI>("admin-ui");
+      }
+    }
+    if (adminUI) {
+      const { staticDir } = await adminUI.getStaticDir();
+      const indexPath = path.join(staticDir, "index.html");
+      if (fs.existsSync(staticDir) && fs.existsSync(indexPath)) {
+        spaIndexHtml = indexPath;
+        await fastify.register(fastifyStatic, {
+          root: staticDir,
+          prefix: "/",
+          wildcard: false,
+          decorateReply: false,
+        });
+        // Dev diagnostic: serve webrtc-test.html from dataPath if it exists.
+        const webrtcTestPath = path.join(dataPath, "webrtc-test.html");
+        if (fs.existsSync(webrtcTestPath)) {
+          fastify.get("/webrtc-test.html", async (_request, reply) => {
+            return reply.type("text/html").send(fs.createReadStream(webrtcTestPath));
+          });
+        }
+
+        // SPA fallback: catch-all route that serves index.html for non-API paths.
+        // Uses a wildcard route instead of setNotFoundHandler.
+        fastify.get("/*", async (request, reply) => {
+          const url = request.url;
+          if (
+            url.startsWith("/trpc") ||
+            url.startsWith("/api/") ||
+            url.startsWith("/agent") ||
+            url.startsWith("/health")
+          ) {
+            return reply.callNotFound();
+          }
+          // A request whose last path segment has a file extension is a
+          // static asset, not a SPA navigation route. If it reached here,
+          // `fastify-static` has no route for it — the file is missing.
+          // Return 404 instead of the SPA `index.html`: serving HTML under
+          // an asset URL makes upstream caches (Cloudflare, the browser)
+          // pin `text/html` for a `.js`/`.css` URL, which then fails the
+          // module MIME check long after the file is actually available.
+          const pathOnly = url.split("?")[0] ?? url;
+          if (/\.[a-zA-Z0-9]+$/.test(pathOnly)) {
+            return reply.callNotFound();
+          }
+          return reply.type("text/html").send(fs.createReadStream(spaIndexHtml!));
+        });
+        const { version } = await adminUI.getVersion();
+        console.log(
+          `[bootstrap] Admin UI served from: ${staticDir} (v${version})`,
+        );
+      } else {
+        console.warn(
+          `[bootstrap] Admin UI dist not found at: ${staticDir} — run 'npm run build' in addon-admin-ui`,
+        );
+      }
+    } else {
+      console.warn(
+        "[bootstrap] admin-ui capability not registered — no static file serving",
+      );
+    }
+  } catch (err) {
+    console.error(
+      "[bootstrap] Failed to set up admin UI static serving:",
+      err,
+    );
+  }
+
+  try {
+    await app.listen(port, host);
+  } catch (listenErr: unknown) {
+    if (
+      listenErr !== null
+      && typeof listenErr === 'object'
+      && 'code' in listenErr
+      && listenErr.code === "EADDRINUSE"
+    ) {
+      console.error(
+        `[bootstrap] FATAL: Port ${port} is already in use. Stop the other process or change server.port in config.yaml.`,
+      );
+      process.exit(1);
+    }
+    throw listenErr;
+  }
+
+  const logger = app.get(LoggingService).createLogger("System");
+  const protocol = tlsOptions ? "https" : "http";
+  logger.info(
+    'CamStack server listening',
+    { meta: { protocol, host, port, trpcRegistered } },
+  );
+
+  // Post-boot: fork workers, register device streams, emit system.boot
+  const postBoot = app.get(PostBootService);
+  await postBoot.run({ port, host, dataPath, trpcRegistered });
+}
+
+/**
+ * Read the hub's own package version (best-effort) for /health responses.
+ */
+function readHubVersion(): string {
+  try {
+    const pkgPath = path.resolve(__dirname, "..", "package.json");
+    if (fs.existsSync(pkgPath)) {
+      const raw = JSON.parse(fs.readFileSync(pkgPath, "utf-8")) as { version?: string };
+      if (typeof raw.version === "string") return raw.version;
+    }
+  } catch {
+    // best-effort
+  }
+  return "unknown";
+}
+
+/**
+ * Build an AddonHttpReply wrapper around a Fastify reply.
+ */
+function buildAddonReply(reply: FastifyReply) {
+  const wrapper = {
+    status(code: number) {
+      reply.status(code);
+      return wrapper;
+    },
+    code(code: number) {
+      reply.code(code);
+      return wrapper;
+    },
+    send(data: unknown) {
+      reply.send(data);
+    },
+    redirect(url: string) {
+      reply.redirect(url);
+    },
+    header(name: string, value: string) {
+      reply.header(name, value);
+      return wrapper;
+    },
+    type(mime: string) {
+      reply.type(mime);
+      return wrapper;
+    },
+  };
+  return wrapper;
+}
+
+bootstrap().catch((err) => {
+  console.error("Bootstrap failed:", err);
+  process.exit(1);
+});