@bevel-software/platform-core-backend 0.20.0 → 0.21.0

package/src/tenancy/static-tenant-source.ts

@@ -0,0 +1,296 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { resolveDatabaseUrl, type TenantConfig } from '../core-config.js';
+import { DEFAULT_DB_SCHEMA, assertSchemaName } from '../modules/database/connection.js';
+import { DEFAULT_GIT_TIMEOUT_MS } from '../modules/workflow/git/node-git-runner.js';
+import { defaultKbTemplateDir } from '../assets.js';
+import { DEFAULT_GIT_USERNAME } from '../shared/git.contract.js';
+import { deriveTenantSecrets } from './tenant-secrets.js';
+import { loopbackTenantBaseUrl } from './tenant-host.js';
+import { assertTenantSlug, normalizeHost, type TenantDescriptor, type TenantSource } from './tenant-source.contract.js';
+
+/**
+ * What every tenant of one process shares: the process's own facts, the one
+ * database, the roots its folders hang off, and the master key its secrets
+ * derive from. A tenant's own record ({@link TenantRecord}) adds what differs.
+ */
+export interface TenantHostSettings {
+  readonly databaseUrl: string;
+  readonly port: number;
+  readonly nodeEnv: string;
+  readonly trustProxy: string;
+  /** Each tenant's workspaces live under `<workspacesRoot>/<slug>`; its backups, spills and caches beside them. */
+  readonly workspacesRoot: string;
+  readonly kbTemplateDir: string;
+  readonly gitTimeoutMs: number;
+  readonly ontologySessionBlock: boolean;
+  readonly updateCheckEnabled: boolean;
+  /** See `tenant-secrets.ts`. */
+  readonly masterKey: string;
+}
+
+/** One entry of the tenants file. Everything optional has the single-tenant default. */
+export interface TenantRecord {
+  slug: string;
+  /** Host names (a port is tolerated and ignored) that reach this tenant. At least one. */
+  hosts: string[];
+  adminEmail: string;
+  /** The bootstrap password; omitted, password login is off for this tenant. */
+  adminPassword?: string;
+  /** Force password login on or off; default: on exactly when `adminPassword` is set. */
+  loginPassword?: boolean;
+  kbRepoUrl?: string;
+  gitToken?: string;
+  gitUsername?: string;
+  kbDirName?: string;
+  /** The branch model; absent, the tenant's admin enters it on the setup screen. */
+  defaultBranch?: string;
+  protectedBranches?: string | string[];
+  /** The bearer a git host's webhook presents to `POST /api/sync`. */
+  kbSyncSecret?: string;
+  /** Default `t_<slug>` with hyphens as underscores. */
+  dbSchema?: string;
+  /** The credential-prefix brand; default: the slug without hyphens. */
+  tenantId?: string;
+  /** Default `https://<first host>`. */
+  publicBackendUrl?: string;
+  /** Default: the backend URL (the backend serves the SPA). */
+  publicFrontendUrl?: string;
+  allowedEmailDomains?: string[];
+  oidc?: {
+    issuerUrl: string;
+    clientId: string;
+    clientSecret: string;
+    scopes?: string;
+    providerLabel?: string;
+  };
+}
+
+/** The file's shape: a list of records. */
+export interface TenantsFile {
+  tenants: TenantRecord[];
+}
+
+const EMAIL_PATTERN = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
+const USERNAME_PATTERN = /^[A-Za-z0-9._-]+$/;
+
+/**
+ * A tenant's {@link TenantConfig} from its record and the host's settings:
+ * the same values `CoreConfig` reads from the environment, resolved the same
+ * way, with the folders and the schema named after the slug so two tenants
+ * can never share a path or a table.
+ */
+export function tenantConfigFrom(record: TenantRecord, settings: TenantHostSettings): TenantConfig {
+  const slug = assertTenantSlug(record.slug);
+  if (!Array.isArray(record.hosts) || record.hosts.length === 0) {
+    throw new Error(`Tenant "${slug}" names no host`);
+  }
+  const hosts = record.hosts.map(normalizeHost);
+  const adminEmail = (record.adminEmail ?? '').trim().toLowerCase();
+  if (!EMAIL_PATTERN.test(adminEmail)) {
+    throw new Error(`Tenant "${slug}": adminEmail is not a valid email: "${record.adminEmail ?? ''}"`);
+  }
+  const gitUsername = (record.gitUsername ?? DEFAULT_GIT_USERNAME).trim();
+  if (!USERNAME_PATTERN.test(gitUsername)) {
+    throw new Error(`Tenant "${slug}": gitUsername must match [A-Za-z0-9._-]+; got "${gitUsername}"`);
+  }
+  const tenantId = (record.tenantId ?? slug.replace(/-/g, '')).trim().toLowerCase();
+  if (!/^[a-z0-9]+$/.test(tenantId)) {
+    throw new Error(`Tenant "${slug}": tenantId must be lowercase alphanumeric; got "${tenantId}"`);
+  }
+  const dbSchema = assertSchemaName(record.dbSchema ?? `t_${slug.replace(/-/g, '_')}`);
+  // The default schema is where a single-tenant deployment on the same
+  // database keeps its tables and, in drizzle's own schema beside it, its
+  // migration ledger; a tenant there would be no tenant at all.
+  if (dbSchema === DEFAULT_DB_SCHEMA) {
+    throw new Error(`Tenant "${slug}": dbSchema must not be "${DEFAULT_DB_SCHEMA}"; a tenant needs a schema of its own`);
+  }
+  const adminPassword = record.adminPassword ?? '';
+  const loginPasswordEnabled = record.loginPassword ?? adminPassword !== '';
+  if (loginPasswordEnabled && !adminPassword) {
+    throw new Error(`Tenant "${slug}": loginPassword is on but no adminPassword is set`);
+  }
+  const publicBackendUrl = (record.publicBackendUrl ?? `https://${hosts[0]}`).trim().replace(/\/+$/, '');
+  const publicFrontendUrl = (record.publicFrontendUrl ?? publicBackendUrl).trim().replace(/\/+$/, '');
+  for (const [name, value] of [
+    ['publicBackendUrl', publicBackendUrl],
+    ['publicFrontendUrl', publicFrontendUrl],
+  ] as const) {
+    try {
+      new URL(value);
+    } catch {
+      throw new Error(`Tenant "${slug}": ${name} is not a valid URL: "${value}"`);
+    }
+  }
+  const secrets = deriveTenantSecrets(settings.masterKey, slug);
+  const besideWorkspaces = (name: string) => path.resolve(settings.workspacesRoot, '..', name, slug);
+  const allowedEmailDomains = (record.allowedEmailDomains ?? [])
+    .map((d) => d.trim().toLowerCase().replace(/^[@.]+/, ''))
+    .filter((d) => d.length > 0);
+  const protectedBranches = Array.isArray(record.protectedBranches)
+    ? record.protectedBranches.join(',')
+    : (record.protectedBranches ?? '');
+  // The record's settings, as the environment a single-tenant deployment
+  // would pin them in: only what the record names, so the setup screen
+  // still collects the rest. Nothing of the host process's environment.
+  const settingsEnv: NodeJS.ProcessEnv = {};
+  for (const [name, value] of [
+    ['KB_REPO_URL', record.kbRepoUrl],
+    ['GIT_TOKEN', record.gitToken],
+    ['GIT_USERNAME', record.gitUsername],
+    ['KB_DIR_NAME', record.kbDirName],
+    ['DEFAULT_BRANCH', record.defaultBranch],
+    ['PROTECTED_BRANCHES', protectedBranches],
+    ['KB_SYNC_SECRET', record.kbSyncSecret],
+    ['OIDC_ISSUER_URL', record.oidc?.issuerUrl],
+    ['OIDC_CLIENT_ID', record.oidc?.clientId],
+    ['OIDC_CLIENT_SECRET', record.oidc?.clientSecret],
+    ['OIDC_SCOPES', record.oidc?.scopes],
+    ['OIDC_PROVIDER_LABEL', record.oidc?.providerLabel],
+    ['ALLOWED_EMAIL_DOMAINS', allowedEmailDomains.join(',')],
+  ] as const) {
+    const trimmed = (value ?? '').trim();
+    if (trimmed) settingsEnv[name] = trimmed;
+  }
+  return {
+    port: settings.port,
+    databaseUrl: settings.databaseUrl,
+    dbSchema,
+    nodeEnv: settings.nodeEnv,
+    tenantId,
+    externalApiKeyPrefix: `${tenantId}_`,
+    internalTokenPrefix: `${tenantId}-int_`,
+    uploadTokenPrefix: `${tenantId}-up_`,
+    mcpOAuthTokenPrefix: `${tenantId}-mcp_`,
+    workspacesRoot: path.resolve(settings.workspacesRoot, slug),
+    backupsRoot: besideWorkspaces('backups'),
+    spillRoot: besideWorkspaces('tool-chain-spills'),
+    docExtractCacheRoot: besideWorkspaces('doc-extract-cache'),
+    jwtSecret: secrets.jwtSecret,
+    secretsEncKey: secrets.secretsEncKey,
+    internalTokenSecret: secrets.internalTokenSecret,
+    adminEmail,
+    adminPassword,
+    loginPasswordEnabled,
+    oidcIssuerUrl: (record.oidc?.issuerUrl ?? '').trim().replace(/\/+$/, ''),
+    oidcClientId: (record.oidc?.clientId ?? '').trim(),
+    oidcClientSecret: (record.oidc?.clientSecret ?? '').trim(),
+    oidcScopes: (record.oidc?.scopes ?? 'openid profile email').trim(),
+    oidcProviderLabel: (record.oidc?.providerLabel ?? 'Single sign-on').trim(),
+    kbRepoUrl: (record.kbRepoUrl ?? '').trim(),
+    kbDirName: (record.kbDirName ?? '').trim(),
+    gitUsername,
+    gitToken: (record.gitToken ?? '').trim(),
+    kbTemplateDir: settings.kbTemplateDir,
+    ontologySessionBlock: settings.ontologySessionBlock,
+    updateCheckEnabled: settings.updateCheckEnabled,
+    allowedEmailDomains,
+    trustProxy: settings.trustProxy,
+    gitTimeoutMs: settings.gitTimeoutMs,
+    publicBackendUrl,
+    publicFrontendUrl,
+    configuredPublicFrontendUrl: publicFrontendUrl,
+    loopbackBaseUrl: loopbackTenantBaseUrl(settings.port, slug),
+    settingsEnv,
+  };
+}
+
+/**
+ * The reference {@link TenantSource}: every tenant known up front, from a
+ * JSON file (`TENANTS_FILE`). Enough for development, tests and a small
+ * self-hosted fleet; the cloud app brings its own source over its registry.
+ */
+export class StaticTenantSource implements TenantSource {
+  private readonly bySlug = new Map<string, TenantDescriptor>();
+  private readonly byHost = new Map<string, TenantDescriptor>();
+
+  constructor(records: readonly TenantRecord[], settings: TenantHostSettings) {
+    for (const record of records) {
+      const config = tenantConfigFrom(record, settings);
+      const descriptor: TenantDescriptor = {
+        slug: record.slug,
+        hosts: record.hosts.map(normalizeHost),
+        config,
+      };
+      if (this.bySlug.has(descriptor.slug)) throw new Error(`Tenant "${descriptor.slug}" is listed twice`);
+      this.bySlug.set(descriptor.slug, descriptor);
+      for (const host of descriptor.hosts) {
+        const taken = this.byHost.get(host);
+        if (taken) throw new Error(`Host "${host}" is claimed by both "${taken.slug}" and "${descriptor.slug}"`);
+        this.byHost.set(host, descriptor);
+      }
+    }
+  }
+
+  static async fromFile(file: string, settings: TenantHostSettings): Promise<StaticTenantSource> {
+    const text = await fs.readFile(file, 'utf8');
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(text);
+    } catch (err) {
+      throw new Error(`${file} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    const tenants = (parsed as Partial<TenantsFile> | null)?.tenants;
+    if (!Array.isArray(tenants)) throw new Error(`${file} must be an object with a "tenants" array`);
+    return new StaticTenantSource(tenants as TenantRecord[], settings);
+  }
+
+  get slugs(): readonly string[] {
+    return [...this.bySlug.keys()];
+  }
+
+  async resolveByHost(host: string): Promise<TenantDescriptor | null> {
+    return this.byHost.get(normalizeHost(host)) ?? null;
+  }
+
+  async describe(slug: string): Promise<TenantDescriptor | null> {
+    return this.bySlug.get(slug) ?? null;
+  }
+}
+
+/** The process facts a tenant host reads from its environment — see `docs/multi-tenant.md`. */
+export interface TenantHostEnv extends TenantHostSettings {
+  /** The tenants file, when the process runs as a host at all. */
+  readonly tenantsFile: string | null;
+  /** Minutes without a request after which a tenant's graph is stopped. */
+  readonly idleMinutes: number;
+}
+
+const MAX_TIMER_MS = 2_147_483_647;
+
+/**
+ * The host's settings from the environment: the same variables a
+ * single-tenant deployment reads for these facts (`DATABASE_URL` and the
+ * `POSTGRES_*` parts, `PORT`, `TRUST_PROXY`, `WORKSPACES_ROOT`,
+ * `KB_TEMPLATE_DIR`, `GIT_TIMEOUT_MS`, …), plus the three that only a host
+ * has: `TENANTS_FILE`, `TENANT_MASTER_KEY` and `TENANT_IDLE_MINUTES`.
+ */
+export function tenantHostEnv(env: NodeJS.ProcessEnv = process.env): TenantHostEnv {
+  const tenantsFile = (env.TENANTS_FILE || '').trim() || null;
+  const masterKey = (env.TENANT_MASTER_KEY || '').trim();
+  if (tenantsFile && !masterKey) {
+    throw new Error(
+      'TENANT_MASTER_KEY is required with TENANTS_FILE — every tenant\'s secrets derive from it. Generate one with: ' +
+        `node -e "console.log(require('crypto').randomBytes(32).toString('base64'))"`,
+    );
+  }
+  const gitTimeout = Number(env.GIT_TIMEOUT_MS);
+  const idle = Number(env.TENANT_IDLE_MINUTES);
+  const domain = (env.DOMAIN || '').trim();
+  const workspacesRoot = env.WORKSPACES_ROOT || path.resolve(process.cwd(), 'workspaces');
+  return {
+    tenantsFile,
+    masterKey,
+    databaseUrl: resolveDatabaseUrl(env),
+    port: parseInt(env.PORT || '3001', 10),
+    nodeEnv: env.NODE_ENV || 'development',
+    trustProxy: (env.TRUST_PROXY || (domain ? '1' : '')).trim(),
+    workspacesRoot,
+    kbTemplateDir: env.KB_TEMPLATE_DIR || defaultKbTemplateDir(),
+    gitTimeoutMs:
+      Number.isFinite(gitTimeout) && gitTimeout > 0 && gitTimeout <= MAX_TIMER_MS ? gitTimeout : DEFAULT_GIT_TIMEOUT_MS,
+    ontologySessionBlock: (env.ONTOLOGY_SESSION_BLOCK ?? 'true').trim().toLowerCase() !== 'false',
+    updateCheckEnabled: (env.UPDATE_CHECK ?? 'true').trim().toLowerCase() !== 'false',
+    idleMinutes: Number.isFinite(idle) && idle > 0 ? idle : 30,
+  };
+}

package/src/tenancy/tenant-host.ts

@@ -0,0 +1,261 @@
+import express, { type Express, type Request } from 'express';
+import path from 'node:path';
+import { isIP } from 'node:net';
+import type { ProcessConfig } from '../core-config.js';
+import { GIT_SHA } from '../version.js';
+import { logger } from '../shared/logging.js';
+import { TenantRuntime, type TenantRuntimeDeps } from './tenant-runtime.js';
+import { TENANT_SLUG_PATTERN, normalizeHost, type TenantDescriptor, type TenantSource } from './tenant-source.contract.js';
+
+const log = logger('tenancy');
+
+/**
+ * The path prefix under which a process reaches ONE of its own tenants over
+ * loopback: `http://127.0.0.1:<port>/_tenant/<slug>/api/...`. The MCP proxy
+ * and the UTCP manuals it seeds dial the tenant's own REST surface this way,
+ * because a loopback request carries no tenant host name and a header cannot
+ * ride a UTCP call (the proxy is a passthrough; it reshapes nothing).
+ *
+ * The host honours the prefix only for a request that came from this
+ * machine AND was forwarded by nobody: the socket peer is loopback and there
+ * is no `X-Forwarded-For`. A reverse proxy on the same host (nginx, a
+ * Coolify proxy) delivers every public request from a loopback socket, and
+ * every such proxy adds that header; the process's own dial never does.
+ * What the prefix could grant anyone who got past this is only what the
+ * tenant's host name grants already — sessions, keys and tokens are per
+ * tenant — but the addressing rule is stated so it can be relied on.
+ */
+export const LOOPBACK_TENANT_PREFIX = '/_tenant';
+
+/** The base URL a tenant's graph dials itself at — see {@link LOOPBACK_TENANT_PREFIX}. */
+export function loopbackTenantBaseUrl(port: number, slug: string): string {
+  return `http://127.0.0.1:${port}${LOOPBACK_TENANT_PREFIX}/${slug}`;
+}
+
+/** Whether `address` is this machine talking to itself. */
+export function isLoopbackAddress(address: string | undefined): boolean {
+  if (!address) return false;
+  const bare = address.startsWith('::ffff:') ? address.slice('::ffff:'.length) : address;
+  if (bare === '::1') return true;
+  return isIP(bare) === 4 && bare.startsWith('127.');
+}
+
+/** How a request names its tenant: by the loopback prefix, or by its host. */
+export type TenantSelector = { kind: 'slug'; slug: string; rest: string } | { kind: 'host'; host: string };
+
+/**
+ * Which tenant a request is for. The loopback prefix wins, but only from a
+ * loopback peer that no proxy forwarded (no `X-Forwarded-For`); everything
+ * else is decided by the host name (the `X-Forwarded-Host` a trusted proxy
+ * set, else `Host`), lowercase and without its port.
+ */
+export function selectTenant(req: {
+  hostname: string;
+  url: string;
+  peer: string | undefined;
+  forwardedFor?: string | string[] | undefined;
+}): TenantSelector {
+  const forwarded = Array.isArray(req.forwardedFor) ? req.forwardedFor.length > 0 : Boolean(req.forwardedFor);
+  if (isLoopbackAddress(req.peer) && !forwarded && req.url.startsWith(`${LOOPBACK_TENANT_PREFIX}/`)) {
+    const afterPrefix = req.url.slice(LOOPBACK_TENANT_PREFIX.length + 1);
+    const end = afterPrefix.search(/[/?#]/);
+    const slug = end === -1 ? afterPrefix : afterPrefix.slice(0, end);
+    if (TENANT_SLUG_PATTERN.test(slug)) {
+      const rest = end === -1 ? '/' : afterPrefix.slice(end).replace(/^(?=[?#])/, '/');
+      return { kind: 'slug', slug, rest };
+    }
+  }
+  return { kind: 'host', host: normalizeHost(req.hostname || '') };
+}
+
+export interface TenantHostOptions {
+  source: TenantSource;
+  process: ProcessConfig;
+  /** The built SPA, served once for every resolved tenant. */
+  staticDir?: string;
+  /** Minutes without a request after which a tenant's graph is stopped. Default 30; 0 never evicts. */
+  idleMinutes?: number;
+  /** How long a request waits for an activation in flight before answering 503. Default 60s. */
+  activationWaitMs?: number;
+  /** Seams for the runtimes the host builds — a suite's fake graphs. */
+  runtime?: TenantRuntimeDeps;
+  /** How often idle tenants are looked for. Default: once a minute. */
+  sweepIntervalMs?: number;
+}
+
+export interface TenantHost {
+  /** The app the process listens on. */
+  app: Express;
+  /** The runtimes built so far, by slug. */
+  runtimes(): ReadonlyMap<string, TenantRuntime>;
+  /** Stop every active tenant; the host keeps serving and reactivates on demand. */
+  evictAll(): Promise<void>;
+  /** Stop the idle sweep and every tenant: the process's shutdown. */
+  stop(): Promise<void>;
+}
+
+const DEFAULT_ACTIVATION_WAIT_MS = 60_000;
+const DEFAULT_IDLE_MINUTES = 30;
+const DEFAULT_SWEEP_INTERVAL_MS = 60_000;
+
+/** `promise`, or `null` once `ms` have passed. */
+function within<T>(promise: Promise<T>, ms: number): Promise<T | null> {
+  let timer: NodeJS.Timeout | undefined;
+  const late = new Promise<null>((resolve) => {
+    timer = setTimeout(() => resolve(null), ms);
+  });
+  return Promise.race([promise, late]).finally(() => clearTimeout(timer));
+}
+
+/**
+ * One Express app in front of many tenants' apps. It answers process health
+ * itself, resolves each request to a tenant (see {@link selectTenant}),
+ * activates that tenant's graph on first use, hands the request to the
+ * tenant's app, and serves the SPA for whatever the tenant's app leaves
+ * unhandled. Nothing below it knows there are other tenants: each graph was
+ * built by the same composition root a single-tenant deployment uses.
+ */
+export function createTenantHost(opts: TenantHostOptions): TenantHost {
+  const runtimes = new Map<string, TenantRuntime>();
+  const activationWaitMs = opts.activationWaitMs ?? DEFAULT_ACTIVATION_WAIT_MS;
+  const idleMs = (opts.idleMinutes ?? DEFAULT_IDLE_MINUTES) * 60_000;
+
+  const runtimeFor = (descriptor: TenantDescriptor): TenantRuntime => {
+    let runtime = runtimes.get(descriptor.slug);
+    if (!runtime) {
+      runtime = new TenantRuntime(descriptor, opts.runtime);
+      runtimes.set(descriptor.slug, runtime);
+    }
+    return runtime;
+  };
+
+  const app = express();
+  // Forwarded headers are honoured exactly as a single-tenant deployment
+  // honours them, and for the same reason: `req.hostname` is what picks the
+  // tenant, and behind a proxy it lives in `X-Forwarded-Host`.
+  if (opts.process.trustProxy) {
+    const raw = opts.process.trustProxy;
+    app.set('trust proxy', /^\d+$/.test(raw) ? Number(raw) : raw);
+  }
+
+  // The process's own health, before any tenant is involved: a load balancer
+  // asks with whatever host name it likes, and a process with no active
+  // tenant is still a healthy process.
+  app.get('/api/health', (_req, res) => {
+    res.json({ status: 'ok', sha: GIT_SHA, timestamp: Date.now(), tenants: runtimes.size });
+  });
+
+  app.use(async (req: Request, res, next) => {
+    const selector = selectTenant({
+      hostname: req.hostname,
+      url: req.url,
+      peer: req.socket.remoteAddress,
+      forwardedFor: req.headers['x-forwarded-for'],
+    });
+    let descriptor: TenantDescriptor | null;
+    try {
+      descriptor =
+        selector.kind === 'slug'
+          ? await opts.source.describe(selector.slug)
+          : await opts.source.resolveByHost(selector.host);
+    } catch (err) {
+      log.error('tenant lookup failed:', { err });
+      res.status(503).set('Retry-After', '5').json({ error: 'Tenant lookup failed' });
+      return;
+    }
+    if (!descriptor) {
+      if (req.path.startsWith('/api/') || req.path.startsWith(LOOPBACK_TENANT_PREFIX)) {
+        res.status(404).json({ error: 'No workspace at this address' });
+      } else {
+        res.status(404).type('text/plain').send('No workspace at this address.');
+      }
+      return;
+    }
+    if (selector.kind === 'slug') req.url = selector.rest;
+
+    const runtime = runtimeFor(descriptor);
+    // Counted from here, before the graph is even asked for, until the
+    // response closes however it ends — a 503 below included. An open
+    // response is what keeps a tenant from being evicted from under it.
+    // Left exactly once: a client that dropped during the tenant lookup
+    // above has had its `close` already, and that event will not come
+    // again, so the socket is asked directly; a socket destroyed but not
+    // yet closed would otherwise be left twice, once here and once by the
+    // listener, taking another request's count with it.
+    runtime.enter();
+    let left = false;
+    const leave = () => {
+      if (left) return;
+      left = true;
+      runtime.leave();
+    };
+    res.on('close', leave);
+    if (req.socket.destroyed) {
+      leave();
+      return;
+    }
+    let graph: Awaited<ReturnType<TenantRuntime['handle']>> | null;
+    try {
+      graph = await within(runtime.handle(), activationWaitMs);
+    } catch {
+      // Logged by the runtime; the caller learns only that it may try again.
+      res.status(503).set('Retry-After', '5').json({ error: 'This workspace could not be started; try again shortly' });
+      return;
+    }
+    if (!graph) {
+      res.status(503).set('Retry-After', '5').json({ error: 'This workspace is starting; try again shortly' });
+      return;
+    }
+    graph.app(req, res, next);
+  });
+
+  // Whatever a tenant's app left unhandled: the SPA, with the same rule a
+  // single-tenant deployment applies to the namespaces that must never
+  // resolve to `index.html` (see `createCoreServer`).
+  if (opts.staticDir) {
+    const frontendDist = opts.staticDir;
+    app.use(express.static(frontendDist));
+    app.get('{*path}', (req, res) => {
+      if (
+        req.path.startsWith('/.well-known/') ||
+        req.path.startsWith('/api/') ||
+        req.path.startsWith('/login/oauth/')
+      ) {
+        res.status(404).json({ error: 'Not found' });
+        return;
+      }
+      res.sendFile(path.join(frontendDist, 'index.html'));
+    });
+  }
+
+  // Idle tenants are stopped on a timer, never on the request that would
+  // have been their last: a graph whose last response closed `idleMs` ago,
+  // with none still open and no queued commit, is not worth its pool and
+  // its lease. `busy()` is what keeps an event stream's tenant alive.
+  const sweep = async () => {
+    for (const runtime of runtimes.values()) {
+      if (runtime.state !== 'active' || runtime.idleFor() < idleMs) continue;
+      if (await runtime.busy()) continue;
+      await runtime.evict().catch((err: unknown) => log.warn(`evicting tenant "${runtime.slug}" failed:`, { err }));
+    }
+  };
+  const sweeper =
+    idleMs > 0
+      ? setInterval(() => void sweep(), Math.min(opts.sweepIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS, idleMs))
+      : null;
+  sweeper?.unref();
+
+  const evictAll = async () => {
+    await Promise.all([...runtimes.values()].map((runtime) => runtime.evict().catch(() => undefined)));
+  };
+
+  return {
+    app,
+    runtimes: () => runtimes,
+    evictAll,
+    async stop() {
+      if (sweeper) clearInterval(sweeper);
+      await evictAll();
+    },
+  };
+}