hammerkit 1.27.0-next.9 → 2.0.0-next.1

package/dist/server/server.js

@@ -1,18 +1,23 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.startServer = exports.getDiscoveryDocument = exports.SERVER_PROTOCOL = exports.DISCOVERY_PATH = void 0;
+exports.startServer = exports.runsTheSame = exports.refuseWake = exports.getDiscoveryDocument = exports.SERVER_PROTOCOL = exports.DISCOVERY_PATH = void 0;
 const http_1 = require("http");
 const https_1 = require("https");
+const tls_1 = require("tls");
 const version_1 = require("../version");
 const server_config_1 = require("./server-config");
 const discovery_1 = require("./discovery");
 const oauth_1 = require("./oauth");
 const policy_1 = require("./policy");
+const repo_1 = require("./repo");
 const run_store_1 = require("./run-store");
 const runs_1 = require("./runs");
 const git_checkout_1 = require("./git-checkout");
 const repo_access_1 = require("./repo-access");
 const executor_1 = require("./executor");
+const capacity_1 = require("./capacity");
+const execute_docker_1 = require("../executer/execute-docker");
+const timings_1 = require("./timings");
 const kubernetes_backend_1 = require("./kubernetes-backend");
 const run_plan_1 = require("./run-plan");
 const work_network_1 = require("../planner/work-network");
@@ -32,10 +37,15 @@ const sandbox_storage_1 = require("./sandbox-storage");
 const kubernetes_sandbox_storage_1 = require("./kubernetes-sandbox-storage");
 const sandbox_docker_1 = require("./sandbox-docker");
 const sandboxes_api_1 = require("./sandboxes-api");
+const endpoints_1 = require("./endpoints");
+const service_runs_1 = require("./service-runs");
 const units_1 = require("../utils/units");
 const run_logs_1 = require("./run-logs");
 const get_cache_directory_1 = require("../optimizer/get-cache-directory");
 const path_1 = require("path");
+const checks_1 = require("./checks");
+const checks_api_1 = require("./checks-api");
+const overview_1 = require("./overview");
 var discovery_2 = require("./discovery");
 Object.defineProperty(exports, "DISCOVERY_PATH", { enumerable: true, get: function () { return discovery_2.DISCOVERY_PATH; } });
 Object.defineProperty(exports, "SERVER_PROTOCOL", { enumerable: true, get: function () { return discovery_2.SERVER_PROTOCOL; } });
@@ -64,6 +74,8 @@ const HEARTBEAT_MS = 15000;
 // the longest `?wait=` of a run: below the 60 seconds after which a proxy in front of the
 // server (nginx by default) closes a connection that says nothing
 const MAX_WAIT_MS = 50000;
+// how often checks older than `runs.checksKeep` are removed
+const CHECK_SWEEP_MS = 6 * 60 * 60 * 1000;
 // Server-sent events: `id` is the sequence number, so a client that lost the
 // connection continues with Last-Event-ID (or ?after=) and misses nothing.
 function streamEvents(registry, config, principal, id, req, res, url) {
@@ -96,8 +108,8 @@ function streamEvents(registry, config, principal, id, req, res, url) {
         watch.stop();
     });
 }
-async function handleRuns(config, registry, definitions, sources, sandboxes, planning, repos, log, principal, req, res, url) {
-    const match = /^\/v1\/runs(?:\/([^/]+)(?:\/(cancel|events|logs))?)?$/.exec(url.pathname);
+async function handleRuns(config, registry, definitions, sources, sandboxes, planning, repos, endpointUrl, log, principal, req, res, url) {
+    const match = /^\/v1\/runs(?:\/([^/]+)(?:\/(cancel|events|logs|extend))?)?$/.exec(url.pathname);
     if (!match) {
         return false;
     }
@@ -106,16 +118,22 @@ async function handleRuns(config, registry, definitions, sources, sandboxes, pla
     if (!id) {
         if (method === 'GET') {
             // the results of each task are read one run at a time
+            const viewer = (0, overview_1.getViewer)(config, principal);
+            let filter;
+            try {
+                filter = (0, overview_1.parseRunFilter)(url.searchParams, viewer.grant);
+            }
+            catch (e) {
+                (0, http_2.send)(res, e instanceof overview_1.FilterRefused ? 403 : 400, { error: e instanceof Error ? e.message : String(e) });
+                return true;
+            }
+            const runs = filter.scope === 'mine' ? registry.list(principal) : registry.all();
             (0, http_2.send)(res, 200, {
-                runs: registry.list(principal).map((run) => {
-                    const { results, ...listed } = toRunResponse(run);
-                    void results;
-                    return listed;
-                }),
+                runs: (0, overview_1.filterRuns)(runs, viewer, filter).map((run) => listRun(registry, viewer, run)),
             });
         }
         else if (method === 'POST') {
-            await submitRun(config, registry, definitions, sources, sandboxes, planning, repos, log, principal, req, res);
+            await submitRun(config, registry, definitions, sources, sandboxes, planning, repos, endpointUrl, log, principal, req, res);
         }
         else {
             (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET, POST' });
@@ -134,6 +152,13 @@ async function handleRuns(config, registry, definitions, sources, sandboxes, pla
         log(`cancel iss=${principal.iss} sub=${principal.sub} run=${id} found=${Boolean(run)}`);
         run ? (0, http_2.send)(res, 200, toRunResponse(run)) : (0, http_2.send)(res, 404, { error: 'not found' });
     }
+    else if (action === 'extend') {
+        if (method !== 'POST') {
+            (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'POST' });
+            return true;
+        }
+        await extendRun(config, registry, log, principal, id, req, res);
+    }
     else if (method !== 'GET') {
         (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET' });
     }
@@ -149,9 +174,12 @@ async function handleRuns(config, registry, definitions, sources, sandboxes, pla
             (0, http_2.send)(res, 400, { error: e instanceof Error ? e.message : String(e) });
             return true;
         }
-        const logs = registry.logs(principal, id, accountsOf(config, principal));
+        // an entitlement with `observe` reads the log of every run
+        const logs = registry.logs(principal, id, accountsOf(config, principal)) ??
+            ((0, policy_1.getGrant)(config.entitlements, principal).observe ? registry.logsOf(id) : undefined);
+        // with the tasks, services and the server's own lines (`hammerkit`) the log has
         logs
-            ? (0, http_2.send)(res, 200, (0, run_logs_1.readLogs)(logs.events, logs.index, query, logs.truncated))
+            ? (0, http_2.send)(res, 200, { ...(0, run_logs_1.readLogs)(logs.events, logs.index, query, logs.truncated), names: logs.index.names() })
             : (0, http_2.send)(res, 404, { error: 'not found' });
     }
     else {
@@ -169,9 +197,18 @@ async function handleRuns(config, registry, definitions, sources, sandboxes, pla
         // a caller that hangs up stops waiting
         const gone = new AbortController();
         res.on('close', () => gone.abort());
-        const run = await registry.wait(principal, id, wait, accountsOf(config, principal), gone.signal);
+        const viewer = (0, overview_1.getViewer)(config, principal);
+        // an entitlement with `observe` reads every run as it is, without waiting
+        const run = (await registry.wait(principal, id, wait, accountsOf(config, principal), gone.signal)) ??
+            (viewer.grant.observe ? registry.peek(id) : undefined);
         if (!gone.signal.aborted) {
-            run ? (0, http_2.send)(res, 200, toRunResponse(run)) : (0, http_2.send)(res, 404, { error: 'not found' });
+            const may = run && (0, overview_1.runActions)(viewer, run);
+            run
+                ? (0, http_2.send)(res, 200, {
+                    ...toRunResponse(run, (0, overview_1.queuePlace)(registry, viewer, run), !(0, overview_1.owns)(viewer, run)),
+                    ...(may ? { may } : {}),
+                })
+                : (0, http_2.send)(res, 404, { error: 'not found' });
         }
     }
     return true;
@@ -181,12 +218,100 @@ function accountsOf(config, principal) {
     const grant = (0, policy_1.getGrant)(config.entitlements, principal);
     return (account) => (0, policy_1.canUse)(grant, account);
 }
-function toRunResponse(run) {
+async function extendRun(config, registry, log, principal, id, req, res) {
+    const body = (await (0, http_2.readJson)(req));
+    let ttl;
+    try {
+        if (typeof body !== 'object' || body === null || typeof body.ttl !== 'string') {
+            throw new Error();
+        }
+        ttl = (0, units_1.parseDuration)(body.ttl);
+        if (!(ttl > 0)) {
+            throw new Error();
+        }
+    }
+    catch {
+        (0, http_2.send)(res, 400, { error: 'ttl: expected a duration like 30m, 8h or 1d' });
+        return;
+    }
+    const grant = (0, policy_1.getGrant)(config.entitlements, principal);
+    const result = registry.extend(principal, id, ttl, (run) => (0, service_runs_1.mayOpenEndpoints)(grant, run));
+    log(`extend iss=${principal.iss} sub=${principal.sub} run=${id} result=${result.status}`);
+    if (result.status === 'not-found') {
+        (0, http_2.send)(res, 404, { error: 'not found' });
+    }
+    else if (result.status === 'not-up') {
+        (0, http_2.send)(res, 409, { error: `the run is ${result.state}, only a service run that is up can be extended` });
+    }
+    else {
+        (0, http_2.send)(res, 200, toRunResponse(result.run));
+    }
+}
+// Why the owner of a service run that sleeps may no longer run it, with the entitlements of
+// the moment: a wake is started by who opens an endpoint, but runs as the owner, as a new
+// run of theirs would.
+function refuseWake(config, repos, run) {
+    if (run.source.type !== 'ref') {
+        return 'a run from a sandbox does not wake';
+    }
+    const grant = (0, policy_1.getGrant)(config.entitlements, {
+        iss: run.owner.iss,
+        sub: run.owner.sub,
+        groups: run.owner.groups ?? [],
+        claims: run.owner.claims ?? {},
+    });
+    const decision = (0, policy_1.decideRun)({
+        grant,
+        accounts: run.accounts,
+        catalog: config.accounts,
+        source: { type: 'ref', repo: run.source.repo, ref: run.source.ref },
+        // an account it may only request is refused, the wake has nobody to approve it
+        approvable: () => true,
+        readable: (repo) => repos.readable(repo),
+    });
+    if (decision.type === 'refuse') {
+        return `its owner ${run.owner.sub} may no longer run it: ${decision.error.replace(/^you /, 'they ')}`;
+    }
+    if (decision.type === 'approval') {
+        return `its owner ${run.owner.sub} may only request ${decision.accounts.join(', ')} now, start it again`;
+    }
+    if (run.up?.endpoints.length && (0, policy_1.endpointLevel)(grant, run.source.repo) !== 'publish') {
+        return `its owner ${run.owner.sub} may no longer publish endpoints of ${run.source.repo}`;
+    }
+    return undefined;
+}
+exports.refuseWake = refuseWake;
+// Whether a service run that is not over runs what a new one would, but for its commit: then
+// the new one moves it there (RunRegistry.move)
+function runsTheSame(run, next) {
+    const sorted = (values) => JSON.stringify([...values].sort());
+    const options = (value) => JSON.stringify(value ?? {}, Object.keys(value ?? {}).sort());
+    const endpoints = (spec) => JSON.stringify(spec.endpoints.map((endpoint) => [endpoint.service, endpoint.port, endpoint.host]));
+    return (!!run.up &&
+        run.source.type === 'ref' &&
+        run.source.repo === next.source.repo &&
+        run.definition === next.definition &&
+        sorted(run.accounts) === sorted(next.accounts) &&
+        sorted(run.up.services) === sorted(next.up.services) &&
+        endpoints(run.up) === endpoints(next.up) &&
+        (run.up.sleepTtl ?? null) === (next.up.sleepTtl ?? null) &&
+        options(run.options) === options(next.request.options));
+}
+exports.runsTheSame = runsTheSame;
+// A run whole, for its owner and who may read it; who is not its owner is told whose it is.
+function toRunResponse(run, queue, named = false) {
     const { owner, ...visible } = run;
-    void owner;
-    return { ...visible, results: run.results ?? {} };
+    return {
+        ...visible,
+        ...(named ? { owner: { iss: owner.iss, sub: owner.sub } } : {}),
+        results: run.results ?? {},
+        ...(queue ? { queue } : {}),
+    };
+}
+function listRun(registry, viewer, run) {
+    return (0, overview_1.toListedRun)(viewer, run, (0, overview_1.queuePlace)(registry, viewer, run));
 }
-async function submitRun(config, registry, definitions, sources, sandboxes, planning, repos, log, principal, req, res) {
+async function submitRun(config, registry, definitions, sources, sandboxes, planning, repos, endpointUrl, log, principal, req, res) {
     const parsed = run_request_1.runRequestSchema.safeParse(await (0, http_2.readJson)(req));
     if (!parsed.success) {
         const issue = parsed.error.issues[0];
@@ -218,6 +343,27 @@ async function submitRun(config, registry, definitions, sources, sandboxes, plan
         (0, http_2.send)(res, 400, { error: `${version.hash} has no task ${unknown.map(oauth_1.quote).join(', ')}` });
         return;
     }
+    const notServices = request.services.filter((service) => !version.services.includes(service));
+    if (notServices.length > 0) {
+        (0, http_2.send)(res, 400, { error: `${version.hash} has no service ${notServices.map(oauth_1.quote).join(', ')}` });
+        return;
+    }
+    if (request.endpoints.length > 0 && !config.endpoints) {
+        (0, http_2.send)(res, 400, { error: 'this server publishes no endpoints (endpoints is not configured)' });
+        return;
+    }
+    const ttl = request.services.length > 0 ? (0, units_1.parseDuration)(request.ttl ?? config.runs.up.ttl) : 0;
+    if (ttl > (0, units_1.parseDuration)(config.runs.up.maxTtl)) {
+        (0, http_2.send)(res, 400, { error: `ttl: at most ${config.runs.up.maxTtl} on this server (runs.up.maxTtl)` });
+        return;
+    }
+    // the files of a sandbox change under a run that sleeps, it would wake to others
+    if (request.sleepTtl !== undefined && request.source.type === 'sandbox') {
+        (0, http_2.send)(res, 400, { error: 'sleepTtl: a service run from a sandbox does not sleep, start it from a branch' });
+        return;
+    }
+    const sleepTtlText = request.source.type === 'ref' ? request.sleepTtl ?? config.runs.up.sleepTtl : undefined;
+    const sleepTtl = request.services.length > 0 && sleepTtlText ? (0, units_1.parseDuration)(sleepTtlText) : undefined;
     const argued = (request.options.args ?? []).length > 0 ? request.tasks[0] : undefined;
     if (argued !== undefined && version.services.includes(argued)) {
         (0, http_2.send)(res, 400, { error: `${argued} is a service, arguments go to a task` });
@@ -239,6 +385,12 @@ async function submitRun(config, registry, definitions, sources, sandboxes, plan
             return;
         }
     }
+    if (request.services.length > 0 && request.source.type === 'sandbox' && config.backend.type === 'kubernetes') {
+        (0, http_2.send)(res, 400, {
+            error: 'on a cluster a service run starts from a branch, push the sandbox and start it from there',
+        });
+        return;
+    }
     const sandbox = request.source.type === 'sandbox' ? sandboxes.registry.get(principal, request.source.id) : undefined;
     if (request.source.type === 'sandbox') {
         if (!sandbox) {
@@ -257,6 +409,24 @@ async function submitRun(config, registry, definitions, sources, sandboxes, plan
             (0, http_2.send)(res, 413, { error: over });
             return;
         }
+        // a check counts like a run from a commit of the repository, and leaves what it made in
+        // the repository's cache, which runs from main restore from: the caller must be allowed
+        // to start runs from it, and to push a branch of it, which with a run from that branch
+        // leaves the same
+        if (request.source.check && !(0, repo_1.matchesAnyRepo)(grant.repos, sandbox.repo)) {
+            log(`refuse run iss=${principal.iss} sub=${principal.sub}: check on ${sandbox.repo} without the repository`);
+            (0, http_2.send)(res, 403, {
+                error: `a check counts like a run from ${sandbox.repo}, which no entitlement of yours lists in repos`,
+            });
+            return;
+        }
+        if (request.source.check && (0, policy_1.sandboxPushBranches)(grant, sandbox.repo).length === 0) {
+            log(`refuse run iss=${principal.iss} sub=${principal.sub}: check on ${sandbox.repo} without a branch to push`);
+            (0, http_2.send)(res, 403, {
+                error: `a check counts like a run from a branch you pushed, and you may push no branch of ${sandbox.repo}`,
+            });
+            return;
+        }
     }
     // the accounts of a run are read from the files it is pinned to, with the engine
     const bundle = version.plan ? await definitions.files(version) : undefined;
@@ -275,12 +445,20 @@ async function submitRun(config, registry, definitions, sources, sandboxes, plan
         (0, http_2.send)(res, 400, { error: `the run does not load: ${e instanceof Error ? e.message : String(e)}` });
         return;
     }
-    const refused = (0, run_plan_1.refuseArguments)(request, planned.plan, sandbox ? sandboxes.policy : planning.policy);
+    const refused = (0, run_plan_1.refuseArguments)(request, planned.plan, sandbox ? sandboxes.policy : planning.policy) ??
+        (0, service_runs_1.refuseEndpoints)(request, planned.plan, config.runs.secretNetwork === 'open');
     if (refused) {
         log(`refuse run iss=${principal.iss} sub=${principal.sub}: ${refused}`);
         (0, http_2.send)(res, 400, { error: refused });
         return;
     }
+    const runRepo = request.source.type === 'ref' ? request.source.repo : sandbox?.repo;
+    if (request.endpoints.length > 0 && (!runRepo || (0, policy_1.endpointLevel)(grant, runRepo) !== 'publish')) {
+        const error = `you may not publish endpoints of ${runRepo ?? 'this source'}, an entitlement needs endpoints with level publish for it`;
+        log(`refuse run iss=${principal.iss} sub=${principal.sub}: ${error}`);
+        (0, http_2.send)(res, 403, { error });
+        return;
+    }
     const decision = (0, policy_1.decideRun)({
         grant,
         accounts: planned.accounts,
@@ -324,13 +502,54 @@ async function submitRun(config, registry, definitions, sources, sandboxes, plan
     else if (request.source.type === 'upload') {
         source = request.source;
     }
-    const submit = (runSource) => registry.submit(principal, {
+    // a service run is named after its branch unless it says, and its endpoints after that
+    const serviceName = request.services.length > 0
+        ? request.name ??
+            (0, service_runs_1.getServiceRunName)(request.source.type === 'ref' ? request.source.ref : sandbox?.ref)
+        : '';
+    const up = request.services.length > 0
+        ? {
+            services: request.services,
+            ttl,
+            ...(sleepTtl ? { sleepTtl } : {}),
+            name: serviceName,
+            endpoints: (0, service_runs_1.toRunEndpoints)(request.endpoints, (0, service_runs_1.getEndpointHosts)(serviceName, principal, request.endpoints), endpointUrl),
+        }
+        : undefined;
+    // a run that may sleep keeps what its owner is entitled by, for the check of a wake
+    const claimNames = new Set(config.entitlements.flatMap((entitlement) => Object.keys(entitlement.claims?.match ?? {})));
+    const owner = {
+        ...principal,
+        claims: Object.fromEntries(Object.entries(principal.claims).filter(([name]) => claimNames.has(name))),
+    };
+    // A service run of the caller of this name that runs the same build file, services and
+    // endpoints moves to the commit, in place of a new run that would replace it: it keeps its
+    // id, its addresses and its volumes. A run that needs an approval is new as before.
+    if (up && source?.type === 'ref' && approvals.length === 0) {
+        const named = registry.named(principal, up.name);
+        if (named && runsTheSame(named, { definition: version.hash, accounts: planned.accounts, request, source, up })) {
+            const from = (registry.logs(principal, named.id)?.index.size ?? 0) + 1;
+            const moved = registry.move(principal, named.id, {
+                source,
+                plan: planned.plan,
+                owner,
+                ...(request.ttl ? { ttl } : {}),
+            });
+            if (moved) {
+                log(`move run ${moved.id} iss=${principal.iss} sub=${principal.sub} name=${up.name} to ${source.commit} (${moved.state})`);
+                (0, http_2.send)(res, 202, { ...toRunResponse(moved), moved: { logFrom: from } });
+                return;
+            }
+        }
+    }
+    const submit = (runSource) => registry.submit(owner, {
         definition: version.hash,
         tasks: request.tasks,
         accounts: planned.accounts,
         plan: planned.plan,
         source: runSource,
         options: request.options,
+        ...(up ? { up } : {}),
     }, approvals);
     try {
         // a sandbox is read and the run submitted while nothing else changes it: the run
@@ -346,10 +565,13 @@ async function submitRun(config, registry, definitions, sources, sandboxes, plan
                     commit: state.commit,
                     tree: state.tree,
                     clean: state.clean,
+                    ...(request.source.type === 'sandbox' && request.source.check ? { check: true } : {}),
                 });
             });
-        log(`run ${run.id} iss=${principal.iss} sub=${principal.sub} accounts=[${planned.accounts.join(',')}] source=${run.source.type}${run.source.type === 'sandbox' ? ` sandbox=${run.source.id}` : ''} tasks=${run.tasks.length}${approvals.length > 0 ? ` approval=[${approvals.join(',')}]` : ''}`);
-        (0, http_2.send)(res, 202, toRunResponse(run));
+        log(`run ${run.id} iss=${principal.iss} sub=${principal.sub} accounts=[${planned.accounts.join(',')}] source=${run.source.type}${run.source.type === 'sandbox' ? ` sandbox=${run.source.id}${run.source.check ? ' check' : ''}` : ''} tasks=${run.tasks.length}${run.up
+            ? ` services=[${run.up.services.join(',')}] name=${run.up.name} endpoints=${run.up.endpoints.length}`
+            : ''}${approvals.length > 0 ? ` approval=[${approvals.join(',')}]` : ''}`);
+        (0, http_2.send)(res, 202, toRunResponse(run, (0, overview_1.queuePlace)(registry, (0, overview_1.getViewer)(config, principal), run)));
     }
     catch (e) {
         if (e instanceof runs_1.RunError) {
@@ -385,7 +607,14 @@ async function authenticate(verifier, log, req, res) {
         return null;
     }
 }
-async function handle(config, registry, definitions, proposals, sources, sandboxes, planning, repos, verifier, log, req, res) {
+async function handle(api, verifier, req, res) {
+    const { config, gateway, log } = api;
+    // the hosts of endpoints are the apps of service runs, not the API
+    const label = gateway?.hostOf(req);
+    if (gateway && label) {
+        await gateway.handle(label, req, res);
+        return;
+    }
     const url = new URL(req.url ?? '/', 'http://localhost');
     const path = url.pathname;
     if (path === discovery_1.DISCOVERY_PATH || path === '/healthz') {
@@ -393,13 +622,30 @@ async function handle(config, registry, definitions, proposals, sources, sandbox
             (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET, HEAD' });
             return;
         }
-        (0, http_2.send)(res, 200, path === '/healthz' ? { status: 'ok' } : getDiscoveryDocument(config));
+        const dashboard = gateway?.dashboardUrl();
+        (0, http_2.send)(res, 200, path === '/healthz' ? { status: 'ok' } : { ...getDiscoveryDocument(config), ...(dashboard ? { dashboard } : {}) });
+        return;
+    }
+    // a person who opens the address of the server in a browser goes to the dashboard
+    const dashboard = gateway?.dashboardUrl();
+    if (path === '/' && req.method === 'GET' && dashboard && (req.headers.accept ?? '').includes('text/html')) {
+        res.writeHead(302, { location: dashboard, 'cache-control': 'no-store' });
+        res.end();
         return;
     }
     const principal = await authenticate(verifier, log, req, res);
     if (!principal) {
         return;
     }
+    await handleApi(api, principal, req, res, url);
+}
+/**
+ * The API for a caller who is known: by their token here, or by the sign-in of the
+ * dashboard, which asks the same API for what its pages show (dashboard.ts).
+ */
+async function handleApi(api, principal, req, res, url) {
+    const { config, registry, gateway, definitions, proposals, sources, sandboxes, planning, repos, checks, capacity, log, } = api;
+    const path = url.pathname;
     if (await (0, accounts_api_1.handleAccounts)(config, log, principal, req, res, url)) {
         return;
     }
@@ -412,7 +658,51 @@ async function handle(config, registry, definitions, proposals, sources, sandbox
     if (await (0, sandboxes_api_1.handleSandboxes)(config, sandboxes.registry, repos, log, principal, req, res, url)) {
         return;
     }
-    if (await handleRuns(config, registry, definitions, sources, sandboxes, planning, repos, log, principal, req, res, url)) {
+    if (await handleRuns(config, registry, definitions, sources, sandboxes, planning, repos, (host) => gateway?.url(host) ?? host, log, principal, req, res, url)) {
+        return;
+    }
+    if (path === '/v1/overview') {
+        if (req.method !== 'GET') {
+            (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET' });
+            return;
+        }
+        const viewer = (0, overview_1.getViewer)(config, principal);
+        if (viewer.grant.matched.length === 0) {
+            (0, http_2.send)(res, 403, { error: 'you are not entitled to anything on this server' });
+            return;
+        }
+        (0, http_2.send)(res, 200, (0, overview_1.getOverview)(registry.snapshot(), registry.usage(principal), viewer));
+        return;
+    }
+    if (path === '/v1/capacity' || path === '/v1/tasks') {
+        if (req.method !== 'GET') {
+            (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET' });
+            return;
+        }
+        const viewer = (0, overview_1.getViewer)(config, principal);
+        if (viewer.grant.matched.length === 0) {
+            (0, http_2.send)(res, 403, { error: 'you are not entitled to anything on this server' });
+            return;
+        }
+        if (path === '/v1/capacity') {
+            (0, http_2.send)(res, 200, (0, overview_1.visibleCapacity)(await capacity.read(registry.all()), viewer));
+            return;
+        }
+        // of every run the caller may see, unless they ask for their own
+        const params = new URLSearchParams(url.searchParams);
+        if (!params.has('scope')) {
+            params.set('scope', 'visible');
+        }
+        let filter;
+        try {
+            filter = (0, overview_1.parseRunFilter)(params, viewer.grant);
+        }
+        catch (e) {
+            (0, http_2.send)(res, e instanceof overview_1.FilterRefused ? 403 : 400, { error: e instanceof Error ? e.message : String(e) });
+            return;
+        }
+        const runs = (0, overview_1.filterRuns)(registry.all(), viewer, filter).map((run) => (0, overview_1.toListedRun)(viewer, run));
+        (0, http_2.send)(res, 200, { tasks: (0, timings_1.getTaskStats)(runs, Date.now()) });
         return;
     }
     if (await (0, approvals_api_1.handleApprovals)(config, registry, log, principal, req, res, url)) {
@@ -421,6 +711,9 @@ async function handle(config, registry, definitions, proposals, sources, sandbox
     if (await (0, proposals_api_1.handleProposals)(config, definitions, proposals, log, principal, req, res, url)) {
         return;
     }
+    if (await (0, checks_api_1.handleChecks)(config, checks, definitions, log, principal, req, res, url)) {
+        return;
+    }
     (0, http_2.send)(res, 404, { error: 'not found' });
 }
 async function startServer(config, environment, options = {}) {
@@ -447,12 +740,13 @@ async function startServer(config, environment, options = {}) {
         },
         policy: {
             envs: config.runs.env.allow,
-            reserved: (0, server_config_1.getSourceAccounts)(config),
+            reserved: (0, server_config_1.getServerAccounts)(config),
             runtimes: config.runtimes,
             backend: config.backend.type,
             network: (0, work_network_1.parseWorkNetwork)(config.runs.network),
             secretNetwork: (0, work_network_1.parseWorkNetwork)(config.runs.secretNetwork),
             args: config.runs.args,
+            capabilities: config.backend.capabilities?.allow ?? [],
             proxiedProviders: config.backend.type === 'kubernetes'
                 ? Object.entries(config.backend.providers)
                     .filter(([, provider]) => provider.allow)
@@ -493,6 +787,8 @@ async function startServer(config, environment, options = {}) {
             envs: config.sandboxes.env.allow,
             envsFrom: 'sandboxes.env.allow',
             runtimes: sandboxRuntimes,
+            capabilities: config.sandboxes.capabilities,
+            capabilitiesFrom: 'sandboxes.capabilities',
         },
     };
     if (sandboxRuntimes.length === 0 && config.entitlements.some((entitlement) => entitlement.sandboxes?.length)) {
@@ -512,6 +808,9 @@ async function startServer(config, environment, options = {}) {
         cache: (0, path_1.join)(storage, 'cache'),
         worker: options.worker,
         sandboxes: { ...sandboxes, network: config.sandboxes.network },
+        ...(config.backend.type === 'host' && config.backend.capabilities
+            ? { capabilityRuntime: config.backend.capabilities.runtime }
+            : {}),
         log,
     };
     const backend = config.backend;
@@ -519,7 +818,16 @@ async function startServer(config, environment, options = {}) {
         (backend.type === 'kubernetes'
             ? new kubernetes_backend_1.KubernetesExecutor({ ...config, backend }, environment.processEnvs, executorOptions)
             : new executor_1.HostExecutor(executorOptions));
+    const capacity = (0, capacity_1.cachedCapacity)(options.capacity ??
+        (executor instanceof kubernetes_backend_1.KubernetesExecutor
+            ? executor.capacity()
+            : new capacity_1.HostCapacity((0, execute_docker_1.createDockerClient)(undefined), !environment.processEnvs.DOCKER_HOST)));
     const sources = new sources_1.SourceStore((0, path_1.join)(storage, 'sources'));
+    const checks = new checks_1.CheckStore((0, path_1.join)(storage, 'checks'), { keep: (0, units_1.parseDuration)(config.runs.checksKeep) });
+    const sweepChecks = () => checks.sweep().catch((e) => log(`the old checks could not be removed: ${e instanceof Error ? e.message : e}`));
+    void sweepChecks();
+    const sweeping = setInterval(sweepChecks, CHECK_SWEEP_MS);
+    sweeping.unref();
     const registry = new runs_1.RunRegistry({
         executor,
         maxConcurrent: config.runs.maxConcurrent,
@@ -529,21 +837,74 @@ async function startServer(config, environment, options = {}) {
         limits: config.runs.limits,
         maxDuration: (0, units_1.parseDuration)(config.runs.maxDuration),
         maxLogBytes: config.runs.maxLogBytes,
+        up: {
+            maxTtl: (0, units_1.parseDuration)(config.runs.up.maxTtl),
+            perCaller: config.runs.up.limits.up,
+            asleep: config.runs.up.limits.asleep,
+            total: config.runs.up.total,
+            keepVolumes: config.runs.up.keepVolumes,
+        },
         store: new run_store_1.FileRunStore((0, path_1.join)(storage, 'records')),
+        // what a check run passed, for `remote verify` to find
+        onFinished: (run, keys) => {
+            const passed = (0, checks_1.getCheckRecords)(run, keys);
+            if (passed.length > 0) {
+                checks
+                    .save(passed)
+                    .then(() => log(`checks run=${run.id} passed=[${passed.map((record) => record.task).join(',')}]`))
+                    .catch((e) => log(`the checks of run ${run.id} were not kept: ${e instanceof Error ? e.message : e}`));
+            }
+        },
         log,
     });
+    // the executor hears which service runs sleep with their volumes kept, and removes what it
+    // kept for any other (RunExecutor.restored)
     await registry.restore();
-    sandboxes.registry.useRuns({
-        holder: (id) => registry.holder(id),
-        stop: (id) => registry.stopSandbox(id, 'the sandbox was deleted'),
-    });
     const proposals = new proposals_1.ProposalRegistry({
         maxWaiting: config.approvals.maxWaiting,
         timeout: (0, units_1.parseDuration)(config.approvals.timeout),
         cooldown: (0, units_1.parseDuration)(config.approvals.cooldown),
     });
+    const login = config.endpoints?.login;
+    const gateway = config.endpoints
+        ? new endpoints_1.EndpointGateway(config, registry, {
+            keyFile: (0, path_1.join)(storage, 'endpoints', 'session.key'),
+            verifier,
+            ...(login?.clientSecret
+                ? { clientSecret: () => repos.serverSecret(login.clientSecret) }
+                : {}),
+            log,
+            ...(options.fetch ? { fetch: options.fetch } : {}),
+            ...(options.now ? { now: options.now } : {}),
+            mayWake: (run) => refuseWake(config, repos, run),
+            // the pages of the dashboard read and do what they show through the API
+            dashboard: {
+                api: (principal, req, res, url) => handleApi(api, principal, req, res, url),
+                ...(options.dashboardAssets ? { assets: options.dashboardAssets } : {}),
+            },
+        })
+        : undefined;
+    const api = {
+        config,
+        registry,
+        gateway,
+        definitions,
+        proposals,
+        sources,
+        sandboxes,
+        planning,
+        repos,
+        checks,
+        capacity,
+        log,
+    };
+    await gateway?.load();
+    sandboxes.registry.useRuns({
+        holder: (id) => registry.holder(id),
+        stop: (id) => registry.stopSandbox(id, 'the sandbox was deleted'),
+    });
     const listener = (req, res) => {
-        handle(config, registry, definitions, proposals, sources, sandboxes, planning, repos, verifier, log, req, res).catch((e) => {
+        handle(api, verifier, req, res).catch((e) => {
             if (e instanceof http_2.HttpError) {
                 (0, http_2.send)(res, e.status, { error: e.message });
                 return;
@@ -563,11 +924,44 @@ async function startServer(config, environment, options = {}) {
             environment.file.read(config.tls.cert),
             environment.file.read(config.tls.key),
         ]);
-        server = (0, https_1.createServer)({ cert, key }, listener);
+        // the hosts of endpoints have a wildcard certificate of their own
+        const endpointsTls = config.endpoints?.tls;
+        const endpointContext = endpointsTls
+            ? (0, tls_1.createSecureContext)({
+                cert: await environment.file.read(endpointsTls.cert),
+                key: await environment.file.read(endpointsTls.key),
+            })
+            : undefined;
+        const domain = config.endpoints?.domain;
+        server = (0, https_1.createServer)({
+            cert,
+            key,
+            ...(endpointContext && domain
+                ? {
+                    SNICallback: (servername, done) => done(null, servername.toLowerCase().endsWith(`.${domain}`) ? endpointContext : undefined),
+                }
+                : {}),
+        }, listener);
     }
     else {
         server = (0, http_1.createServer)(listener);
     }
+    // WebSockets of the apps behind endpoints; the API has none. An upgraded connection is
+    // no longer the HTTP server's, it goes when the server closes.
+    const upgraded = new Set();
+    server.on('upgrade', (req, socket, head) => {
+        upgraded.add(socket);
+        socket.once('close', () => upgraded.delete(socket));
+        const label = gateway?.hostOf(req);
+        if (!gateway || !label) {
+            socket.end('HTTP/1.1 404 Not Found\r\nconnection: close\r\ncontent-length: 0\r\n\r\n');
+            return;
+        }
+        gateway.upgrade(label, req, socket, head).catch((e) => {
+            log(`error upgrade ${label}: ${e instanceof Error ? e.message : String(e)}`);
+            socket.destroy();
+        });
+    });
     await new Promise((resolve, reject) => {
         server.once('error', reject);
         server.listen(config.listen.port, config.listen.host, () => {
@@ -576,11 +970,12 @@ async function startServer(config, environment, options = {}) {
         });
     });
     const address = server.address();
+    gateway?.listening(address.port, Boolean(config.tls));
     const host = address.family === 'IPv6' ? `[${address.address}]` : address.address;
     return {
         url: `${config.tls ? 'https' : 'http'}://${host}:${address.port}`,
         reload: (next) => {
-            const readers = (0, server_config_1.getSourceAccounts)(config);
+            const readers = (0, server_config_1.getServerAccounts)(config);
             next.entitlements.forEach((entitlement, index) => {
                 const iss = (entitlement.subject ?? entitlement.group ?? entitlement.claims)?.iss;
                 if (iss && !config.issuers.some((issuer) => issuer.issuer === iss)) {
@@ -600,11 +995,13 @@ async function startServer(config, environment, options = {}) {
             return { restart };
         },
         close: async () => {
+            clearInterval(sweeping);
             proposals.close();
             await registry.close();
             await new Promise((resolve, reject) => {
                 server.close((err) => (err ? reject(err) : resolve()));
                 server.closeAllConnections();
+                upgraded.forEach((socket) => socket.destroy());
             });
         },
     };