hammerkit 1.27.0-next.2 → 1.27.0-next.20

package/dist/server/server.js

@@ -1,13 +1,15 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.startServer = exports.getDiscoveryDocument = exports.SERVER_PROTOCOL = exports.DISCOVERY_PATH = void 0;
+exports.startServer = exports.refuseWake = exports.getDiscoveryDocument = exports.SERVER_PROTOCOL = exports.DISCOVERY_PATH = void 0;
 const http_1 = require("http");
 const https_1 = require("https");
+const tls_1 = require("tls");
 const version_1 = require("../version");
 const server_config_1 = require("./server-config");
 const discovery_1 = require("./discovery");
 const oauth_1 = require("./oauth");
 const policy_1 = require("./policy");
+const repo_1 = require("./repo");
 const run_store_1 = require("./run-store");
 const runs_1 = require("./runs");
 const git_checkout_1 = require("./git-checkout");
@@ -32,10 +34,14 @@ const sandbox_storage_1 = require("./sandbox-storage");
 const kubernetes_sandbox_storage_1 = require("./kubernetes-sandbox-storage");
 const sandbox_docker_1 = require("./sandbox-docker");
 const sandboxes_api_1 = require("./sandboxes-api");
+const endpoints_1 = require("./endpoints");
+const service_runs_1 = require("./service-runs");
 const units_1 = require("../utils/units");
 const run_logs_1 = require("./run-logs");
 const get_cache_directory_1 = require("../optimizer/get-cache-directory");
 const path_1 = require("path");
+const checks_1 = require("./checks");
+const checks_api_1 = require("./checks-api");
 var discovery_2 = require("./discovery");
 Object.defineProperty(exports, "DISCOVERY_PATH", { enumerable: true, get: function () { return discovery_2.DISCOVERY_PATH; } });
 Object.defineProperty(exports, "SERVER_PROTOCOL", { enumerable: true, get: function () { return discovery_2.SERVER_PROTOCOL; } });
@@ -64,6 +70,8 @@ const HEARTBEAT_MS = 15000;
 // the longest `?wait=` of a run: below the 60 seconds after which a proxy in front of the
 // server (nginx by default) closes a connection that says nothing
 const MAX_WAIT_MS = 50000;
+// how often checks older than `runs.checksKeep` are removed
+const CHECK_SWEEP_MS = 6 * 60 * 60 * 1000;
 // Server-sent events: `id` is the sequence number, so a client that lost the
 // connection continues with Last-Event-ID (or ?after=) and misses nothing.
 function streamEvents(registry, config, principal, id, req, res, url) {
@@ -96,8 +104,8 @@ function streamEvents(registry, config, principal, id, req, res, url) {
         watch.stop();
     });
 }
-async function handleRuns(config, registry, definitions, sources, sandboxes, planning, repos, log, principal, req, res, url) {
-    const match = /^\/v1\/runs(?:\/([^/]+)(?:\/(cancel|events|logs))?)?$/.exec(url.pathname);
+async function handleRuns(config, registry, definitions, sources, sandboxes, planning, repos, endpointUrl, log, principal, req, res, url) {
+    const match = /^\/v1\/runs(?:\/([^/]+)(?:\/(cancel|events|logs|extend))?)?$/.exec(url.pathname);
     if (!match) {
         return false;
     }
@@ -115,7 +123,7 @@ async function handleRuns(config, registry, definitions, sources, sandboxes, pla
             });
         }
         else if (method === 'POST') {
-            await submitRun(config, registry, definitions, sources, sandboxes, planning, repos, log, principal, req, res);
+            await submitRun(config, registry, definitions, sources, sandboxes, planning, repos, endpointUrl, log, principal, req, res);
         }
         else {
             (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET, POST' });
@@ -134,6 +142,13 @@ async function handleRuns(config, registry, definitions, sources, sandboxes, pla
         log(`cancel iss=${principal.iss} sub=${principal.sub} run=${id} found=${Boolean(run)}`);
         run ? (0, http_2.send)(res, 200, toRunResponse(run)) : (0, http_2.send)(res, 404, { error: 'not found' });
     }
+    else if (action === 'extend') {
+        if (method !== 'POST') {
+            (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'POST' });
+            return true;
+        }
+        await extendRun(config, registry, log, principal, id, req, res);
+    }
     else if (method !== 'GET') {
         (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET' });
     }
@@ -181,12 +196,75 @@ function accountsOf(config, principal) {
     const grant = (0, policy_1.getGrant)(config.entitlements, principal);
     return (account) => (0, policy_1.canUse)(grant, account);
 }
+async function extendRun(config, registry, log, principal, id, req, res) {
+    const body = (await (0, http_2.readJson)(req));
+    let ttl;
+    try {
+        if (typeof body !== 'object' || body === null || typeof body.ttl !== 'string') {
+            throw new Error();
+        }
+        ttl = (0, units_1.parseDuration)(body.ttl);
+        if (!(ttl > 0)) {
+            throw new Error();
+        }
+    }
+    catch {
+        (0, http_2.send)(res, 400, { error: 'ttl: expected a duration like 30m, 8h or 1d' });
+        return;
+    }
+    const grant = (0, policy_1.getGrant)(config.entitlements, principal);
+    const result = registry.extend(principal, id, ttl, (run) => (0, service_runs_1.mayOpenEndpoints)(grant, run));
+    log(`extend iss=${principal.iss} sub=${principal.sub} run=${id} result=${result.status}`);
+    if (result.status === 'not-found') {
+        (0, http_2.send)(res, 404, { error: 'not found' });
+    }
+    else if (result.status === 'not-up') {
+        (0, http_2.send)(res, 409, { error: `the run is ${result.state}, only a service run that is up can be extended` });
+    }
+    else {
+        (0, http_2.send)(res, 200, toRunResponse(result.run));
+    }
+}
+// Why the owner of a service run that sleeps may no longer run it, with the entitlements of
+// the moment: a wake is started by who opens an endpoint, but runs as the owner, as a new
+// run of theirs would.
+function refuseWake(config, repos, run) {
+    if (run.source.type !== 'ref') {
+        return 'a run from a sandbox does not wake';
+    }
+    const grant = (0, policy_1.getGrant)(config.entitlements, {
+        iss: run.owner.iss,
+        sub: run.owner.sub,
+        groups: run.owner.groups ?? [],
+        claims: run.owner.claims ?? {},
+    });
+    const decision = (0, policy_1.decideRun)({
+        grant,
+        accounts: run.accounts,
+        catalog: config.accounts,
+        source: { type: 'ref', repo: run.source.repo, ref: run.source.ref },
+        // an account it may only request is refused, the wake has nobody to approve it
+        approvable: () => true,
+        readable: (repo) => repos.readable(repo),
+    });
+    if (decision.type === 'refuse') {
+        return `its owner ${run.owner.sub} may no longer run it: ${decision.error.replace(/^you /, 'they ')}`;
+    }
+    if (decision.type === 'approval') {
+        return `its owner ${run.owner.sub} may only request ${decision.accounts.join(', ')} now, start it again`;
+    }
+    if (run.up?.endpoints.length && (0, policy_1.endpointLevel)(grant, run.source.repo) !== 'publish') {
+        return `its owner ${run.owner.sub} may no longer publish endpoints of ${run.source.repo}`;
+    }
+    return undefined;
+}
+exports.refuseWake = refuseWake;
 function toRunResponse(run) {
     const { owner, ...visible } = run;
     void owner;
     return { ...visible, results: run.results ?? {} };
 }
-async function submitRun(config, registry, definitions, sources, sandboxes, planning, repos, log, principal, req, res) {
+async function submitRun(config, registry, definitions, sources, sandboxes, planning, repos, endpointUrl, log, principal, req, res) {
     const parsed = run_request_1.runRequestSchema.safeParse(await (0, http_2.readJson)(req));
     if (!parsed.success) {
         const issue = parsed.error.issues[0];
@@ -218,6 +296,27 @@ async function submitRun(config, registry, definitions, sources, sandboxes, plan
         (0, http_2.send)(res, 400, { error: `${version.hash} has no task ${unknown.map(oauth_1.quote).join(', ')}` });
         return;
     }
+    const notServices = request.services.filter((service) => !version.services.includes(service));
+    if (notServices.length > 0) {
+        (0, http_2.send)(res, 400, { error: `${version.hash} has no service ${notServices.map(oauth_1.quote).join(', ')}` });
+        return;
+    }
+    if (request.endpoints.length > 0 && !config.endpoints) {
+        (0, http_2.send)(res, 400, { error: 'this server publishes no endpoints (endpoints is not configured)' });
+        return;
+    }
+    const ttl = request.services.length > 0 ? (0, units_1.parseDuration)(request.ttl ?? config.runs.up.ttl) : 0;
+    if (ttl > (0, units_1.parseDuration)(config.runs.up.maxTtl)) {
+        (0, http_2.send)(res, 400, { error: `ttl: at most ${config.runs.up.maxTtl} on this server (runs.up.maxTtl)` });
+        return;
+    }
+    // the files of a sandbox change under a run that sleeps, it would wake to others
+    if (request.sleepTtl !== undefined && request.source.type === 'sandbox') {
+        (0, http_2.send)(res, 400, { error: 'sleepTtl: a service run from a sandbox does not sleep, start it from a branch' });
+        return;
+    }
+    const sleepTtlText = request.source.type === 'ref' ? request.sleepTtl ?? config.runs.up.sleepTtl : undefined;
+    const sleepTtl = request.services.length > 0 && sleepTtlText ? (0, units_1.parseDuration)(sleepTtlText) : undefined;
     const argued = (request.options.args ?? []).length > 0 ? request.tasks[0] : undefined;
     if (argued !== undefined && version.services.includes(argued)) {
         (0, http_2.send)(res, 400, { error: `${argued} is a service, arguments go to a task` });
@@ -239,6 +338,12 @@ async function submitRun(config, registry, definitions, sources, sandboxes, plan
             return;
         }
     }
+    if (request.services.length > 0 && request.source.type === 'sandbox' && config.backend.type === 'kubernetes') {
+        (0, http_2.send)(res, 400, {
+            error: 'on a cluster a service run starts from a branch, push the sandbox and start it from there',
+        });
+        return;
+    }
     const sandbox = request.source.type === 'sandbox' ? sandboxes.registry.get(principal, request.source.id) : undefined;
     if (request.source.type === 'sandbox') {
         if (!sandbox) {
@@ -257,6 +362,24 @@ async function submitRun(config, registry, definitions, sources, sandboxes, plan
             (0, http_2.send)(res, 413, { error: over });
             return;
         }
+        // a check counts like a run from a commit of the repository, and leaves what it made in
+        // the repository's cache, which runs from main restore from: the caller must be allowed
+        // to start runs from it, and to push a branch of it, which with a run from that branch
+        // leaves the same
+        if (request.source.check && !(0, repo_1.matchesAnyRepo)(grant.repos, sandbox.repo)) {
+            log(`refuse run iss=${principal.iss} sub=${principal.sub}: check on ${sandbox.repo} without the repository`);
+            (0, http_2.send)(res, 403, {
+                error: `a check counts like a run from ${sandbox.repo}, which no entitlement of yours lists in repos`,
+            });
+            return;
+        }
+        if (request.source.check && (0, policy_1.sandboxPushBranches)(grant, sandbox.repo).length === 0) {
+            log(`refuse run iss=${principal.iss} sub=${principal.sub}: check on ${sandbox.repo} without a branch to push`);
+            (0, http_2.send)(res, 403, {
+                error: `a check counts like a run from a branch you pushed, and you may push no branch of ${sandbox.repo}`,
+            });
+            return;
+        }
     }
     // the accounts of a run are read from the files it is pinned to, with the engine
     const bundle = version.plan ? await definitions.files(version) : undefined;
@@ -275,12 +398,20 @@ async function submitRun(config, registry, definitions, sources, sandboxes, plan
         (0, http_2.send)(res, 400, { error: `the run does not load: ${e instanceof Error ? e.message : String(e)}` });
         return;
     }
-    const refused = (0, run_plan_1.refuseArguments)(request, planned.plan);
+    const refused = (0, run_plan_1.refuseArguments)(request, planned.plan, sandbox ? sandboxes.policy : planning.policy) ??
+        (0, service_runs_1.refuseEndpoints)(request, planned.plan, config.runs.secretNetwork === 'open');
     if (refused) {
         log(`refuse run iss=${principal.iss} sub=${principal.sub}: ${refused}`);
         (0, http_2.send)(res, 400, { error: refused });
         return;
     }
+    const runRepo = request.source.type === 'ref' ? request.source.repo : sandbox?.repo;
+    if (request.endpoints.length > 0 && (!runRepo || (0, policy_1.endpointLevel)(grant, runRepo) !== 'publish')) {
+        const error = `you may not publish endpoints of ${runRepo ?? 'this source'}, an entitlement needs endpoints with level publish for it`;
+        log(`refuse run iss=${principal.iss} sub=${principal.sub}: ${error}`);
+        (0, http_2.send)(res, 403, { error });
+        return;
+    }
     const decision = (0, policy_1.decideRun)({
         grant,
         accounts: planned.accounts,
@@ -304,6 +435,12 @@ async function submitRun(config, registry, definitions, sources, sandboxes, plan
     if (request.source.type === 'ref') {
         try {
             const commit = await repos.resolve(request.source.repo, request.source.ref);
+            if (request.source.commit && commit !== request.source.commit) {
+                const error = `${request.source.ref} is at ${commit} now, not at ${request.source.commit}`;
+                log(`refuse run iss=${principal.iss} sub=${principal.sub}: ${error}`);
+                (0, http_2.send)(res, 409, { error });
+                return;
+            }
             source = { type: 'ref', repo: request.source.repo, ref: request.source.ref, commit };
         }
         catch (e) {
@@ -318,13 +455,34 @@ async function submitRun(config, registry, definitions, sources, sandboxes, plan
     else if (request.source.type === 'upload') {
         source = request.source;
     }
-    const submit = (runSource) => registry.submit(principal, {
+    // a service run is named after its branch unless it says, and its endpoints after that
+    const serviceName = request.services.length > 0
+        ? request.name ??
+            (0, service_runs_1.getServiceRunName)(request.source.type === 'ref' ? request.source.ref : sandbox?.ref)
+        : '';
+    const up = request.services.length > 0
+        ? {
+            services: request.services,
+            ttl,
+            ...(sleepTtl ? { sleepTtl } : {}),
+            name: serviceName,
+            endpoints: (0, service_runs_1.toRunEndpoints)(request.endpoints, (0, service_runs_1.getEndpointHosts)(serviceName, principal, request.endpoints), endpointUrl),
+        }
+        : undefined;
+    // a run that may sleep keeps what its owner is entitled by, for the check of a wake
+    const claimNames = new Set(config.entitlements.flatMap((entitlement) => Object.keys(entitlement.claims?.match ?? {})));
+    const owner = {
+        ...principal,
+        claims: Object.fromEntries(Object.entries(principal.claims).filter(([name]) => claimNames.has(name))),
+    };
+    const submit = (runSource) => registry.submit(owner, {
         definition: version.hash,
         tasks: request.tasks,
         accounts: planned.accounts,
         plan: planned.plan,
         source: runSource,
         options: request.options,
+        ...(up ? { up } : {}),
     }, approvals);
     try {
         // a sandbox is read and the run submitted while nothing else changes it: the run
@@ -340,9 +498,12 @@ async function submitRun(config, registry, definitions, sources, sandboxes, plan
                     commit: state.commit,
                     tree: state.tree,
                     clean: state.clean,
+                    ...(request.source.type === 'sandbox' && request.source.check ? { check: true } : {}),
                 });
             });
-        log(`run ${run.id} iss=${principal.iss} sub=${principal.sub} accounts=[${planned.accounts.join(',')}] source=${run.source.type}${run.source.type === 'sandbox' ? ` sandbox=${run.source.id}` : ''} tasks=${run.tasks.length}${approvals.length > 0 ? ` approval=[${approvals.join(',')}]` : ''}`);
+        log(`run ${run.id} iss=${principal.iss} sub=${principal.sub} accounts=[${planned.accounts.join(',')}] source=${run.source.type}${run.source.type === 'sandbox' ? ` sandbox=${run.source.id}${run.source.check ? ' check' : ''}` : ''} tasks=${run.tasks.length}${run.up
+            ? ` services=[${run.up.services.join(',')}] name=${run.up.name} endpoints=${run.up.endpoints.length}`
+            : ''}${approvals.length > 0 ? ` approval=[${approvals.join(',')}]` : ''}`);
         (0, http_2.send)(res, 202, toRunResponse(run));
     }
     catch (e) {
@@ -379,7 +540,13 @@ async function authenticate(verifier, log, req, res) {
         return null;
     }
 }
-async function handle(config, registry, definitions, proposals, sources, sandboxes, planning, repos, verifier, log, req, res) {
+async function handle(config, registry, gateway, definitions, proposals, sources, sandboxes, planning, repos, checks, verifier, log, req, res) {
+    // the hosts of endpoints are the apps of service runs, not the API
+    const label = gateway?.hostOf(req);
+    if (gateway && label) {
+        await gateway.handle(label, req, res);
+        return;
+    }
     const url = new URL(req.url ?? '/', 'http://localhost');
     const path = url.pathname;
     if (path === discovery_1.DISCOVERY_PATH || path === '/healthz') {
@@ -406,7 +573,7 @@ async function handle(config, registry, definitions, proposals, sources, sandbox
     if (await (0, sandboxes_api_1.handleSandboxes)(config, sandboxes.registry, repos, log, principal, req, res, url)) {
         return;
     }
-    if (await handleRuns(config, registry, definitions, sources, sandboxes, planning, repos, log, principal, req, res, url)) {
+    if (await handleRuns(config, registry, definitions, sources, sandboxes, planning, repos, (host) => gateway?.url(host) ?? host, log, principal, req, res, url)) {
         return;
     }
     if (await (0, approvals_api_1.handleApprovals)(config, registry, log, principal, req, res, url)) {
@@ -415,6 +582,9 @@ async function handle(config, registry, definitions, proposals, sources, sandbox
     if (await (0, proposals_api_1.handleProposals)(config, definitions, proposals, log, principal, req, res, url)) {
         return;
     }
+    if (await (0, checks_api_1.handleChecks)(config, checks, definitions, log, principal, req, res, url)) {
+        return;
+    }
     (0, http_2.send)(res, 404, { error: 'not found' });
 }
 async function startServer(config, environment, options = {}) {
@@ -436,14 +606,22 @@ async function startServer(config, environment, options = {}) {
         environment: {
             ...environment,
             processEnvs: (0, run_environment_1.runEnvironment)(config, environment.processEnvs),
-            hosted: { catalog: (0, server_config_1.getSecretCatalog)(config) },
+            // on a cluster a local task runs in a pod of a pool's tool image
+            hosted: { catalog: (0, server_config_1.getSecretCatalog)(config), localTasksInContainers: config.backend.type === 'kubernetes' },
         },
         policy: {
             envs: config.runs.env.allow,
-            reserved: (0, server_config_1.getSourceAccounts)(config),
+            reserved: (0, server_config_1.getServerAccounts)(config),
             runtimes: config.runtimes,
             backend: config.backend.type,
             network: (0, work_network_1.parseWorkNetwork)(config.runs.network),
+            secretNetwork: (0, work_network_1.parseWorkNetwork)(config.runs.secretNetwork),
+            args: config.runs.args,
+            proxiedProviders: config.backend.type === 'kubernetes'
+                ? Object.entries(config.backend.providers)
+                    .filter(([, provider]) => provider.allow)
+                    .map(([name]) => name)
+                : [],
         },
     };
     const storage = config.storage.directory ?? (0, path_1.join)((0, get_cache_directory_1.getHammerkitDirectory)(), 'server');
@@ -506,6 +684,11 @@ async function startServer(config, environment, options = {}) {
             ? new kubernetes_backend_1.KubernetesExecutor({ ...config, backend }, environment.processEnvs, executorOptions)
             : new executor_1.HostExecutor(executorOptions));
     const sources = new sources_1.SourceStore((0, path_1.join)(storage, 'sources'));
+    const checks = new checks_1.CheckStore((0, path_1.join)(storage, 'checks'), { keep: (0, units_1.parseDuration)(config.runs.checksKeep) });
+    const sweepChecks = () => checks.sweep().catch((e) => log(`the old checks could not be removed: ${e instanceof Error ? e.message : e}`));
+    void sweepChecks();
+    const sweeping = setInterval(sweepChecks, CHECK_SWEEP_MS);
+    sweeping.unref();
     const registry = new runs_1.RunRegistry({
         executor,
         maxConcurrent: config.runs.maxConcurrent,
@@ -515,10 +698,44 @@ async function startServer(config, environment, options = {}) {
         limits: config.runs.limits,
         maxDuration: (0, units_1.parseDuration)(config.runs.maxDuration),
         maxLogBytes: config.runs.maxLogBytes,
+        up: {
+            maxTtl: (0, units_1.parseDuration)(config.runs.up.maxTtl),
+            perCaller: config.runs.up.limits.up,
+            total: config.runs.up.total,
+        },
         store: new run_store_1.FileRunStore((0, path_1.join)(storage, 'records')),
+        // what a check run passed, for `remote verify` to find
+        onFinished: (run, keys) => {
+            const passed = (0, checks_1.getCheckRecords)(run, keys);
+            if (passed.length > 0) {
+                checks
+                    .save(passed)
+                    .then(() => log(`checks run=${run.id} passed=[${passed.map((record) => record.task).join(',')}]`))
+                    .catch((e) => log(`the checks of run ${run.id} were not kept: ${e instanceof Error ? e.message : e}`));
+            }
+        },
         log,
     });
     await registry.restore();
+    if (executor instanceof kubernetes_backend_1.KubernetesExecutor) {
+        // the services of the runs of a server that stopped are gone with it
+        executor.sweepServiceRuns();
+    }
+    const login = config.endpoints?.login;
+    const gateway = config.endpoints
+        ? new endpoints_1.EndpointGateway(config, registry, {
+            keyFile: (0, path_1.join)(storage, 'endpoints', 'session.key'),
+            verifier,
+            ...(login?.clientSecret
+                ? { clientSecret: () => repos.serverSecret(login.clientSecret) }
+                : {}),
+            log,
+            ...(options.fetch ? { fetch: options.fetch } : {}),
+            ...(options.now ? { now: options.now } : {}),
+            mayWake: (run) => refuseWake(config, repos, run),
+        })
+        : undefined;
+    await gateway?.load();
     sandboxes.registry.useRuns({
         holder: (id) => registry.holder(id),
         stop: (id) => registry.stopSandbox(id, 'the sandbox was deleted'),
@@ -529,7 +746,7 @@ async function startServer(config, environment, options = {}) {
         cooldown: (0, units_1.parseDuration)(config.approvals.cooldown),
     });
     const listener = (req, res) => {
-        handle(config, registry, definitions, proposals, sources, sandboxes, planning, repos, verifier, log, req, res).catch((e) => {
+        handle(config, registry, gateway, definitions, proposals, sources, sandboxes, planning, repos, checks, verifier, log, req, res).catch((e) => {
             if (e instanceof http_2.HttpError) {
                 (0, http_2.send)(res, e.status, { error: e.message });
                 return;
@@ -549,11 +766,44 @@ async function startServer(config, environment, options = {}) {
             environment.file.read(config.tls.cert),
             environment.file.read(config.tls.key),
         ]);
-        server = (0, https_1.createServer)({ cert, key }, listener);
+        // the hosts of endpoints have a wildcard certificate of their own
+        const endpointsTls = config.endpoints?.tls;
+        const endpointContext = endpointsTls
+            ? (0, tls_1.createSecureContext)({
+                cert: await environment.file.read(endpointsTls.cert),
+                key: await environment.file.read(endpointsTls.key),
+            })
+            : undefined;
+        const domain = config.endpoints?.domain;
+        server = (0, https_1.createServer)({
+            cert,
+            key,
+            ...(endpointContext && domain
+                ? {
+                    SNICallback: (servername, done) => done(null, servername.toLowerCase().endsWith(`.${domain}`) ? endpointContext : undefined),
+                }
+                : {}),
+        }, listener);
     }
     else {
         server = (0, http_1.createServer)(listener);
     }
+    // WebSockets of the apps behind endpoints; the API has none. An upgraded connection is
+    // no longer the HTTP server's, it goes when the server closes.
+    const upgraded = new Set();
+    server.on('upgrade', (req, socket, head) => {
+        upgraded.add(socket);
+        socket.once('close', () => upgraded.delete(socket));
+        const label = gateway?.hostOf(req);
+        if (!gateway || !label) {
+            socket.end('HTTP/1.1 404 Not Found\r\nconnection: close\r\ncontent-length: 0\r\n\r\n');
+            return;
+        }
+        gateway.upgrade(label, req, socket, head).catch((e) => {
+            log(`error upgrade ${label}: ${e instanceof Error ? e.message : String(e)}`);
+            socket.destroy();
+        });
+    });
     await new Promise((resolve, reject) => {
         server.once('error', reject);
         server.listen(config.listen.port, config.listen.host, () => {
@@ -562,11 +812,12 @@ async function startServer(config, environment, options = {}) {
         });
     });
     const address = server.address();
+    gateway?.listening(address.port, Boolean(config.tls));
     const host = address.family === 'IPv6' ? `[${address.address}]` : address.address;
     return {
         url: `${config.tls ? 'https' : 'http'}://${host}:${address.port}`,
         reload: (next) => {
-            const readers = (0, server_config_1.getSourceAccounts)(config);
+            const readers = (0, server_config_1.getServerAccounts)(config);
             next.entitlements.forEach((entitlement, index) => {
                 const iss = (entitlement.subject ?? entitlement.group ?? entitlement.claims)?.iss;
                 if (iss && !config.issuers.some((issuer) => issuer.issuer === iss)) {
@@ -586,11 +837,13 @@ async function startServer(config, environment, options = {}) {
             return { restart };
         },
         close: async () => {
+            clearInterval(sweeping);
             proposals.close();
             await registry.close();
             await new Promise((resolve, reject) => {
                 server.close((err) => (err ? reject(err) : resolve()));
                 server.closeAllConnections();
+                upgraded.forEach((socket) => socket.destroy());
             });
         },
     };

package/dist/server/service-runs.js

@@ -0,0 +1,82 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mayOpenEndpoints = exports.getRunRepo = exports.toRunEndpoints = exports.refuseEndpoints = exports.getEndpointHosts = exports.getServiceRunName = void 0;
+const crypto_1 = require("crypto");
+const plan_1 = require("./plan");
+const policy_1 = require("./policy");
+const run_request_1 = require("./run-request");
+// The name of a service run when the request names none: the branch it runs from, as a DNS
+// label, so the runs of several branches stand side by side and each keeps its hosts.
+function getServiceRunName(branch) {
+    const name = (branch ?? '')
+        .toLowerCase()
+        .replace(/[^a-z0-9-]+/g, '-')
+        .replace(/-+/g, '-')
+        .slice(0, 40)
+        .replace(/^-+|-+$/g, '');
+    return run_request_1.SERVICE_RUN_NAME_PATTERN.test(name) ? name : 'run';
+}
+exports.getServiceRunName = getServiceRunName;
+/**
+ * The host labels of a run's endpoints: its name, the port and twelve hex digits of its
+ * owner (at most 40 + 5 + 2 + 12 and three dashes: a DNS label), so two callers with a branch of the same name get hosts of their own, and a run
+ * that replaces another of the same name takes over its hosts. Two endpoints on the same
+ * port (of two services) are told apart by their place.
+ */
+function getEndpointHosts(name, owner, endpoints) {
+    const of = (0, crypto_1.createHash)('sha256').update(`${owner.iss}\0${owner.sub}`).digest('hex').slice(0, 12);
+    return endpoints.map((endpoint, index) => endpoints.findIndex((other) => other.port === endpoint.port) === index
+        ? `${name}-${endpoint.port}-${of}`
+        : `${name}-${endpoint.port}-${index}-${of}`);
+}
+exports.getEndpointHosts = getEndpointHosts;
+// The container port of each `host:container` port of a planned service.
+function containerPorts(plan) {
+    return plan.ports.map((port) => Number(port.split(':').pop()));
+}
+/**
+ * Why the endpoints of a service run can't be published, or null. Each is a port its
+ * service declares, of a service the run starts. Under a `runs.secretNetwork` below `open`
+ * a service behind an endpoint may not read a secret or a credential of the server: a page
+ * is a way out that no mask of the log covers.
+ */
+function refuseEndpoints(request, plan, secretNetworkOpen) {
+    for (const endpoint of request.endpoints) {
+        const item = plan[endpoint.service];
+        if (!item || item.kind !== 'service') {
+            return `the endpoint ${endpoint.service}:${endpoint.port} names no service the run starts`;
+        }
+        if (!containerPorts(item).includes(endpoint.port)) {
+            const ports = containerPorts(item);
+            return `${endpoint.service} has no port ${endpoint.port}${ports.length > 0 ? `, its ports are ${ports.join(', ')}` : ', it declares none'}`;
+        }
+        const reads = (0, plan_1.getCredentialReads)(item);
+        if (!secretNetworkOpen && reads.length > 0) {
+            return `${endpoint.service} reads ${reads.join(', ')}, it can not be behind an endpoint on this server (runs.secretNetwork): a page could show it`;
+        }
+    }
+    return null;
+}
+exports.refuseEndpoints = refuseEndpoints;
+function toRunEndpoints(endpoints, hosts, url) {
+    return endpoints.map((endpoint, index) => ({
+        service: endpoint.service,
+        port: endpoint.port,
+        host: hosts[index],
+        url: url(hosts[index]),
+    }));
+}
+exports.toRunEndpoints = toRunEndpoints;
+// The repository of a run's source, which the endpoints grants are written for.
+function getRunRepo(run) {
+    return run.source.type === 'upload' ? null : run.source.repo;
+}
+exports.getRunRepo = getRunRepo;
+// Who may open the endpoints of a service run, and extend it, besides its owner: those
+// with an `open` or `publish` level for its repository.
+function mayOpenEndpoints(grant, run) {
+    const repo = getRunRepo(run);
+    return repo !== null && (0, policy_1.endpointLevel)(grant, repo) !== null;
+}
+exports.mayOpenEndpoints = mayOpenEndpoints;
+//# sourceMappingURL=service-runs.js.map

package/dist/utils/abort-on-signals.js

@@ -1,17 +1,57 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.abortOnSignals = exports.ABORT_SIGNALS = void 0;
+exports.takeOverSignal = exports.abortOnSignals = exports.ABORT_SIGNALS = void 0;
 // Ctrl-C (SIGINT), a CI job being cancelled or a supervisor stopping hammerkit
 // (SIGTERM) and a closed terminal (SIGHUP) all stop the run the same way: abort,
 // so services are removed and local tasks' process groups are killed. Without a
 // handler Node exits on SIGTERM/SIGHUP at once and leaves both behind.
 exports.ABORT_SIGNALS = ['SIGINT', 'SIGTERM', 'SIGHUP'];
-function abortOnSignals(source, abortCtrl) {
-    for (const signal of exports.ABORT_SIGNALS) {
-        source.on(signal, () => {
-            abortCtrl.abort();
-        });
+// signals a running command gives a meaning of its own (the server reloads its
+// entitlements on SIGHUP), per source and counted by how many hold them: they don't
+// abort the run
+const takenOver = new WeakMap();
+function heldOn(source) {
+    let held = takenOver.get(source);
+    if (!held) {
+        held = new Map();
+        takenOver.set(source, held);
     }
+    return held;
+}
+/**
+ * Stops the run on the abort signals, all but those a command took over with
+ * `takeOverSignal`. Returns what removes the listeners again.
+ */
+function abortOnSignals(source, abortCtrl) {
+    const listeners = exports.ABORT_SIGNALS.map((signal) => {
+        const listener = () => {
+            if (!takenOver.get(source)?.get(signal)) {
+                abortCtrl.abort();
+            }
+        };
+        source.on(signal, listener);
+        return { signal, listener };
+    });
+    return () => listeners.forEach(({ signal, listener }) => source.off(signal, listener));
 }
 exports.abortOnSignals = abortOnSignals;
+/**
+ * Calls `handler` on `signal` instead of stopping the run, until the returned function
+ * gives the signal back.
+ */
+function takeOverSignal(source, signal, handler) {
+    const held = heldOn(source);
+    held.set(signal, (held.get(signal) ?? 0) + 1);
+    source.on(signal, handler);
+    let released = false;
+    return () => {
+        if (released) {
+            return;
+        }
+        released = true;
+        source.off(signal, handler);
+        held.set(signal, (held.get(signal) ?? 1) - 1);
+    };
+}
+exports.takeOverSignal = takeOverSignal;
 //# sourceMappingURL=abort-on-signals.js.map