hammerkit 1.22.0 → 1.24.0

package/dist/server/server.js

@@ -0,0 +1,338 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startServer = exports.getDiscoveryDocument = exports.SERVER_PROTOCOL = exports.DISCOVERY_PATH = void 0;
+const http_1 = require("http");
+const https_1 = require("https");
+const version_1 = require("../version");
+const server_config_1 = require("./server-config");
+const discovery_1 = require("./discovery");
+const oauth_1 = require("./oauth");
+const policy_1 = require("./policy");
+const runs_1 = require("./runs");
+const executor_1 = require("./executor");
+const run_request_1 = require("./run-request");
+const http_2 = require("./http");
+const definitions_1 = require("./definitions");
+const definitions_api_1 = require("./definitions-api");
+const approvals_api_1 = require("./approvals-api");
+const proposals_api_1 = require("./proposals-api");
+const proposals_1 = require("./proposals");
+const sources_1 = require("./sources");
+const sources_api_1 = require("./sources-api");
+const units_1 = require("../utils/units");
+const get_cache_directory_1 = require("../optimizer/get-cache-directory");
+const path_1 = require("path");
+var discovery_2 = require("./discovery");
+Object.defineProperty(exports, "DISCOVERY_PATH", { enumerable: true, get: function () { return discovery_2.DISCOVERY_PATH; } });
+Object.defineProperty(exports, "SERVER_PROTOCOL", { enumerable: true, get: function () { return discovery_2.SERVER_PROTOCOL; } });
+function getDiscoveryDocument(config) {
+    return {
+        name: 'hammerkit',
+        version: (0, version_1.getVersion)(),
+        protocol: discovery_1.SERVER_PROTOCOL,
+        issuers: config.issuers.map((i) => ({
+            issuer: i.issuer,
+            audience: i.audience,
+            ...(i.clientId ? { clientId: i.clientId } : {}),
+            scope: i.scope,
+        })),
+        backend: { type: config.backend.type, platform: config.backend.platform ?? (0, server_config_1.detectPlatform)() },
+    };
+}
+exports.getDiscoveryDocument = getDiscoveryDocument;
+const HEARTBEAT_MS = 15000;
+// Server-sent events: `id` is the sequence number, so a client that lost the
+// connection continues with Last-Event-ID (or ?after=) and misses nothing.
+function streamEvents(registry, principal, id, req, res, url) {
+    const after = Number(req.headers['last-event-id'] ?? url.searchParams.get('after') ?? 0);
+    const write = (event) => res.write(`id: ${event.seq}\nevent: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);
+    const watch = registry.watch(principal, id, Number.isFinite(after) && after > 0 ? after : 0, (event) => {
+        write(event);
+        if (event.type === 'state' && (0, runs_1.isFinal)(event.state)) {
+            end();
+        }
+    });
+    if (!watch) {
+        (0, http_2.send)(res, 404, { error: 'not found' });
+        return;
+    }
+    const heartbeat = setInterval(() => res.write(': keep-alive\n\n'), HEARTBEAT_MS);
+    const end = () => {
+        clearInterval(heartbeat);
+        watch.stop();
+        res.end();
+    };
+    res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-store', connection: 'keep-alive' });
+    watch.events.forEach(write);
+    if (watch.finished) {
+        end();
+        return;
+    }
+    res.on('close', () => {
+        clearInterval(heartbeat);
+        watch.stop();
+    });
+}
+async function handleRuns(config, registry, definitions, sources, log, principal, req, res, url) {
+    const match = /^\/v1\/runs(?:\/([^/]+)(?:\/(cancel|events))?)?$/.exec(url.pathname);
+    if (!match) {
+        return false;
+    }
+    const [, id, action] = match;
+    const method = req.method;
+    if (!id) {
+        if (method === 'GET') {
+            (0, http_2.send)(res, 200, { runs: registry.list(principal).map(toRunResponse) });
+        }
+        else if (method === 'POST') {
+            await submitRun(config, registry, definitions, sources, log, principal, req, res);
+        }
+        else {
+            (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET, POST' });
+        }
+        return true;
+    }
+    if (!runs_1.RUN_ID_PATTERN.test(id)) {
+        (0, http_2.send)(res, 404, { error: 'not found' });
+    }
+    else if (action === 'cancel') {
+        if (method !== 'POST') {
+            (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'POST' });
+            return true;
+        }
+        const run = registry.cancel(principal, id);
+        log(`cancel iss=${principal.iss} sub=${principal.sub} run=${id} found=${Boolean(run)}`);
+        run ? (0, http_2.send)(res, 200, toRunResponse(run)) : (0, http_2.send)(res, 404, { error: 'not found' });
+    }
+    else if (method !== 'GET') {
+        (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET' });
+    }
+    else if (action === 'events') {
+        streamEvents(registry, principal, id, req, res, url);
+    }
+    else {
+        const run = registry.get(principal, id);
+        run ? (0, http_2.send)(res, 200, toRunResponse(run)) : (0, http_2.send)(res, 404, { error: 'not found' });
+    }
+    return true;
+}
+function toRunResponse(run) {
+    const { owner, ...visible } = run;
+    void owner;
+    return visible;
+}
+async function submitRun(config, registry, definitions, sources, log, principal, req, res) {
+    const parsed = run_request_1.runRequestSchema.safeParse(await (0, http_2.readJson)(req));
+    if (!parsed.success) {
+        const issue = parsed.error.issues[0];
+        (0, http_2.send)(res, 400, { error: `${issue.path.join('.') || 'request'}: ${issue.message}` });
+        return;
+    }
+    const request = parsed.data;
+    const grant = (0, policy_1.getGrant)(config.entitlements, principal);
+    const account = request.account ?? grant.default;
+    // what the caller may not do is said plainly: they already know who they are
+    if (account !== null && !grant.accounts.includes(account) && !grant.request.includes(account)) {
+        log(`refuse run iss=${principal.iss} sub=${principal.sub}: service account ${(0, oauth_1.quote)(account)} is not granted`);
+        (0, http_2.send)(res, 403, { error: `you may not use the service account ${account}` });
+        return;
+    }
+    if (!grant.source.includes(request.source.type)) {
+        log(`refuse run iss=${principal.iss} sub=${principal.sub}: source ${request.source.type} is not granted`);
+        (0, http_2.send)(res, 403, { error: `runs from ${request.source.type} are not allowed for you` });
+        return;
+    }
+    if (!(0, policy_1.maySeeDefinitions)(grant)) {
+        log(`refuse run iss=${principal.iss} sub=${principal.sub}: no entitlement`);
+        (0, http_2.send)(res, 403, { error: 'you are not entitled to run anything on this server' });
+        return;
+    }
+    if (request.source.type === 'upload') {
+        // a run starts from files the server has all of, never from a half-sent tree
+        const tree = await sources.get(principal, request.source.snapshot);
+        if (!tree) {
+            (0, http_2.send)(res, 400, {
+                error: `the files ${request.source.snapshot} are not known, announce them with POST /v1/sources`,
+            });
+            return;
+        }
+        if (tree.missing.length > 0) {
+            (0, http_2.send)(res, 400, {
+                error: `the files ${tree.snapshot} are not complete, ${tree.missing.length} of them still have to be sent`,
+            });
+            return;
+        }
+    }
+    const version = definitions.find(request.definition.name, request.definition.hash);
+    if (!version) {
+        (0, http_2.send)(res, 400, { error: `there is no build file ${request.definition.name} at ${request.definition.hash}` });
+        return;
+    }
+    const known = [...version.tasks, ...version.services];
+    const unknown = request.tasks.filter((task) => !known.includes(task));
+    if (unknown.length > 0) {
+        (0, http_2.send)(res, 400, { error: `${version.name} has no task ${unknown.map(oauth_1.quote).join(', ')}` });
+        return;
+    }
+    // asking for an account is not using it: an admin of it has to approve
+    const needsApproval = account !== null && !grant.accounts.includes(account);
+    try {
+        const run = registry.submit(principal, {
+            definition: { name: version.name, hash: version.hash },
+            tasks: request.tasks,
+            account,
+            source: request.source,
+            options: request.options,
+        }, needsApproval);
+        log(`run ${run.id} iss=${principal.iss} sub=${principal.sub} account=${account ?? '-'} source=${run.source.type} tasks=${run.tasks.length}${needsApproval ? ' approval=required' : ''}`);
+        (0, http_2.send)(res, 202, toRunResponse(run));
+    }
+    catch (e) {
+        if (e instanceof runs_1.RunError) {
+            (0, http_2.send)(res, 503, { error: e.message });
+            return;
+        }
+        throw e;
+    }
+}
+function bearerToken(req) {
+    const match = /^Bearer ([A-Za-z0-9._~+/-]+=*)$/.exec(req.headers.authorization ?? '');
+    return match ? match[1] : null;
+}
+// Callers are known by their validated token alone. The reason a token was
+// refused goes to the log, the caller only learns that it was.
+async function authenticate(verifier, log, req, res) {
+    const token = bearerToken(req);
+    try {
+        if (!token) {
+            throw new oauth_1.TokenError('no bearer token');
+        }
+        return await verifier.verify(token);
+    }
+    catch (e) {
+        log(`deny ${req.method} ${req.url}: ${e instanceof oauth_1.TokenError ? e.message : 'token could not be checked'}`);
+        (0, http_2.send)(res, 401, { error: 'invalid token' }, { 'www-authenticate': 'Bearer error="invalid_token"' });
+        return null;
+    }
+}
+async function handle(config, registry, definitions, proposals, sources, verifier, log, req, res) {
+    const url = new URL(req.url ?? '/', 'http://localhost');
+    const path = url.pathname;
+    if (path === discovery_1.DISCOVERY_PATH || path === '/healthz') {
+        if (req.method !== 'GET' && req.method !== 'HEAD') {
+            (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET, HEAD' });
+            return;
+        }
+        (0, http_2.send)(res, 200, path === '/healthz' ? { status: 'ok' } : getDiscoveryDocument(config));
+        return;
+    }
+    const principal = await authenticate(verifier, log, req, res);
+    if (!principal) {
+        return;
+    }
+    if (path === '/v1/me' && req.method === 'GET') {
+        const grant = (0, policy_1.getGrant)(config.entitlements, principal);
+        log(`grant iss=${principal.iss} sub=${principal.sub} accounts=[${grant.accounts.join(',')}] entitlements=[${grant.matched.join(',')}]`);
+        (0, http_2.send)(res, 200, {
+            iss: principal.iss,
+            sub: principal.sub,
+            accounts: grant.accounts,
+            default: grant.default,
+            request: grant.request,
+            publish: grant.publish,
+            propose: grant.propose,
+        });
+        return;
+    }
+    if (await (0, definitions_api_1.handleDefinitions)(config, definitions, proposals, log, principal, req, res, url)) {
+        return;
+    }
+    if (await (0, sources_api_1.handleSources)(config, sources, log, principal, req, res, url)) {
+        return;
+    }
+    if (await handleRuns(config, registry, definitions, sources, log, principal, req, res, url)) {
+        return;
+    }
+    if (await (0, approvals_api_1.handleApprovals)(config, registry, log, principal, req, res, url)) {
+        return;
+    }
+    if (await (0, proposals_api_1.handleProposals)(config, definitions, proposals, log, principal, req, res, url)) {
+        return;
+    }
+    (0, http_2.send)(res, 404, { error: 'not found' });
+}
+async function startServer(config, environment, options = {}) {
+    const verifier = new oauth_1.TokenVerifier(config.issuers, options);
+    const write = options.log ?? (() => undefined);
+    // request urls and subjects come from callers: no control characters, so no forged log lines
+    const log = (line) => write(line.replace(/\p{Cc}/gu, '?').slice(0, 1000));
+    const definitions = new definitions_1.DefinitionStore((0, path_1.join)(config.storage.directory ?? (0, path_1.join)((0, get_cache_directory_1.getHammerkitDirectory)(), 'server'), 'definitions'), environment);
+    await definitions.load();
+    const executor = options.executor ??
+        new executor_1.HostExecutor({
+            config,
+            definitions,
+            environment,
+            directory: (0, path_1.join)(config.storage.directory ?? (0, path_1.join)((0, get_cache_directory_1.getHammerkitDirectory)(), 'server'), 'runs'),
+        });
+    const sources = new sources_1.SourceStore((0, path_1.join)(config.storage.directory ?? (0, path_1.join)((0, get_cache_directory_1.getHammerkitDirectory)(), 'server'), 'sources'));
+    const registry = new runs_1.RunRegistry({
+        executor,
+        maxConcurrent: config.runs.maxConcurrent,
+        keep: config.runs.keep,
+        maxWaiting: config.approvals.maxWaiting,
+        approvalTimeout: (0, units_1.parseDuration)(config.approvals.timeout),
+    });
+    const proposals = new proposals_1.ProposalRegistry({
+        maxWaiting: config.approvals.maxWaiting,
+        timeout: (0, units_1.parseDuration)(config.approvals.timeout),
+    });
+    const listener = (req, res) => {
+        handle(config, registry, definitions, proposals, sources, verifier, log, req, res).catch((e) => {
+            if (e instanceof http_2.HttpError) {
+                (0, http_2.send)(res, e.status, { error: e.message });
+                return;
+            }
+            log(`error ${req.method} ${req.url}: ${e instanceof Error ? e.message : String(e)}`);
+            if (!res.headersSent) {
+                (0, http_2.send)(res, 500, { error: 'internal error' });
+            }
+            else {
+                res.end();
+            }
+        });
+    };
+    let server;
+    if (config.tls) {
+        const [cert, key] = await Promise.all([
+            environment.file.read(config.tls.cert),
+            environment.file.read(config.tls.key),
+        ]);
+        server = (0, https_1.createServer)({ cert, key }, listener);
+    }
+    else {
+        server = (0, http_1.createServer)(listener);
+    }
+    await new Promise((resolve, reject) => {
+        server.once('error', reject);
+        server.listen(config.listen.port, config.listen.host, () => {
+            server.off('error', reject);
+            resolve();
+        });
+    });
+    const address = server.address();
+    const host = address.family === 'IPv6' ? `[${address.address}]` : address.address;
+    return {
+        url: `${config.tls ? 'https' : 'http'}://${host}:${address.port}`,
+        close: async () => {
+            proposals.close();
+            await registry.close();
+            await new Promise((resolve, reject) => {
+                server.close((err) => (err ? reject(err) : resolve()));
+                server.closeAllConnections();
+            });
+        },
+    };
+}
+exports.startServer = startServer;
+//# sourceMappingURL=server.js.map

package/dist/server/sources-api.js

@@ -0,0 +1,106 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleSources = void 0;
+const bundle_1 = require("../remote/bundle");
+const source_manifest_1 = require("../remote/source-manifest");
+const http_1 = require("./http");
+const policy_1 = require("./policy");
+const sources_1 = require("./sources");
+/**
+ * The project files of a caller, in two steps so that only what the server
+ * lacks is sent:
+ *
+ *   POST   /v1/sources                  announce a tree (a manifest of paths and digests);
+ *                                       answers its name and the digests still missing
+ *   PUT    /v1/sources/blobs/<sha256>   send the content of one file
+ *   GET    /v1/sources                  the caller's trees and what they take
+ *   GET    /v1/sources/<hash>           one tree and what is still missing
+ *   DELETE /v1/sources/<hash>           forget a tree and the content nothing else needs
+ *
+ * Everything is the caller's own: another caller's tree is `404`, and content is
+ * never shared between callers.
+ */
+async function handleSources(config, store, log, principal, req, res, url) {
+    const match = /^\/v1\/sources(?:\/(blobs)\/([^/]+)|\/([^/]+))?$/.exec(url.pathname);
+    if (!match) {
+        return false;
+    }
+    const [, blobs, digest, rawSnapshot] = match;
+    const method = req.method;
+    try {
+        if (blobs) {
+            if (method !== 'PUT') {
+                (0, http_1.send)(res, 405, { error: 'method not allowed' }, { allow: 'PUT' });
+                return true;
+            }
+            mustMayUpload(config, principal);
+            const result = await store.put(principal, digest, req);
+            (0, http_1.send)(res, result === 'created' ? 201 : 200, { stored: digest });
+            return true;
+        }
+        if (rawSnapshot === undefined) {
+            if (method === 'GET') {
+                (0, http_1.send)(res, 200, await store.list(principal));
+            }
+            else if (method === 'POST') {
+                mustMayUpload(config, principal);
+                const parsed = source_manifest_1.sourceManifestSchema.safeParse(await (0, http_1.readJson)(req, source_manifest_1.MAX_MANIFEST_BODY_BYTES));
+                if (!parsed.success) {
+                    const issue = parsed.error.issues[0];
+                    (0, http_1.send)(res, 400, { error: `${issue.path.join('.') || 'request'}: ${issue.message}` });
+                    return true;
+                }
+                const { status, created } = await store.announce(principal, parsed.data);
+                log(`source ${status.snapshot} iss=${principal.iss} sub=${principal.sub} files=${status.files} missing=${status.missing.length}`);
+                (0, http_1.send)(res, created ? 201 : 200, { ...status, complete: status.missing.length === 0 });
+            }
+            else {
+                (0, http_1.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET, POST' });
+            }
+            return true;
+        }
+        const snapshot = decodeSnapshot(rawSnapshot);
+        if (method === 'GET') {
+            const status = await store.get(principal, snapshot);
+            status ? (0, http_1.send)(res, 200, { ...status, complete: status.missing.length === 0 }) : notFound(res);
+        }
+        else if (method === 'DELETE') {
+            const removed = await store.remove(principal, snapshot);
+            log(`source remove ${snapshot} iss=${principal.iss} sub=${principal.sub} found=${removed}`);
+            removed ? (0, http_1.send)(res, 200, { removed: snapshot }) : notFound(res);
+        }
+        else {
+            (0, http_1.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET, DELETE' });
+        }
+        return true;
+    }
+    catch (e) {
+        if (e instanceof sources_1.SourceError) {
+            (0, http_1.send)(res, e.status, { error: e.message });
+            return true;
+        }
+        throw e;
+    }
+}
+exports.handleSources = handleSources;
+// The colon of `sha256:<hex>` may arrive escaped.
+function decodeSnapshot(raw) {
+    try {
+        const value = decodeURIComponent(raw);
+        return bundle_1.HASH_PATTERN.test(value) ? value : '';
+    }
+    catch {
+        return '';
+    }
+}
+function notFound(res) {
+    (0, http_1.send)(res, 404, { error: 'not found' });
+}
+// Sending files is only for callers who may run something from an upload.
+function mustMayUpload(config, principal) {
+    const grant = (0, policy_1.getGrant)(config.entitlements, principal);
+    if (!(0, policy_1.maySeeDefinitions)(grant) || !grant.source.includes('upload')) {
+        throw new sources_1.SourceError(403, 'uploads are not allowed for you');
+    }
+}
+//# sourceMappingURL=sources-api.js.map