hammerkit 1.23.0 → 1.25.0

package/dist/server/server.js

@@ -0,0 +1,434 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startServer = exports.getDiscoveryDocument = exports.SERVER_PROTOCOL = exports.DISCOVERY_PATH = void 0;
+const http_1 = require("http");
+const https_1 = require("https");
+const version_1 = require("../version");
+const server_config_1 = require("./server-config");
+const discovery_1 = require("./discovery");
+const oauth_1 = require("./oauth");
+const policy_1 = require("./policy");
+const run_store_1 = require("./run-store");
+const runs_1 = require("./runs");
+const git_checkout_1 = require("./git-checkout");
+const repo_access_1 = require("./repo-access");
+const executor_1 = require("./executor");
+const run_plan_1 = require("./run-plan");
+const run_environment_1 = require("./run-environment");
+const accounts_api_1 = require("./accounts-api");
+const run_request_1 = require("./run-request");
+const http_2 = require("./http");
+const definitions_1 = require("./definitions");
+const definitions_api_1 = require("./definitions-api");
+const approvals_api_1 = require("./approvals-api");
+const proposals_api_1 = require("./proposals-api");
+const proposals_1 = require("./proposals");
+const sources_1 = require("./sources");
+const sources_api_1 = require("./sources-api");
+const units_1 = require("../utils/units");
+const get_cache_directory_1 = require("../optimizer/get-cache-directory");
+const path_1 = require("path");
+var discovery_2 = require("./discovery");
+Object.defineProperty(exports, "DISCOVERY_PATH", { enumerable: true, get: function () { return discovery_2.DISCOVERY_PATH; } });
+Object.defineProperty(exports, "SERVER_PROTOCOL", { enumerable: true, get: function () { return discovery_2.SERVER_PROTOCOL; } });
+function getDiscoveryDocument(config) {
+    return {
+        name: 'hammerkit',
+        version: (0, version_1.getVersion)(),
+        protocol: discovery_1.SERVER_PROTOCOL,
+        issuers: config.issuers.map((i) => ({
+            issuer: i.issuer,
+            audience: i.audience,
+            ...(i.clientId ? { clientId: i.clientId } : {}),
+            scope: i.scope,
+        })),
+        backend: { type: config.backend.type, platform: config.backend.platform ?? (0, server_config_1.detectPlatform)() },
+    };
+}
+exports.getDiscoveryDocument = getDiscoveryDocument;
+const HEARTBEAT_MS = 15000;
+// Server-sent events: `id` is the sequence number, so a client that lost the
+// connection continues with Last-Event-ID (or ?after=) and misses nothing.
+function streamEvents(registry, config, principal, id, req, res, url) {
+    const after = Number(req.headers['last-event-id'] ?? url.searchParams.get('after') ?? 0);
+    const write = (event) => res.write(`id: ${event.seq}\nevent: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);
+    const watch = registry.watch(principal, id, Number.isFinite(after) && after > 0 ? after : 0, (event) => {
+        write(event);
+        if (event.type === 'state' && (0, runs_1.isFinal)(event.state)) {
+            end();
+        }
+    }, accountsOf(config, principal));
+    if (!watch) {
+        (0, http_2.send)(res, 404, { error: 'not found' });
+        return;
+    }
+    const heartbeat = setInterval(() => res.write(': keep-alive\n\n'), HEARTBEAT_MS);
+    const end = () => {
+        clearInterval(heartbeat);
+        watch.stop();
+        res.end();
+    };
+    res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-store', connection: 'keep-alive' });
+    watch.events.forEach(write);
+    if (watch.finished) {
+        end();
+        return;
+    }
+    res.on('close', () => {
+        clearInterval(heartbeat);
+        watch.stop();
+    });
+}
+async function handleRuns(config, registry, definitions, sources, planning, repos, log, principal, req, res, url) {
+    const match = /^\/v1\/runs(?:\/([^/]+)(?:\/(cancel|events))?)?$/.exec(url.pathname);
+    if (!match) {
+        return false;
+    }
+    const [, id, action] = match;
+    const method = req.method;
+    if (!id) {
+        if (method === 'GET') {
+            (0, http_2.send)(res, 200, { runs: registry.list(principal).map(toRunResponse) });
+        }
+        else if (method === 'POST') {
+            await submitRun(config, registry, definitions, sources, planning, repos, log, principal, req, res);
+        }
+        else {
+            (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET, POST' });
+        }
+        return true;
+    }
+    if (!runs_1.RUN_ID_PATTERN.test(id)) {
+        (0, http_2.send)(res, 404, { error: 'not found' });
+    }
+    else if (action === 'cancel') {
+        if (method !== 'POST') {
+            (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'POST' });
+            return true;
+        }
+        const run = registry.cancel(principal, id, accountsOf(config, principal));
+        log(`cancel iss=${principal.iss} sub=${principal.sub} run=${id} found=${Boolean(run)}`);
+        run ? (0, http_2.send)(res, 200, toRunResponse(run)) : (0, http_2.send)(res, 404, { error: 'not found' });
+    }
+    else if (method !== 'GET') {
+        (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET' });
+    }
+    else if (action === 'events') {
+        streamEvents(registry, config, principal, id, req, res, url);
+    }
+    else {
+        const run = registry.get(principal, id, accountsOf(config, principal));
+        run ? (0, http_2.send)(res, 200, toRunResponse(run)) : (0, http_2.send)(res, 404, { error: 'not found' });
+    }
+    return true;
+}
+// The accounts the caller uses: who approves a run on them follows and stops it.
+function accountsOf(config, principal) {
+    const grant = (0, policy_1.getGrant)(config.entitlements, principal);
+    return (account) => (0, policy_1.canUse)(grant, account);
+}
+function toRunResponse(run) {
+    const { owner, ...visible } = run;
+    void owner;
+    return visible;
+}
+async function submitRun(config, registry, definitions, sources, planning, repos, log, principal, req, res) {
+    const parsed = run_request_1.runRequestSchema.safeParse(await (0, http_2.readJson)(req));
+    if (!parsed.success) {
+        const issue = parsed.error.issues[0];
+        (0, http_2.send)(res, 400, { error: `${issue.path.join('.') || 'request'}: ${issue.message}` });
+        return;
+    }
+    const request = parsed.data;
+    const grant = (0, policy_1.getGrant)(config.entitlements, principal);
+    // what the caller may not do is said plainly: they already know who they are
+    if (!(0, policy_1.mayRun)(grant)) {
+        log(`refuse run iss=${principal.iss} sub=${principal.sub}: no entitlement`);
+        (0, http_2.send)(res, 403, { error: 'you are not entitled to run anything on this server' });
+        return;
+    }
+    const version = definitions.find(request.definition.name, request.definition.hash);
+    if (!version) {
+        (0, http_2.send)(res, 400, { error: `there is no build file ${request.definition.name} at ${request.definition.hash}` });
+        return;
+    }
+    if (version.yanked) {
+        (0, http_2.send)(res, 409, {
+            error: `${version.name} at ${version.hash} was yanked${version.yanked.reason ? `: ${version.yanked.reason}` : ''}`,
+        });
+        return;
+    }
+    const known = [...version.tasks, ...version.services];
+    const unknown = request.tasks.filter((task) => !known.includes(task));
+    if (unknown.length > 0) {
+        (0, http_2.send)(res, 400, { error: `${version.name} has no task ${unknown.map(oauth_1.quote).join(', ')}` });
+        return;
+    }
+    if (request.source.type === 'upload') {
+        // a run starts from files the server has all of, never from a half-sent tree
+        const tree = await sources.get(principal, request.source.snapshot);
+        if (!tree) {
+            (0, http_2.send)(res, 400, {
+                error: `the files ${request.source.snapshot} are not known, announce them with POST /v1/sources`,
+            });
+            return;
+        }
+        if (tree.missing.length > 0) {
+            (0, http_2.send)(res, 400, {
+                error: `the files ${tree.snapshot} are not complete, ${tree.missing.length} of them still have to be sent`,
+            });
+            return;
+        }
+    }
+    // the accounts of a run are read from the files it is pinned to, with the engine
+    const bundle = version.plan ? await definitions.files(version) : undefined;
+    if (!version.plan || !bundle) {
+        (0, http_2.send)(res, 409, {
+            error: `${version.name} at ${version.hash} was published before accounts were recorded, publish it again`,
+        });
+        return;
+    }
+    let planned;
+    try {
+        planned = await (0, run_plan_1.planRun)(bundle, request, planning.environment, planning.policy);
+    }
+    catch (e) {
+        (0, http_2.send)(res, 400, { error: `the run does not load: ${e instanceof Error ? e.message : String(e)}` });
+        return;
+    }
+    const decision = (0, policy_1.decideRun)({
+        grant,
+        accounts: planned.accounts,
+        catalog: config.accounts,
+        source: request.source.type === 'ref'
+            ? { type: 'ref', repo: request.source.repo, ref: request.source.ref }
+            : { type: 'upload' },
+        approvable: (account) => config.entitlements.some((entitlement) => entitlement.accounts[account] === 'use'),
+        readable: (repo) => repos.readable(repo),
+    });
+    if (decision.type === 'refuse') {
+        log(`refuse run iss=${principal.iss} sub=${principal.sub}: ${decision.error}`);
+        (0, http_2.send)(res, decision.status, { error: decision.error });
+        return;
+    }
+    const approvals = decision.type === 'approval' ? decision.accounts : [];
+    // the ref is pinned now: the approver sees the commit, and the run executes that one
+    let source;
+    if (request.source.type === 'ref') {
+        try {
+            const commit = await repos.resolve(request.source.repo, request.source.ref);
+            source = { type: 'ref', repo: request.source.repo, ref: request.source.ref, commit };
+        }
+        catch (e) {
+            const message = e instanceof Error ? e.message : String(e);
+            log(`refuse run iss=${principal.iss} sub=${principal.sub}: ${message}`);
+            (0, http_2.send)(res, e instanceof git_checkout_1.RefError ? 400 : 502, {
+                error: e instanceof git_checkout_1.RefError ? message : `could not resolve ${request.source.ref}: ${message}`,
+            });
+            return;
+        }
+    }
+    else {
+        source = request.source;
+    }
+    try {
+        const run = registry.submit(principal, {
+            definition: { name: version.name, hash: version.hash },
+            tasks: request.tasks,
+            accounts: planned.accounts,
+            plan: planned.plan,
+            source,
+            options: request.options,
+        }, approvals);
+        log(`run ${run.id} iss=${principal.iss} sub=${principal.sub} accounts=[${planned.accounts.join(',')}] source=${run.source.type} tasks=${run.tasks.length}${approvals.length > 0 ? ` approval=[${approvals.join(',')}]` : ''}`);
+        (0, http_2.send)(res, 202, toRunResponse(run));
+    }
+    catch (e) {
+        if (e instanceof runs_1.RunError) {
+            (0, http_2.send)(res, e.status, { error: e.message });
+            return;
+        }
+        throw e;
+    }
+}
+function bearerToken(req) {
+    const match = /^Bearer ([A-Za-z0-9._~+/-]+=*)$/.exec(req.headers.authorization ?? '');
+    return match ? match[1] : null;
+}
+// Callers are known by their validated token alone. The reason a token was
+// refused goes to the log, the caller only learns that it was.
+async function authenticate(verifier, log, req, res) {
+    const token = bearerToken(req);
+    try {
+        if (!token) {
+            throw new oauth_1.TokenError('no bearer token');
+        }
+        return await verifier.verify(token);
+    }
+    catch (e) {
+        log(`deny ${req.method} ${req.url}: ${e instanceof oauth_1.TokenError ? e.message : 'token could not be checked'}`);
+        (0, http_2.send)(res, 401, { error: 'invalid token' }, { 'www-authenticate': 'Bearer error="invalid_token"' });
+        return null;
+    }
+}
+async function handle(config, registry, definitions, proposals, sources, planning, repos, verifier, log, req, res) {
+    const url = new URL(req.url ?? '/', 'http://localhost');
+    const path = url.pathname;
+    if (path === discovery_1.DISCOVERY_PATH || path === '/healthz') {
+        if (req.method !== 'GET' && req.method !== 'HEAD') {
+            (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET, HEAD' });
+            return;
+        }
+        (0, http_2.send)(res, 200, path === '/healthz' ? { status: 'ok' } : getDiscoveryDocument(config));
+        return;
+    }
+    const principal = await authenticate(verifier, log, req, res);
+    if (!principal) {
+        return;
+    }
+    if (await (0, accounts_api_1.handleAccounts)(config, log, principal, req, res, url)) {
+        return;
+    }
+    if (await (0, definitions_api_1.handleDefinitions)(config, definitions, proposals, log, principal, req, res, url)) {
+        return;
+    }
+    if (await (0, sources_api_1.handleSources)(config, sources, log, principal, req, res, url)) {
+        return;
+    }
+    if (await handleRuns(config, registry, definitions, sources, planning, repos, log, principal, req, res, url)) {
+        return;
+    }
+    if (await (0, approvals_api_1.handleApprovals)(config, registry, log, principal, req, res, url)) {
+        return;
+    }
+    if (await (0, proposals_api_1.handleProposals)(config, definitions, proposals, log, principal, req, res, url)) {
+        return;
+    }
+    (0, http_2.send)(res, 404, { error: 'not found' });
+}
+async function startServer(config, environment, options = {}) {
+    // tasks find their tools through PATH, a server that hides it would fail every run
+    if (!config.runs.env.allow.some((name) => name.toUpperCase() === 'PATH') && !('PATH' in config.runs.env.set)) {
+        throw new Error('runs.env.allow has no PATH, tasks could not find their tools: add PATH (and on Windows SystemRoot, ComSpec, PATHEXT and TEMP)');
+    }
+    const verifier = new oauth_1.TokenVerifier(config.issuers, options);
+    const write = options.log ?? (() => undefined);
+    // request urls and subjects come from callers: no control characters, so no forged log lines
+    const log = (line) => write(line.replace(/\p{Cc}/gu, '?').slice(0, 1000));
+    if (config.runtimes.includes('local')) {
+        log('warning: the local runtime is allowed, local tasks run as the user of this server and read all it can read; ' +
+            'an account is then an approval class, not a credential boundary');
+    }
+    // build files are read, planned and run as the server's runs see the world: the
+    // operator's accounts and the allowlisted environment, nothing of the server's own
+    const planning = {
+        environment: {
+            ...environment,
+            processEnvs: (0, run_environment_1.runEnvironment)(config, environment.processEnvs),
+            hosted: { catalog: (0, server_config_1.getSecretCatalog)(config) },
+        },
+        policy: { envs: config.runs.env.allow, reserved: (0, server_config_1.getSourceAccounts)(config), runtimes: config.runtimes },
+    };
+    const storage = config.storage.directory ?? (0, path_1.join)((0, get_cache_directory_1.getHammerkitDirectory)(), 'server');
+    const definitions = new definitions_1.DefinitionStore((0, path_1.join)(storage, 'definitions'), planning.environment, {
+        policy: planning.policy,
+    });
+    await definitions.load();
+    const repos = new repo_access_1.RepoAccess(config, planning.environment, options.repos);
+    const executor = options.executor ??
+        new executor_1.HostExecutor({
+            definitions,
+            planning,
+            repos,
+            directory: (0, path_1.join)(storage, 'runs'),
+            cache: (0, path_1.join)(storage, 'cache'),
+            worker: options.worker,
+        });
+    const sources = new sources_1.SourceStore((0, path_1.join)(storage, 'sources'));
+    const registry = new runs_1.RunRegistry({
+        executor,
+        maxConcurrent: config.runs.maxConcurrent,
+        keep: config.runs.keep,
+        maxWaiting: config.approvals.maxWaiting,
+        approvalTimeout: (0, units_1.parseDuration)(config.approvals.timeout),
+        limits: config.runs.limits,
+        maxDuration: (0, units_1.parseDuration)(config.runs.maxDuration),
+        maxLogBytes: config.runs.maxLogBytes,
+        store: new run_store_1.FileRunStore((0, path_1.join)(storage, 'records')),
+        log,
+    });
+    await registry.restore();
+    const proposals = new proposals_1.ProposalRegistry({
+        maxWaiting: config.approvals.maxWaiting,
+        timeout: (0, units_1.parseDuration)(config.approvals.timeout),
+        cooldown: (0, units_1.parseDuration)(config.approvals.cooldown),
+    });
+    const listener = (req, res) => {
+        handle(config, registry, definitions, proposals, sources, planning, repos, verifier, log, req, res).catch((e) => {
+            if (e instanceof http_2.HttpError) {
+                (0, http_2.send)(res, e.status, { error: e.message });
+                return;
+            }
+            log(`error ${req.method} ${req.url}: ${e instanceof Error ? e.message : String(e)}`);
+            if (!res.headersSent) {
+                (0, http_2.send)(res, 500, { error: 'internal error' });
+            }
+            else {
+                res.end();
+            }
+        });
+    };
+    let server;
+    if (config.tls) {
+        const [cert, key] = await Promise.all([
+            environment.file.read(config.tls.cert),
+            environment.file.read(config.tls.key),
+        ]);
+        server = (0, https_1.createServer)({ cert, key }, listener);
+    }
+    else {
+        server = (0, http_1.createServer)(listener);
+    }
+    await new Promise((resolve, reject) => {
+        server.once('error', reject);
+        server.listen(config.listen.port, config.listen.host, () => {
+            server.off('error', reject);
+            resolve();
+        });
+    });
+    const address = server.address();
+    const host = address.family === 'IPv6' ? `[${address.address}]` : address.address;
+    return {
+        url: `${config.tls ? 'https' : 'http'}://${host}:${address.port}`,
+        reload: (next) => {
+            const readers = (0, server_config_1.getSourceAccounts)(config);
+            next.entitlements.forEach((entitlement, index) => {
+                const iss = (entitlement.subject ?? entitlement.group ?? entitlement.claims)?.iss;
+                if (iss && !config.issuers.some((issuer) => issuer.issuer === iss)) {
+                    throw new Error(`entitlements.${index}: issuer ${iss} is not accepted by the running server`);
+                }
+                for (const account of Object.keys(entitlement.accounts)) {
+                    if (!config.accounts[account] || readers.includes(account)) {
+                        throw new Error(`entitlements.${index}: the running server has no account ${account} to give`);
+                    }
+                }
+            });
+            const restart = Object.keys(next).filter((key) => key !== 'entitlements' &&
+                JSON.stringify(next[key]) !== JSON.stringify(config[key]));
+            // every request reads the entitlements of this object
+            config.entitlements = next.entitlements;
+            log(`reloaded ${next.entitlements.length} entitlements${restart.length > 0 ? `, restart for: ${restart.join(', ')}` : ''}`);
+            return { restart };
+        },
+        close: async () => {
+            proposals.close();
+            await registry.close();
+            await new Promise((resolve, reject) => {
+                server.close((err) => (err ? reject(err) : resolve()));
+                server.closeAllConnections();
+            });
+        },
+    };
+}
+exports.startServer = startServer;
+//# sourceMappingURL=server.js.map

package/dist/server/sources-api.js

@@ -0,0 +1,106 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleSources = void 0;
+const bundle_1 = require("../remote/bundle");
+const source_manifest_1 = require("../remote/source-manifest");
+const http_1 = require("./http");
+const policy_1 = require("./policy");
+const sources_1 = require("./sources");
+/**
+ * The project files of a caller, in two steps so that only what the server
+ * lacks is sent:
+ *
+ *   POST   /v1/sources                  announce a tree (a manifest of paths and digests);
+ *                                       answers its name and the digests still missing
+ *   PUT    /v1/sources/blobs/<sha256>   send the content of one file
+ *   GET    /v1/sources                  the caller's trees and what they take
+ *   GET    /v1/sources/<hash>           one tree and what is still missing
+ *   DELETE /v1/sources/<hash>           forget a tree and the content nothing else needs
+ *
+ * Everything is the caller's own: another caller's tree is `404`, and content is
+ * never shared between callers.
+ */
+async function handleSources(config, store, log, principal, req, res, url) {
+    const match = /^\/v1\/sources(?:\/(blobs)\/([^/]+)|\/([^/]+))?$/.exec(url.pathname);
+    if (!match) {
+        return false;
+    }
+    const [, blobs, digest, rawSnapshot] = match;
+    const method = req.method;
+    try {
+        if (blobs) {
+            if (method !== 'PUT') {
+                (0, http_1.send)(res, 405, { error: 'method not allowed' }, { allow: 'PUT' });
+                return true;
+            }
+            mustMayUpload(config, principal);
+            const result = await store.put(principal, digest, req);
+            (0, http_1.send)(res, result === 'created' ? 201 : 200, { stored: digest });
+            return true;
+        }
+        if (rawSnapshot === undefined) {
+            if (method === 'GET') {
+                (0, http_1.send)(res, 200, await store.list(principal));
+            }
+            else if (method === 'POST') {
+                mustMayUpload(config, principal);
+                const parsed = source_manifest_1.sourceManifestSchema.safeParse(await (0, http_1.readJson)(req, source_manifest_1.MAX_MANIFEST_BODY_BYTES));
+                if (!parsed.success) {
+                    const issue = parsed.error.issues[0];
+                    (0, http_1.send)(res, 400, { error: `${issue.path.join('.') || 'request'}: ${issue.message}` });
+                    return true;
+                }
+                const { status, created } = await store.announce(principal, parsed.data);
+                log(`source ${status.snapshot} iss=${principal.iss} sub=${principal.sub} files=${status.files} missing=${status.missing.length}`);
+                (0, http_1.send)(res, created ? 201 : 200, { ...status, complete: status.missing.length === 0 });
+            }
+            else {
+                (0, http_1.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET, POST' });
+            }
+            return true;
+        }
+        const snapshot = decodeSnapshot(rawSnapshot);
+        if (method === 'GET') {
+            const status = await store.get(principal, snapshot);
+            status ? (0, http_1.send)(res, 200, { ...status, complete: status.missing.length === 0 }) : notFound(res);
+        }
+        else if (method === 'DELETE') {
+            const removed = await store.remove(principal, snapshot);
+            log(`source remove ${snapshot} iss=${principal.iss} sub=${principal.sub} found=${removed}`);
+            removed ? (0, http_1.send)(res, 200, { removed: snapshot }) : notFound(res);
+        }
+        else {
+            (0, http_1.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET, DELETE' });
+        }
+        return true;
+    }
+    catch (e) {
+        if (e instanceof sources_1.SourceError) {
+            (0, http_1.send)(res, e.status, { error: e.message });
+            return true;
+        }
+        throw e;
+    }
+}
+exports.handleSources = handleSources;
+// The colon of `sha256:<hex>` may arrive escaped.
+function decodeSnapshot(raw) {
+    try {
+        const value = decodeURIComponent(raw);
+        return bundle_1.HASH_PATTERN.test(value) ? value : '';
+    }
+    catch {
+        return '';
+    }
+}
+function notFound(res) {
+    (0, http_1.send)(res, 404, { error: 'not found' });
+}
+// Sending files is only for callers who may run something from an upload.
+function mustMayUpload(config, principal) {
+    const grant = (0, policy_1.getGrant)(config.entitlements, principal);
+    if (!(0, policy_1.mayRun)(grant)) {
+        throw new sources_1.SourceError(403, 'uploads are not allowed for you');
+    }
+}
+//# sourceMappingURL=sources-api.js.map