hammerkit 1.24.0 → 1.25.0

package/dist/server/server.js

@@ -8,8 +8,14 @@ const server_config_1 = require("./server-config");
 const discovery_1 = require("./discovery");
 const oauth_1 = require("./oauth");
 const policy_1 = require("./policy");
+const run_store_1 = require("./run-store");
 const runs_1 = require("./runs");
+const git_checkout_1 = require("./git-checkout");
+const repo_access_1 = require("./repo-access");
 const executor_1 = require("./executor");
+const run_plan_1 = require("./run-plan");
+const run_environment_1 = require("./run-environment");
+const accounts_api_1 = require("./accounts-api");
 const run_request_1 = require("./run-request");
 const http_2 = require("./http");
 const definitions_1 = require("./definitions");
@@ -43,7 +49,7 @@ exports.getDiscoveryDocument = getDiscoveryDocument;
 const HEARTBEAT_MS = 15000;
 // Server-sent events: `id` is the sequence number, so a client that lost the
 // connection continues with Last-Event-ID (or ?after=) and misses nothing.
-function streamEvents(registry, principal, id, req, res, url) {
+function streamEvents(registry, config, principal, id, req, res, url) {
     const after = Number(req.headers['last-event-id'] ?? url.searchParams.get('after') ?? 0);
     const write = (event) => res.write(`id: ${event.seq}\nevent: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);
     const watch = registry.watch(principal, id, Number.isFinite(after) && after > 0 ? after : 0, (event) => {
@@ -51,7 +57,7 @@ function streamEvents(registry, principal, id, req, res, url) {
         if (event.type === 'state' && (0, runs_1.isFinal)(event.state)) {
             end();
         }
-    });
+    }, accountsOf(config, principal));
     if (!watch) {
         (0, http_2.send)(res, 404, { error: 'not found' });
         return;
@@ -73,7 +79,7 @@ function streamEvents(registry, principal, id, req, res, url) {
         watch.stop();
     });
 }
-async function handleRuns(config, registry, definitions, sources, log, principal, req, res, url) {
+async function handleRuns(config, registry, definitions, sources, planning, repos, log, principal, req, res, url) {
     const match = /^\/v1\/runs(?:\/([^/]+)(?:\/(cancel|events))?)?$/.exec(url.pathname);
     if (!match) {
         return false;
@@ -85,7 +91,7 @@ async function handleRuns(config, registry, definitions, sources, log, principal
             (0, http_2.send)(res, 200, { runs: registry.list(principal).map(toRunResponse) });
         }
         else if (method === 'POST') {
-            await submitRun(config, registry, definitions, sources, log, principal, req, res);
+            await submitRun(config, registry, definitions, sources, planning, repos, log, principal, req, res);
         }
         else {
             (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET, POST' });
@@ -100,7 +106,7 @@ async function handleRuns(config, registry, definitions, sources, log, principal
             (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'POST' });
             return true;
         }
-        const run = registry.cancel(principal, id);
+        const run = registry.cancel(principal, id, accountsOf(config, principal));
         log(`cancel iss=${principal.iss} sub=${principal.sub} run=${id} found=${Boolean(run)}`);
         run ? (0, http_2.send)(res, 200, toRunResponse(run)) : (0, http_2.send)(res, 404, { error: 'not found' });
     }
@@ -108,20 +114,25 @@ async function handleRuns(config, registry, definitions, sources, log, principal
         (0, http_2.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET' });
     }
     else if (action === 'events') {
-        streamEvents(registry, principal, id, req, res, url);
+        streamEvents(registry, config, principal, id, req, res, url);
     }
     else {
-        const run = registry.get(principal, id);
+        const run = registry.get(principal, id, accountsOf(config, principal));
         run ? (0, http_2.send)(res, 200, toRunResponse(run)) : (0, http_2.send)(res, 404, { error: 'not found' });
     }
     return true;
 }
+// The accounts the caller uses: who approves a run on them follows and stops it.
+function accountsOf(config, principal) {
+    const grant = (0, policy_1.getGrant)(config.entitlements, principal);
+    return (account) => (0, policy_1.canUse)(grant, account);
+}
 function toRunResponse(run) {
     const { owner, ...visible } = run;
     void owner;
     return visible;
 }
-async function submitRun(config, registry, definitions, sources, log, principal, req, res) {
+async function submitRun(config, registry, definitions, sources, planning, repos, log, principal, req, res) {
     const parsed = run_request_1.runRequestSchema.safeParse(await (0, http_2.readJson)(req));
     if (!parsed.success) {
         const issue = parsed.error.issues[0];
@@ -130,21 +141,27 @@ async function submitRun(config, registry, definitions, sources, log, principal,
     }
     const request = parsed.data;
     const grant = (0, policy_1.getGrant)(config.entitlements, principal);
-    const account = request.account ?? grant.default;
     // what the caller may not do is said plainly: they already know who they are
-    if (account !== null && !grant.accounts.includes(account) && !grant.request.includes(account)) {
-        log(`refuse run iss=${principal.iss} sub=${principal.sub}: service account ${(0, oauth_1.quote)(account)} is not granted`);
-        (0, http_2.send)(res, 403, { error: `you may not use the service account ${account}` });
+    if (!(0, policy_1.mayRun)(grant)) {
+        log(`refuse run iss=${principal.iss} sub=${principal.sub}: no entitlement`);
+        (0, http_2.send)(res, 403, { error: 'you are not entitled to run anything on this server' });
         return;
     }
-    if (!grant.source.includes(request.source.type)) {
-        log(`refuse run iss=${principal.iss} sub=${principal.sub}: source ${request.source.type} is not granted`);
-        (0, http_2.send)(res, 403, { error: `runs from ${request.source.type} are not allowed for you` });
+    const version = definitions.find(request.definition.name, request.definition.hash);
+    if (!version) {
+        (0, http_2.send)(res, 400, { error: `there is no build file ${request.definition.name} at ${request.definition.hash}` });
         return;
     }
-    if (!(0, policy_1.maySeeDefinitions)(grant)) {
-        log(`refuse run iss=${principal.iss} sub=${principal.sub}: no entitlement`);
-        (0, http_2.send)(res, 403, { error: 'you are not entitled to run anything on this server' });
+    if (version.yanked) {
+        (0, http_2.send)(res, 409, {
+            error: `${version.name} at ${version.hash} was yanked${version.yanked.reason ? `: ${version.yanked.reason}` : ''}`,
+        });
+        return;
+    }
+    const known = [...version.tasks, ...version.services];
+    const unknown = request.tasks.filter((task) => !known.includes(task));
+    if (unknown.length > 0) {
+        (0, http_2.send)(res, 400, { error: `${version.name} has no task ${unknown.map(oauth_1.quote).join(', ')}` });
         return;
     }
     if (request.source.type === 'upload') {
@@ -163,33 +180,72 @@ async function submitRun(config, registry, definitions, sources, log, principal,
             return;
         }
     }
-    const version = definitions.find(request.definition.name, request.definition.hash);
-    if (!version) {
-        (0, http_2.send)(res, 400, { error: `there is no build file ${request.definition.name} at ${request.definition.hash}` });
+    // the accounts of a run are read from the files it is pinned to, with the engine
+    const bundle = version.plan ? await definitions.files(version) : undefined;
+    if (!version.plan || !bundle) {
+        (0, http_2.send)(res, 409, {
+            error: `${version.name} at ${version.hash} was published before accounts were recorded, publish it again`,
+        });
         return;
     }
-    const known = [...version.tasks, ...version.services];
-    const unknown = request.tasks.filter((task) => !known.includes(task));
-    if (unknown.length > 0) {
-        (0, http_2.send)(res, 400, { error: `${version.name} has no task ${unknown.map(oauth_1.quote).join(', ')}` });
+    let planned;
+    try {
+        planned = await (0, run_plan_1.planRun)(bundle, request, planning.environment, planning.policy);
+    }
+    catch (e) {
+        (0, http_2.send)(res, 400, { error: `the run does not load: ${e instanceof Error ? e.message : String(e)}` });
+        return;
+    }
+    const decision = (0, policy_1.decideRun)({
+        grant,
+        accounts: planned.accounts,
+        catalog: config.accounts,
+        source: request.source.type === 'ref'
+            ? { type: 'ref', repo: request.source.repo, ref: request.source.ref }
+            : { type: 'upload' },
+        approvable: (account) => config.entitlements.some((entitlement) => entitlement.accounts[account] === 'use'),
+        readable: (repo) => repos.readable(repo),
+    });
+    if (decision.type === 'refuse') {
+        log(`refuse run iss=${principal.iss} sub=${principal.sub}: ${decision.error}`);
+        (0, http_2.send)(res, decision.status, { error: decision.error });
         return;
     }
-    // asking for an account is not using it: an admin of it has to approve
-    const needsApproval = account !== null && !grant.accounts.includes(account);
+    const approvals = decision.type === 'approval' ? decision.accounts : [];
+    // the ref is pinned now: the approver sees the commit, and the run executes that one
+    let source;
+    if (request.source.type === 'ref') {
+        try {
+            const commit = await repos.resolve(request.source.repo, request.source.ref);
+            source = { type: 'ref', repo: request.source.repo, ref: request.source.ref, commit };
+        }
+        catch (e) {
+            const message = e instanceof Error ? e.message : String(e);
+            log(`refuse run iss=${principal.iss} sub=${principal.sub}: ${message}`);
+            (0, http_2.send)(res, e instanceof git_checkout_1.RefError ? 400 : 502, {
+                error: e instanceof git_checkout_1.RefError ? message : `could not resolve ${request.source.ref}: ${message}`,
+            });
+            return;
+        }
+    }
+    else {
+        source = request.source;
+    }
     try {
         const run = registry.submit(principal, {
             definition: { name: version.name, hash: version.hash },
             tasks: request.tasks,
-            account,
-            source: request.source,
+            accounts: planned.accounts,
+            plan: planned.plan,
+            source,
             options: request.options,
-        }, needsApproval);
-        log(`run ${run.id} iss=${principal.iss} sub=${principal.sub} account=${account ?? '-'} source=${run.source.type} tasks=${run.tasks.length}${needsApproval ? ' approval=required' : ''}`);
+        }, approvals);
+        log(`run ${run.id} iss=${principal.iss} sub=${principal.sub} accounts=[${planned.accounts.join(',')}] source=${run.source.type} tasks=${run.tasks.length}${approvals.length > 0 ? ` approval=[${approvals.join(',')}]` : ''}`);
         (0, http_2.send)(res, 202, toRunResponse(run));
     }
     catch (e) {
         if (e instanceof runs_1.RunError) {
-            (0, http_2.send)(res, 503, { error: e.message });
+            (0, http_2.send)(res, e.status, { error: e.message });
             return;
         }
         throw e;
@@ -215,7 +271,7 @@ async function authenticate(verifier, log, req, res) {
         return null;
     }
 }
-async function handle(config, registry, definitions, proposals, sources, verifier, log, req, res) {
+async function handle(config, registry, definitions, proposals, sources, planning, repos, verifier, log, req, res) {
     const url = new URL(req.url ?? '/', 'http://localhost');
     const path = url.pathname;
     if (path === discovery_1.DISCOVERY_PATH || path === '/healthz') {
@@ -230,18 +286,7 @@ async function handle(config, registry, definitions, proposals, sources, verifie
     if (!principal) {
         return;
     }
-    if (path === '/v1/me' && req.method === 'GET') {
-        const grant = (0, policy_1.getGrant)(config.entitlements, principal);
-        log(`grant iss=${principal.iss} sub=${principal.sub} accounts=[${grant.accounts.join(',')}] entitlements=[${grant.matched.join(',')}]`);
-        (0, http_2.send)(res, 200, {
-            iss: principal.iss,
-            sub: principal.sub,
-            accounts: grant.accounts,
-            default: grant.default,
-            request: grant.request,
-            publish: grant.publish,
-            propose: grant.propose,
-        });
+    if (await (0, accounts_api_1.handleAccounts)(config, log, principal, req, res, url)) {
         return;
     }
     if (await (0, definitions_api_1.handleDefinitions)(config, definitions, proposals, log, principal, req, res, url)) {
@@ -250,7 +295,7 @@ async function handle(config, registry, definitions, proposals, sources, verifie
     if (await (0, sources_api_1.handleSources)(config, sources, log, principal, req, res, url)) {
         return;
     }
-    if (await handleRuns(config, registry, definitions, sources, log, principal, req, res, url)) {
+    if (await handleRuns(config, registry, definitions, sources, planning, repos, log, principal, req, res, url)) {
         return;
     }
     if (await (0, approvals_api_1.handleApprovals)(config, registry, log, principal, req, res, url)) {
@@ -262,33 +307,64 @@ async function handle(config, registry, definitions, proposals, sources, verifie
     (0, http_2.send)(res, 404, { error: 'not found' });
 }
 async function startServer(config, environment, options = {}) {
+    // tasks find their tools through PATH, a server that hides it would fail every run
+    if (!config.runs.env.allow.some((name) => name.toUpperCase() === 'PATH') && !('PATH' in config.runs.env.set)) {
+        throw new Error('runs.env.allow has no PATH, tasks could not find their tools: add PATH (and on Windows SystemRoot, ComSpec, PATHEXT and TEMP)');
+    }
     const verifier = new oauth_1.TokenVerifier(config.issuers, options);
     const write = options.log ?? (() => undefined);
     // request urls and subjects come from callers: no control characters, so no forged log lines
     const log = (line) => write(line.replace(/\p{Cc}/gu, '?').slice(0, 1000));
-    const definitions = new definitions_1.DefinitionStore((0, path_1.join)(config.storage.directory ?? (0, path_1.join)((0, get_cache_directory_1.getHammerkitDirectory)(), 'server'), 'definitions'), environment);
+    if (config.runtimes.includes('local')) {
+        log('warning: the local runtime is allowed, local tasks run as the user of this server and read all it can read; ' +
+            'an account is then an approval class, not a credential boundary');
+    }
+    // build files are read, planned and run as the server's runs see the world: the
+    // operator's accounts and the allowlisted environment, nothing of the server's own
+    const planning = {
+        environment: {
+            ...environment,
+            processEnvs: (0, run_environment_1.runEnvironment)(config, environment.processEnvs),
+            hosted: { catalog: (0, server_config_1.getSecretCatalog)(config) },
+        },
+        policy: { envs: config.runs.env.allow, reserved: (0, server_config_1.getSourceAccounts)(config), runtimes: config.runtimes },
+    };
+    const storage = config.storage.directory ?? (0, path_1.join)((0, get_cache_directory_1.getHammerkitDirectory)(), 'server');
+    const definitions = new definitions_1.DefinitionStore((0, path_1.join)(storage, 'definitions'), planning.environment, {
+        policy: planning.policy,
+    });
     await definitions.load();
+    const repos = new repo_access_1.RepoAccess(config, planning.environment, options.repos);
     const executor = options.executor ??
         new executor_1.HostExecutor({
-            config,
             definitions,
-            environment,
-            directory: (0, path_1.join)(config.storage.directory ?? (0, path_1.join)((0, get_cache_directory_1.getHammerkitDirectory)(), 'server'), 'runs'),
+            planning,
+            repos,
+            directory: (0, path_1.join)(storage, 'runs'),
+            cache: (0, path_1.join)(storage, 'cache'),
+            worker: options.worker,
         });
-    const sources = new sources_1.SourceStore((0, path_1.join)(config.storage.directory ?? (0, path_1.join)((0, get_cache_directory_1.getHammerkitDirectory)(), 'server'), 'sources'));
+    const sources = new sources_1.SourceStore((0, path_1.join)(storage, 'sources'));
     const registry = new runs_1.RunRegistry({
         executor,
         maxConcurrent: config.runs.maxConcurrent,
         keep: config.runs.keep,
         maxWaiting: config.approvals.maxWaiting,
         approvalTimeout: (0, units_1.parseDuration)(config.approvals.timeout),
+        limits: config.runs.limits,
+        maxDuration: (0, units_1.parseDuration)(config.runs.maxDuration),
+        maxLogBytes: config.runs.maxLogBytes,
+        store: new run_store_1.FileRunStore((0, path_1.join)(storage, 'records')),
+        log,
     });
+    await registry.restore();
     const proposals = new proposals_1.ProposalRegistry({
         maxWaiting: config.approvals.maxWaiting,
         timeout: (0, units_1.parseDuration)(config.approvals.timeout),
+        cooldown: (0, units_1.parseDuration)(config.approvals.cooldown),
     });
     const listener = (req, res) => {
-        handle(config, registry, definitions, proposals, sources, verifier, log, req, res).catch((e) => {
+        handle(config, registry, definitions, proposals, sources, planning, repos, verifier, log, req, res).catch((e) => {
             if (e instanceof http_2.HttpError) {
                 (0, http_2.send)(res, e.status, { error: e.message });
                 return;
@@ -324,6 +400,26 @@ async function startServer(config, environment, options = {}) {
     const host = address.family === 'IPv6' ? `[${address.address}]` : address.address;
     return {
         url: `${config.tls ? 'https' : 'http'}://${host}:${address.port}`,
+        reload: (next) => {
+            const readers = (0, server_config_1.getSourceAccounts)(config);
+            next.entitlements.forEach((entitlement, index) => {
+                const iss = (entitlement.subject ?? entitlement.group ?? entitlement.claims)?.iss;
+                if (iss && !config.issuers.some((issuer) => issuer.issuer === iss)) {
+                    throw new Error(`entitlements.${index}: issuer ${iss} is not accepted by the running server`);
+                }
+                for (const account of Object.keys(entitlement.accounts)) {
+                    if (!config.accounts[account] || readers.includes(account)) {
+                        throw new Error(`entitlements.${index}: the running server has no account ${account} to give`);
+                    }
+                }
+            });
+            const restart = Object.keys(next).filter((key) => key !== 'entitlements' &&
+                JSON.stringify(next[key]) !== JSON.stringify(config[key]));
+            // every request reads the entitlements of this object
+            config.entitlements = next.entitlements;
+            log(`reloaded ${next.entitlements.length} entitlements${restart.length > 0 ? `, restart for: ${restart.join(', ')}` : ''}`);
+            return { restart };
+        },
         close: async () => {
             proposals.close();
             await registry.close();

package/dist/server/sources-api.js

@@ -99,7 +99,7 @@ function notFound(res) {
 // Sending files is only for callers who may run something from an upload.
 function mustMayUpload(config, principal) {
     const grant = (0, policy_1.getGrant)(config.entitlements, principal);
-    if (!(0, policy_1.maySeeDefinitions)(grant) || !grant.source.includes('upload')) {
+    if (!(0, policy_1.mayRun)(grant)) {
         throw new sources_1.SourceError(403, 'uploads are not allowed for you');
     }
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.24.0",
+  "version": "1.25.0",
   "keywords": [
     "build",
     "build-tool",