hammerkit 1.27.0-next.2 → 1.27.0-next.21

package/dist/server/sandboxes.js

@@ -412,6 +412,14 @@ class SandboxRegistry {
     async state(id, signal) {
         return this.options.storage.state(keyOf(this.existing(id)), signal);
     }
+    // The files of `tree`, which the sandbox's files must be now, written into `into`: what a
+    // check runs on.
+    async exportTree(id, tree, into, signal) {
+        const sandbox = this.existing(id);
+        sandbox.lastUsedAt = this.timestamp();
+        await this.save(sandbox);
+        await this.options.storage.exportTree(keyOf(sandbox), tree, into, signal);
+    }
     // What a run works on, `into` a directory of the run when the storage copies the files;
     // the run is the sandbox's use.
     async attach(id, into) {

package/dist/server/server-command.js

@@ -5,6 +5,7 @@ const commander_1 = require("commander");
 const log_1 = require("../log");
 const server_config_1 = require("./server-config");
 const server_1 = require("./server");
+const abort_on_signals_1 = require("../utils/abort-on-signals");
 // Runs this machine as a hammerkit server until interrupted. Like `remote`, it
 // only needs its own config file, not a build file.
 function registerServerCommand(program, environment) {
@@ -13,31 +14,46 @@ function registerServerCommand(program, environment) {
         .description('serve this machine to remote runs (experimental)')
         .addOption(new commander_1.Option('-c, --config <file>', 'server config file').env('HAMMERKIT_SERVER_CONFIG'))
         .action(async (options) => {
+        // SIGHUP reads the entitlements again, so who may do what changes without a
+        // restart; the rest of the config needs one. The server takes it over from the
+        // abort of the CLI, which stops other commands on it, before it starts: a SIGHUP
+        // while it reads its config and starts waits for it. Windows sends SIGHUP when the
+        // console closes, which stops the server there too.
+        let server;
+        let reloadWhenStarted = false;
+        const reload = async (running) => {
+            try {
+                const result = running.reload(await (0, server_config_1.readServerConfig)(environment, options.config));
+                if (result.restart.length > 0) {
+                    environment.stdout.write(`changed but not reloaded, restart for: ${result.restart.join(', ')}\n`);
+                }
+            }
+            catch (e) {
+                environment.stdout.write(`entitlements were not reloaded: ${(0, log_1.getErrorMessage)(e)}\n`);
+            }
+        };
+        const releaseReload = process.platform !== 'win32'
+            ? (0, abort_on_signals_1.takeOverSignal)(process, 'SIGHUP', () => {
+                if (server) {
+                    void reload(server);
+                }
+                else {
+                    reloadWhenStarted = true;
+                }
+            })
+            : () => undefined;
         try {
             if (!options.config) {
                 throw new Error('pass the server config with --config or HAMMERKIT_SERVER_CONFIG');
             }
             const config = await (0, server_config_1.readServerConfig)(environment, options.config);
-            const server = await (0, server_1.startServer)(config, environment, {
+            const running = await (0, server_1.startServer)(config, environment, {
                 log: (line) => environment.stdout.write(`${line}\n`),
             });
-            environment.stdout.write(`hammerkit server listening on ${server.url}\n`);
-            // SIGHUP reads the entitlements again, so who may do what changes without
-            // a restart; the rest of the config needs one
-            const reload = async () => {
-                try {
-                    const result = server.reload(await (0, server_config_1.readServerConfig)(environment, options.config));
-                    if (result.restart.length > 0) {
-                        environment.stdout.write(`changed but not reloaded, restart for: ${result.restart.join(', ')}\n`);
-                    }
-                }
-                catch (e) {
-                    environment.stdout.write(`entitlements were not reloaded: ${(0, log_1.getErrorMessage)(e)}\n`);
-                }
-            };
-            const onReload = () => void reload();
-            if (process.platform !== 'win32') {
-                process.on('SIGHUP', onReload);
+            server = running;
+            environment.stdout.write(`hammerkit server listening on ${running.url}\n`);
+            if (reloadWhenStarted) {
+                void reload(running);
             }
             await new Promise((resolve) => {
                 if (environment.abortCtrl.signal.aborted) {
@@ -45,8 +61,8 @@ function registerServerCommand(program, environment) {
                 }
                 environment.abortCtrl.signal.addEventListener('abort', () => resolve(), { once: true });
             });
-            process.off('SIGHUP', onReload);
-            await server.close();
+            releaseReload();
+            await running.close();
             environment.stdout.write('hammerkit server stopped\n');
         }
         catch (e) {
@@ -55,6 +71,9 @@ function registerServerCommand(program, environment) {
             }
             program.error(`Server was not successful: ${(0, log_1.getErrorMessage)(e)}`, { exitCode: 1 });
         }
+        finally {
+            releaseReload();
+        }
     });
 }
 exports.registerServerCommand = registerServerCommand;

package/dist/server/server-config.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.detectPlatform = exports.getSecretCatalog = exports.readServerConfig = exports.getSourceAccounts = exports.getSandboxRuntimes = exports.serverConfigSchema = exports.sourceSchema = exports.entitlementSchema = exports.sandboxGrantSchema = exports.sandboxBranchTypeSchema = exports.buildsLevelSchema = exports.accountLevelSchema = exports.getMaxVolumeSize = exports.DEFAULT_MAX_VOLUME_SIZE = exports.DEFAULT_DENIED_EGRESS = exports.kubernetesPoolSchema = exports.accountSchema = exports.issuerSchema = exports.serverPlatformSchema = void 0;
+exports.detectPlatform = exports.getSecretCatalog = exports.readServerConfig = exports.getServerAccounts = exports.getSandboxRuntimes = exports.serverConfigSchema = exports.sourceSchema = exports.entitlementSchema = exports.endpointGrantSchema = exports.endpointLevelSchema = exports.sandboxGrantSchema = exports.sandboxBranchTypeSchema = exports.buildsLevelSchema = exports.accountLevelSchema = exports.getMaxVolumeSize = exports.DEFAULT_MAX_VOLUME_SIZE = exports.DEFAULT_DENIED_EGRESS = exports.kubernetesPoolSchema = exports.accountSchema = exports.issuerSchema = exports.serverPlatformSchema = void 0;
 const net_1 = require("net");
 const zod_1 = require("zod");
 const yaml_1 = require("yaml");
@@ -188,8 +188,12 @@ const kubernetesSandboxesSchema = zod_1.z
     // when left out. With a class of their own, a ResourceQuota on that class in each
     // caller's namespace holds `sandboxes.limits`.
     storageClass: zod_1.z.string().min(1).optional(),
-    // how long the pod of a sandbox's files stays after the last request
+    // how long the pod of a sandbox's files stays after the last request; the runner of the
+    // sandbox goes with it
     idle: duration.default('1m'),
+    // exec the commands of a task shaped like a shell (it mounts the sandbox only, has no
+    // needs, outputs or secrets) into one pod per sandbox, in place of a Job each
+    runner: zod_1.z.boolean().default(true),
     // of the pod of a sandbox's files, which runs git and ripgrep over all of them
     resources: zod_1.z
         .object({ requests: quantitiesSchema.optional(), limits: quantitiesSchema.optional() })
@@ -238,8 +242,22 @@ const kubernetesBackendSchema = zod_1.z
         .min(1)
         .default([{ name: 'default', platform: { os: 'linux', arch: 'amd64' }, nodeSelector: {}, tolerations: [] }]),
     // per secret provider, the image its command runs in: secrets are read in the
-    // Job of the task, not by the server
-    providers: zod_1.z.record(zod_1.z.object({ image: zod_1.z.string().min(1) }).strict()).default({}),
+    // Job of the task, not by the server. `allow` are the hosts its command reaches, such
+    // as secretmanager.googleapis.com: the egress proxy of a task with a limited network
+    // lets the init containers that read its secrets reach them, and nothing else does
+    providers: zod_1.z
+        .record(zod_1.z
+        .object({
+        image: zod_1.z.string().min(1),
+        allow: zod_1.z
+            .array(zod_1.z
+            .string()
+            .refine((host) => (0, network_schema_1.isAllowEntry)(host) && !host.includes(':'), 'expected a host name such as secretmanager.googleapis.com or *.example.com'))
+            .min(1)
+            .optional(),
+    })
+        .strict())
+        .default({}),
     // the ResourceQuota of every caller's namespace, such as `requests.cpu: '8'`
     quota: zod_1.z.record(zod_1.z.union([zod_1.z.string().min(1), zod_1.z.number()]).transform(String)).optional(),
     // what a container without `resources` requests (a LimitRange), so it is
@@ -309,6 +327,11 @@ exports.sandboxGrantSchema = zod_1.z
         .refine((branches) => Object.keys(branches).length > 0, 'name the branches a sandbox may start from'),
 })
     .strict();
+// `open`: open the endpoints of the service runs of a repository, whoever started them, and
+// extend those runs; `publish`: also name endpoints in one's own service runs.
+exports.endpointLevelSchema = zod_1.z.enum(['open', 'publish']);
+// One repository (`host/path` pattern) whose endpoints an entitlement opens or publishes.
+exports.endpointGrantSchema = zod_1.z.object({ repo: repoPatternSchema, level: exports.endpointLevelSchema }).strict();
 exports.entitlementSchema = zod_1.z
     .object({
     ...whoShape,
@@ -318,19 +341,33 @@ exports.entitlementSchema = zod_1.z
     // the repositories (`host/path` patterns) the caller may run from a git ref
     repos: zod_1.z.array(repoPatternSchema).default([]),
     builds: exports.buildsLevelSchema.optional(),
+    // `read`: may look up which tasks runs passed for files of the `repos` (remote verify),
+    // and nothing more; a builds level includes it
+    checks: zod_1.z.literal('read').optional(),
     // the repositories the caller may keep sandboxes of, checkouts on the server that runs
     // work on: a list of its own, the `repos` above are for runs from a ref
     sandboxes: zod_1.z.array(exports.sandboxGrantSchema).optional(),
+    // the repositories whose service runs' endpoints the caller may open, or publish
+    endpoints: zod_1.z.array(exports.endpointGrantSchema).optional(),
 })
     .strict()
     .superRefine((entitlement, ctx) => {
     checkWho(entitlement, ctx);
     if (Object.keys(entitlement.accounts).length === 0 &&
         entitlement.builds === undefined &&
-        !entitlement.sandboxes?.length) {
+        entitlement.checks === undefined &&
+        !entitlement.sandboxes?.length &&
+        !entitlement.endpoints?.length) {
+        ctx.addIssue({
+            code: zod_1.z.ZodIssueCode.custom,
+            message: 'name accounts to use or request, a builds level, checks, sandboxes or endpoints',
+        });
+    }
+    if (entitlement.checks && entitlement.repos.length === 0) {
         ctx.addIssue({
             code: zod_1.z.ZodIssueCode.custom,
-            message: 'name accounts to use or request, a builds level or sandboxes',
+            path: ['repos'],
+            message: 'checks are read for repositories: name them in repos',
         });
     }
 });
@@ -451,6 +488,40 @@ exports.serverConfigSchema = zod_1.z
         // the most any task or service of a run reaches besides its needs: none, open
         // or { allow: [hosts] }. A version that asks for more is refused at publish.
         network: network_schema_1.networkSchema.default('open'),
+        // the most a task or service that reads a secret, or a variable of the server
+        // other than PATH, HOME and CI, reaches, under `network`: so a tool it runs can't
+        // send the value anywhere. Checked like `network`.
+        secretNetwork: network_schema_1.networkSchema.default('open'),
+        // which tasks take arguments: `any` but those that say `args: false`, or
+        // `declared`, only those that say `args: true`
+        args: zod_1.z.enum(['any', 'declared']).default('any'),
+        // runs of services (`services` in a run), kept up until their ttl ends
+        up: zod_1.z
+            .object({
+            // how long the services stay once ready, when the run names no ttl; then they are removed
+            ttl: duration.default('8h'),
+            // how long the services stay up without a request to an endpoint before they sleep,
+            // when the run names none: the next request wakes them. Unset, they never sleep.
+            sleepTtl: duration.optional(),
+            // the furthest a run, or an extend, sets the end from now
+            maxTtl: duration.default('72h'),
+            limits: zod_1.z
+                .object({
+                // service runs one caller has starting or up at a time; one more is 429
+                up: zod_1.z.number().int().min(1).default(3),
+            })
+                .strict()
+                .default({}),
+            // service runs starting or up on the whole server; one more is 503
+            total: zod_1.z.number().int().min(1).default(20),
+            // on a Kubernetes backend, the ResourceQuota of the namespace of each service
+            // run; backend.quota when left out
+            quota: zod_1.z.record(zod_1.z.union([zod_1.z.string().min(1), zod_1.z.number()]).transform(String)).optional(),
+        })
+            .strict()
+            .default({}),
+        // how long a passed check stays for `remote verify` to find (see checks.ts)
+        checksKeep: duration.default('30d'),
     })
         .strict()
         .default({}),
@@ -488,6 +559,45 @@ exports.serverConfigSchema = zod_1.z
     })
         .strict()
         .default({}),
+    // ports of service runs published at hosts of their own below `domain`, behind the
+    // server's sign-in. Absent, a run naming endpoints is refused.
+    endpoints: zod_1.z
+        .object({
+        // `run.example.com`: an endpoint is https://<label>.run.example.com, all of them
+        // pointing at this server with one wildcard DNS record
+        domain: zod_1.z
+            .string()
+            .regex(/^(?=.{1,200}$)([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)*[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/, 'expected a domain name in lower case, such as run.example.com'),
+        // a wildcard certificate for *.<domain>, served for the hosts of endpoints; left out
+        // behind a TLS proxy, which holds it
+        tls: zod_1.z
+            .object({ cert: zod_1.z.string().min(1), key: zod_1.z.string().min(1) })
+            .strict()
+            .optional(),
+        // how a person signs in with a browser: the authorization code flow of an issuer of
+        // `issuers`, with a client of its own; without it only tokens open endpoints
+        login: zod_1.z
+            .object({
+            issuer: zod_1.z.string().min(1),
+            clientId: zod_1.z.string().min(1),
+            // for providers whose web clients need one (Google): read as an account no
+            // entitlement may name, as the secret of a source is
+            clientSecret: zod_1.z
+                .object({
+                from: zod_1.z.string().regex(/^[A-Za-z][A-Za-z0-9_-]*:.+$/, 'expected provider:reference'),
+                account: accountNameSchema,
+            })
+                .strict()
+                .optional(),
+            scope: zod_1.z.string().min(1).default('openid'),
+        })
+            .strict()
+            .optional(),
+        // how long a sign-in at one endpoint lasts, at most until its run ends
+        session: duration.default('8h'),
+    })
+        .strict()
+        .optional(),
     backend: zod_1.z.discriminatedUnion('type', [hostBackendSchema, kubernetesBackendSchema]).default({ type: 'host' }),
 })
     .strict()
@@ -555,7 +665,7 @@ exports.serverConfigSchema = zod_1.z
             });
         }
     });
-    const readers = getSourceAccounts(config);
+    const readers = getServerAccounts(config);
     config.entitlements.forEach((entitlement, index) => {
         for (const account of Object.keys(entitlement.accounts)) {
             if (readers.includes(account)) {
@@ -586,7 +696,10 @@ exports.serverConfigSchema = zod_1.z
             message: `under runs.network, ${shared}`,
         });
     }
-    if (config.sandboxes.runtimes?.includes('local') && config.sandboxes.network !== 'open') {
+    // on a cluster a local task runs in a pod of a pool's tool image, which can be limited
+    if (config.sandboxes.runtimes?.includes('local') &&
+        config.sandboxes.network !== 'open' &&
+        config.backend.type === 'host') {
         ctx.addIssue({
             code: zod_1.z.ZodIssueCode.custom,
             path: ['sandboxes', 'network'],
@@ -635,7 +748,7 @@ exports.serverConfigSchema = zod_1.z
         });
         // the secrets of tasks are read in their Job, with the image of the provider; the
         // ones of the sources are read by the server, which checks the code out itself
-        const readers = getSourceAccounts(config);
+        const readers = getServerAccounts(config);
         for (const [name, account] of Object.entries(config.accounts)) {
             if (readers.includes(name)) {
                 continue;
@@ -697,6 +810,13 @@ exports.serverConfigSchema = zod_1.z
                 message: 'runs.network limits every task with a NetworkPolicy, set networkPolicy: enforced once the network plugin of the cluster enforces them',
             });
         }
+        if (config.runs.secretNetwork !== 'open' && backend.networkPolicy !== 'enforced') {
+            ctx.addIssue({
+                code: zod_1.z.ZodIssueCode.custom,
+                path: ['backend', 'networkPolicy'],
+                message: 'runs.secretNetwork limits the tasks that read secrets with a NetworkPolicy, set networkPolicy: enforced once the network plugin of the cluster enforces them',
+            });
+        }
         if (config.sandboxes.network !== 'open' &&
             backend.networkPolicy !== 'enforced' &&
             config.entitlements.some((entitlement) => entitlement.sandboxes?.length)) {
@@ -707,13 +827,85 @@ exports.serverConfigSchema = zod_1.z
             });
         }
     }
-    if (config.runs.network !== 'open' && config.runtimes.includes('local')) {
+    if (config.runs.network !== 'open' && config.runtimes.includes('local') && backend.type === 'host') {
         ctx.addIssue({
             code: zod_1.z.ZodIssueCode.custom,
             path: ['runs', 'network'],
             message: 'the network of a local task can not be limited, allow only the docker runtime with runs.network',
         });
     }
+    if ((0, units_1.parseDuration)(config.runs.up.ttl) > (0, units_1.parseDuration)(config.runs.up.maxTtl)) {
+        ctx.addIssue({
+            code: zod_1.z.ZodIssueCode.custom,
+            path: ['runs', 'up', 'ttl'],
+            message: `a ttl of ${config.runs.up.ttl} is more than maxTtl of ${config.runs.up.maxTtl}`,
+        });
+    }
+    const endpoints = config.endpoints;
+    if (endpoints) {
+        if (endpoints.tls && config.behindTlsProxy) {
+            ctx.addIssue({
+                code: zod_1.z.ZodIssueCode.custom,
+                path: ['endpoints', 'tls'],
+                message: 'behind a TLS proxy the proxy holds the certificate of the endpoints, leave out endpoints.tls',
+            });
+        }
+        if (endpoints.tls && !config.tls) {
+            ctx.addIssue({
+                code: zod_1.z.ZodIssueCode.custom,
+                path: ['endpoints', 'tls'],
+                message: 'the certificate of the endpoints is served next to the one of the server, set tls as well',
+            });
+        }
+        if (!endpoints.tls && !config.behindTlsProxy && (config.tls || !LOOPBACK_HOSTS.includes(config.listen.host))) {
+            ctx.addIssue({
+                code: zod_1.z.ZodIssueCode.custom,
+                path: ['endpoints', 'tls'],
+                message: `name the certificate of *.${endpoints.domain} in endpoints.tls, or set behindTlsProxy: true when a proxy in front holds it`,
+            });
+        }
+        const login = endpoints.login;
+        if (login && !config.issuers.some((issuer) => issuer.issuer === login.issuer)) {
+            ctx.addIssue({
+                code: zod_1.z.ZodIssueCode.custom,
+                path: ['endpoints', 'login', 'issuer'],
+                message: `issuer ${login.issuer} is not one of the accepted issuers`,
+            });
+        }
+        if (login?.clientSecret) {
+            const { from, account } = login.clientSecret;
+            const kind = from.substring(0, from.indexOf(':'));
+            const where = ['endpoints', 'login', 'clientSecret'];
+            if (!config.accounts[account]) {
+                ctx.addIssue({
+                    code: zod_1.z.ZodIssueCode.custom,
+                    path: [...where, 'account'],
+                    message: `unknown account ${account}`,
+                });
+            }
+            else if (!config.accounts[account].providers[kind]) {
+                ctx.addIssue({
+                    code: zod_1.z.ZodIssueCode.custom,
+                    path: [...where, 'from'],
+                    message: `account ${account} has no credentials for the provider ${kind}`,
+                });
+            }
+            if (!config.secretProviders[kind]) {
+                ctx.addIssue({
+                    code: zod_1.z.ZodIssueCode.custom,
+                    path: [...where, 'from'],
+                    message: `secret provider ${kind} is not declared in secretProviders`,
+                });
+            }
+        }
+        if (backend.type === 'kubernetes' && backend.kubeconfig) {
+            ctx.addIssue({
+                code: zod_1.z.ZodIssueCode.custom,
+                path: ['endpoints'],
+                message: 'endpoints reach the services of runs at their Service in the cluster, run the server in the cluster rather than with a kubeconfig',
+            });
+        }
+    }
     if (!config.tls && !config.behindTlsProxy && !LOOPBACK_HOSTS.includes(config.listen.host)) {
         ctx.addIssue({
             code: zod_1.z.ZodIssueCode.custom,
@@ -729,11 +921,18 @@ function getSandboxRuntimes(config) {
     return config.sandboxes.runtimes ?? config.runtimes.filter((runtime) => runtime === 'docker');
 }
 exports.getSandboxRuntimes = getSandboxRuntimes;
-// The accounts that read repositories: no caller uses them and no build file names them.
-function getSourceAccounts(config) {
-    return [...new Set(config.sources.flatMap((source) => (source.secret ? [source.secret.account] : [])))];
+// The accounts only the server reads secrets as: those that read the repositories of
+// `sources`, and the one of the client secret of `endpoints.login`. No caller uses them and
+// no build file names them.
+function getServerAccounts(config) {
+    return [
+        ...new Set([
+            ...config.sources.flatMap((source) => (source.secret ? [source.secret.account] : [])),
+            ...(config.endpoints?.login?.clientSecret ? [config.endpoints.login.clientSecret.account] : []),
+        ]),
+    ];
 }
-exports.getSourceAccounts = getSourceAccounts;
+exports.getServerAccounts = getServerAccounts;
 async function readServerConfig(environment, path) {
     if (!(await environment.file.exists(path))) {
         throw new Error(`unable to read ${path}: no such file`);