hammerkit 1.25.0 → 1.27.0-next.1

package/dist/server/executor.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.HostExecutor = exports.defaultWorker = void 0;
+exports.HostExecutor = exports.volumeScope = exports.cacheNamespace = exports.defaultWorker = void 0;
 const child_process_1 = require("child_process");
 const crypto_1 = require("crypto");
 const fs_1 = require("fs");
@@ -10,13 +10,15 @@ const readline_1 = require("readline");
 const resolve_git_source_1 = require("../includes/resolve-git-source");
 const git_checkout_1 = require("./git-checkout");
 const run_plan_1 = require("./run-plan");
+const work_network_1 = require("../planner/work-network");
 // `.git` as a file system may read it: any case, and a name Windows reads without its
 // trailing dots and spaces, or by its short name
 function isGitDirectory(segment) {
     const name = segment.toLowerCase().replace(/[. ]+$/, '');
     return name === '.git' || name === 'git~1';
 }
-// A cancelled run gets this long to stop its tasks and remove its containers
+// A cancelled run gets this long to stop its tasks and remove its containers, after that
+// its process is killed
 const STOP_GRACE_MS = 15000;
 // what is kept of the error output of a process that died without saying why
 const MAX_STDERR = 2 * 1024;
@@ -30,19 +32,29 @@ function defaultWorker() {
 }
 exports.defaultWorker = defaultWorker;
 // The cache a run shares with others is the one of its source: the runs of a repository
-// share one, those of an upload only the uploader's. Somebody who can only upload can
-// not leave an entry for a run from a reviewed repository to find.
+// share one, those of an upload or a sandbox only their owner's. Somebody who can only
+// upload, or an agent on a sandbox, can not leave an entry for a run from a reviewed
+// repository to find.
 function cacheNamespace(run) {
-    const key = run.source.type === 'ref' ? `ref\0${run.source.repo}` : `upload\0${run.owner.iss}\0${run.owner.sub}`;
+    const key = run.source.type === 'ref' ? `ref\0${run.source.repo}` : `${run.source.type}\0${run.owner.iss}\0${run.owner.sub}`;
     return `${run.source.type}-${(0, crypto_1.createHash)('sha256').update(key).digest('hex').slice(0, 24)}`;
 }
+exports.cacheNamespace = cacheNamespace;
+// The docker volumes of a run's services are its caller's, within the cache's source trust:
+// two callers of one repository share a cache, a task is keyed by what it reads, but not the
+// data their services keep.
+function volumeScope(run) {
+    const key = `${cacheNamespace(run)}\0${run.owner.iss}\0${run.owner.sub}`;
+    return `${run.source.type}-${(0, crypto_1.createHash)('sha256').update(key).digest('hex').slice(0, 24)}`;
+}
+exports.volumeScope = volumeScope;
 /**
  * Runs the tasks of a run on this machine with the engine of `hammerkit run`, in a
  * process of its own.
  *
- * The working directory is made for the run and removed afterwards. Its build
- * file is the published version the run is pinned to, written over the code of the
- * source, so the commands are the published ones whatever the source says. The process
+ * The working directory is made for the run and removed afterwards, or reported if it can
+ * not be. Its build file is the published version the run is pinned to, written over the
+ * code of the source, so the commands are the published ones whatever the source says. The process
  * starts with an environment the server builds: the variables `runs.env` lists, a home
  * directory that is the run's alone, and a cache directory per source trust. So what
  * the engine reads of its process (the docker host, the registry logins in the home
@@ -56,21 +68,25 @@ class HostExecutor {
     }
     async execute(run, context) {
         const { definitions, planning, repos } = this.options;
-        const version = definitions.find(run.definition.name, run.definition.hash);
+        const version = definitions.find(run.definition);
         const bundle = version ? await definitions.files(version) : undefined;
         if (!version || !bundle) {
-            throw new Error(`the build file ${run.definition.name} at ${run.definition.hash} is no longer on this server`);
+            throw new Error(`the build file ${run.definition} is no longer on this server`);
         }
         // pulled back after the run was accepted, while it waited for an approval or in the queue
         if (version.yanked) {
-            throw new Error(`the build file ${run.definition.name} at ${run.definition.hash} was yanked${version.yanked.reason ? `: ${version.yanked.reason}` : ''}`);
+            throw new Error(`the build file ${run.definition} was yanked${version.yanked.reason ? `: ${version.yanked.reason}` : ''}`);
         }
-        if (run.source.type !== 'ref') {
+        if (run.source.type === 'upload') {
             throw new Error('runs from an upload are not supported by this server yet, use a git ref');
         }
-        const { repo, ref, commit: pinned } = run.source;
-        if (!repos.readable(repo)) {
-            throw new Error(`this server can not read ${repo}, the operator lists it in sources`);
+        const source = run.source;
+        const sandboxes = this.options.sandboxes;
+        if (source.type === 'sandbox' && !sandboxes) {
+            throw new Error('this server keeps no sandboxes');
+        }
+        if (source.type === 'ref' && !repos.readable(source.repo)) {
+            throw new Error(`this server can not read ${source.repo}, the operator lists it in sources`);
         }
         const worker = this.options.worker ?? defaultWorker();
         if (!worker) {
@@ -78,11 +94,16 @@ class HostExecutor {
         }
         // what was approved is what runs: the accounts are read again from the files with
         // the environment of the run, and the run may use no more than it was admitted for
-        const planned = await (0, run_plan_1.planRun)(bundle, run, planning.environment, planning.policy);
+        const policy = source.type === 'sandbox' && sandboxes ? sandboxes.policy : planning.policy;
+        const planned = await (0, run_plan_1.planRun)(bundle, run, planning.environment, policy);
         const extra = planned.accounts.filter((account) => !run.accounts.includes(account));
         if (extra.length > 0) {
             throw new Error(`the run would use ${extra.join(', ')}, which it was not admitted for`);
         }
+        const refused = (0, run_plan_1.refuseArguments)(run, planned.plan);
+        if (refused) {
+            throw new Error(refused);
+        }
         const abortCtrl = new AbortController();
         const abort = () => abortCtrl.abort();
         if (context.signal.aborted) {
@@ -90,34 +111,119 @@ class HostExecutor {
         }
         context.signal.addEventListener('abort', abort, { once: true });
         const root = (0, path_1.join)(this.options.directory, run.id);
-        const checkout = (0, path_1.join)(root, 'source');
+        let attachment;
         try {
-            const credentials = await repos.credentials(repo);
-            let commit;
-            try {
-                commit = await (0, git_checkout_1.checkoutRef)(repos.url(repo), ref, checkout, credentials.env, abortCtrl.signal, pinned);
+            // where the process of the run starts: the checkout of a ref, or for a sandbox the
+            // published files, so what the process and the secret providers it starts read of
+            // their directory is never what a task wrote
+            let cwd;
+            let input;
+            let claim;
+            // the files of the sandbox the tasks work on
+            let work;
+            if (source.type === 'ref') {
+                cwd = (0, path_1.join)(root, 'source');
+                const { repo, ref, commit: pinned } = source;
+                const credentials = await repos.credentials(repo);
+                let commit;
+                try {
+                    commit = await (0, git_checkout_1.checkoutRef)(repos.url(repo), ref, cwd, credentials.env, abortCtrl.signal, pinned);
+                }
+                catch (e) {
+                    throw repos.scrub(e, credentials.values);
+                }
+                context.emit({ type: 'log', task: 'hammerkit', stream: 'stdout', text: `checked out ${repo} at ${commit}` });
+                await this.writeBundle(cwd, bundle.files);
+                // what the version was checked against at publish holds while it runs too
+                input = {
+                    entry: (0, path_1.join)(cwd, version.entry),
+                    ...(planning.policy.network ? { network: (0, work_network_1.toNetworkSchema)(planning.policy.network) } : {}),
+                };
             }
-            catch (e) {
-                throw repos.scrub(e, credentials.values);
+            else {
+                // the published files are read from a directory of the run and work on the
+                // sandbox, whose own build files are just files
+                const { registry, network } = sandboxes;
+                // read before the run attaches: until then the walk of the last run's measure holds
+                const state = await registry.state(source.id, abortCtrl.signal);
+                attachment = await registry.attach(source.id, (0, path_1.join)(root, 'sandbox'));
+                work = attachment.directory;
+                claim = attachment.claim;
+                // what was approved is what runs: the files may have changed since, by a run
+                // before this one on the sandbox
+                if (run.approvals?.length && state.tree !== source.tree) {
+                    throw new Error(`the files of the sandbox ${source.id} changed since the run was approved, they are ${state.tree} now and were ${source.tree}, submit the run again`);
+                }
+                if (claim && !this.options.placement) {
+                    throw new Error(`the sandbox ${source.id} is a claim on a cluster, this server runs tasks on its own docker`);
+                }
+                context.emit({
+                    type: 'log',
+                    task: 'hammerkit',
+                    stream: 'stdout',
+                    text: `sandbox ${source.id} of ${source.repo} at ${state.commit}, files ${state.tree}${state.clean ? '' : ' with changes'}`,
+                });
+                const published = (0, path_1.join)(root, 'bundle');
+                await (0, promises_1.mkdir)(published, { recursive: true });
+                await this.writeBundle(published, bundle.files);
+                cwd = published;
+                input = {
+                    entry: (0, path_1.join)(published, version.entry),
+                    directory: { read: published, work },
+                    // the worker takes the lower of sandboxes.network and runs.network
+                    ...(planning.policy.network ? { network: (0, work_network_1.toNetworkSchema)(planning.policy.network) } : {}),
+                    sandboxNetwork: network,
+                };
             }
-            context.emit({ type: 'log', task: 'hammerkit', stream: 'stdout', text: `checked out ${repo} at ${commit}` });
-            await this.writeBundle(checkout, bundle.files);
             const home = (0, path_1.join)(root, 'home');
             await (0, promises_1.mkdir)(home, { recursive: true });
             const cache = (0, path_1.join)(this.options.cache, cacheNamespace(run));
             await (0, promises_1.mkdir)(cache, { recursive: true });
-            const input = {
-                entry: (0, path_1.join)(checkout, version.entry),
-                tasks: run.tasks,
-                options: run.options,
-                accounts: run.accounts,
-                catalog: planning.environment.hosted?.catalog,
-            };
-            await this.runInProcess(worker, input, checkout, this.processEnvironment(home, cache), context, abortCtrl.signal);
+            const placement = this.options.placement;
+            const placed = placement ? await placement.prepare(run, context, { root: work ?? cwd, claim }) : undefined;
+            try {
+                await this.runInProcess(worker, {
+                    ...input,
+                    tasks: run.tasks,
+                    options: run.options,
+                    accounts: run.accounts,
+                    volumeScope: volumeScope(run),
+                    catalog: planning.environment.hosted?.catalog,
+                    ...(placed ? { kubernetes: placed.kubernetes } : {}),
+                }, cwd, { ...this.processEnvironment(home, cache), ...placed?.env }, context, abortCtrl.signal);
+            }
+            finally {
+                if (placement && placed) {
+                    await placement.release(run, context);
+                }
+            }
         }
         finally {
             context.signal.removeEventListener('abort', abort);
-            await (0, promises_1.rm)(root, { recursive: true, force: true });
+            attachment?.release?.();
+            await this.removeRunDirectory(run, root, context);
+            if (source.type === 'sandbox') {
+                // what the run wrote counts against the owner's limits, from their next run or create
+                sandboxes?.registry.measure(source.id).catch(() => undefined);
+            }
+        }
+    }
+    // A task of the run may still hold a file of its directory for a moment after the run
+    // ended, which Windows refuses to remove. A directory that stays is reported, to the run
+    // by its error code and to the operator with its path; it does not change how the run ended.
+    async removeRunDirectory(run, root, context) {
+        try {
+            await (0, promises_1.rm)(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
+        }
+        catch (e) {
+            const error = e;
+            context.emit({
+                type: 'log',
+                task: 'hammerkit',
+                stream: 'stderr',
+                text: `the working directory of the run could not be removed: ${error.code ?? 'unknown error'}`,
+            });
+            this.options.log?.(`run=${run.id} could not remove its directory: ${error.message}`);
         }
     }
     // What the process of a run sees: the variables the operator listed, and a home
@@ -136,9 +242,11 @@ class HostExecutor {
         env.HAMMERKIT_DIR = cache;
         return env;
     }
-    // Starts the process, hands it the run on its standard input and relays what it reports. It
-    // is stopped with SIGTERM when the run is cancelled, so the engine removes its containers
-    // and stops its tasks, and killed if it does not stop in time.
+    // Starts the process, hands it the run on its standard input and relays what it reports.
+    // A cancelled run is told so on the standard input, the same on every platform, so the
+    // engine stops its tasks and removes its containers; a signal on Windows would end the
+    // process at once and leave its tasks running in the directory of the run. The process is
+    // killed if it has not stopped after the grace period.
     runInProcess(command, input, cwd, env, context, signal) {
         return new Promise((resolvePromise, reject) => {
             const child = (0, child_process_1.spawn)(command[0], command.slice(1), {
@@ -147,12 +255,19 @@ class HostExecutor {
                 stdio: ['pipe', 'pipe', 'pipe'],
                 windowsHide: true,
             });
+            child.stdin.on('error', () => undefined);
+            // the run first, so a cancel comes after it; kept open, it is how a cancel reaches the process
+            child.stdin.write(`${JSON.stringify(input)}\n`);
             let result;
             let stderr = '';
             let killTimer;
             const stop = () => {
-                child.kill('SIGTERM');
-                killTimer = setTimeout(() => child.kill('SIGKILL'), STOP_GRACE_MS);
+                child.stdin.write('cancel\n', (e) => {
+                    if (e) {
+                        child.kill();
+                    }
+                });
+                killTimer = setTimeout(() => child.kill('SIGKILL'), this.options.stopGraceMs ?? STOP_GRACE_MS);
                 killTimer.unref();
             };
             if (signal.aborted) {
@@ -169,6 +284,8 @@ class HostExecutor {
                     }
                     else if (message.kind === 'result') {
                         result = message;
+                        // nothing left to tell the process
+                        child.stdin.end();
                     }
                 }
                 catch {
@@ -178,8 +295,6 @@ class HostExecutor {
             child.stderr.on('data', (chunk) => {
                 stderr = (stderr + chunk.toString('utf8')).slice(-MAX_STDERR);
             });
-            child.stdin.on('error', () => undefined);
-            child.stdin.end(JSON.stringify(input));
             child.on('error', (e) => {
                 signal.removeEventListener('abort', stop);
                 clearTimeout(killTimer);

package/dist/server/git-checkout.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.resolveRef = exports.RefError = exports.checkoutRef = exports.isValidRef = void 0;
+exports.git = exports.remoteBranch = exports.isValidBranch = exports.assertBranchName = exports.resolveRef = exports.GitOutputTooLarge = exports.RefError = exports.checkoutRef = exports.isValidRef = void 0;
 const child_process_1 = require("child_process");
 const promises_1 = require("fs/promises");
 const util_1 = require("util");
@@ -42,13 +42,20 @@ exports.checkoutRef = checkoutRef;
 class RefError extends Error {
 }
 exports.RefError = RefError;
+class GitOutputTooLarge extends Error {
+}
+exports.GitOutputTooLarge = GitOutputTooLarge;
 const COMMIT = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
 /**
  * The commit a branch or tag of a repository points to now: `refs/heads/<ref>`, else
  * `refs/tags/<ref>` (the commit of an annotated tag). A name that is both a branch and
- * a tag is ambiguous. A full commit hash is taken as it is.
+ * a tag is ambiguous. A full commit hash is taken as it is. With `branch`, only a
+ * branch: a sandbox starts from one.
  */
-async function resolveRef(repository, ref, env, signal) {
+async function resolveRef(repository, ref, env, signal, only) {
+    if (only === 'branch') {
+        return resolveBranch(repository, ref, env, signal);
+    }
     if (!isValidRef(ref)) {
         throw new RefError(`${JSON.stringify(ref.slice(0, 80))} is not a branch, tag or commit`);
     }
@@ -84,7 +91,52 @@ async function resolveRef(repository, ref, env, signal) {
     return commit;
 }
 exports.resolveRef = resolveRef;
-async function git(args, baseEnv, signal, what) {
+async function resolveBranch(repository, branch, env, signal) {
+    assertBranchName(branch);
+    const commit = await remoteBranch(repository, branch, env, signal);
+    if (!commit) {
+        throw new RefError(`there is no branch ${branch}`);
+    }
+    return commit;
+}
+// A name a sandbox can start from: a branch, never a tag or a commit, written without `refs/heads/`.
+function assertBranchName(branch) {
+    if (!isValidRef(branch) || branch.startsWith('refs/')) {
+        throw new RefError(`${JSON.stringify(branch.slice(0, 80))} is not a branch`);
+    }
+    if (COMMIT.test(branch.toLowerCase())) {
+        throw new RefError(`a sandbox starts from a branch, ${branch} is a commit`);
+    }
+}
+exports.assertBranchName = assertBranchName;
+// A branch git takes as a name of its own: a ref, and no part of it starts or ends with a
+// dot or ends with `.lock` (`git check-ref-format` refuses those).
+function isValidBranch(branch) {
+    return (isValidRef(branch) &&
+        branch
+            .split('/')
+            .every((part) => !part.startsWith('.') && !part.startsWith('-') && !part.endsWith('.') && !part.endsWith('.lock')));
+}
+exports.isValidBranch = isValidBranch;
+// The commit a branch of a repository points to now, none when there is no such branch.
+async function remoteBranch(repository, branch, env, signal) {
+    const output = await git(['ls-remote', '--heads', '--', repository, `refs/heads/${branch}`], env, signal, `the branches of ${repository}`);
+    for (const line of output.split('\n')) {
+        const [commit, name] = line.split('\t');
+        if (commit && name === `refs/heads/${branch}`) {
+            return commit;
+        }
+    }
+    return undefined;
+}
+exports.remoteBranch = remoteBranch;
+// Runs git without a prompt and with the network transports only. Errors say what could
+// not be done (`unable to <action> <what>`) with the last line git printed.
+async function git(args, baseEnv, signal, what, action = 'fetch', 
+// what git reads on stdin, the exit codes that answer rather than fail (check-ignore exits
+// 1 when nothing is ignored), and what git may print (a diff of a sandbox can be larger
+// than the default)
+options = {}) {
     const env = {};
     for (const [key, value] of Object.entries(baseEnv)) {
         if (value !== undefined) {
@@ -94,22 +146,34 @@ async function git(args, baseEnv, signal, what) {
     env.GIT_TERMINAL_PROMPT = '0';
     env.GIT_ALLOW_PROTOCOL = ALLOWED_PROTOCOLS;
     try {
-        return (await execFileAsync('git', args, { env, signal })).stdout;
+        // warnings of a large tree must not fail the command
+        const running = execFileAsync('git', args, { env, signal, maxBuffer: options.maxBuffer ?? 64 * 1024 * 1024 });
+        // a git that exits before it read its input fails the command, not the server (EPIPE)
+        running.child.stdin?.on('error', () => undefined);
+        running.child.stdin?.end(options.input);
+        return (await running).stdout;
     }
     catch (e) {
         const error = e;
+        if (typeof error.code === 'number' && options.exitCodes?.includes(error.code)) {
+            return error.stdout ?? '';
+        }
         if (error.code === 'ABORT_ERR') {
-            throw new Error(`fetching ${what} was cancelled`);
+            throw new Error(`${action === 'fetch' ? 'fetching' : `${action} of`} ${what} was cancelled`);
         }
         if (error.code === 'ENOENT') {
-            throw new Error(`unable to fetch ${what}: git is not installed`);
+            throw new Error(`unable to ${action} ${what}: git is not installed`);
+        }
+        if (error.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER') {
+            throw new GitOutputTooLarge(`unable to ${action} ${what}: git printed more than ${options.maxBuffer ?? 64 * 1024 * 1024} bytes`);
         }
         const reason = (error.stderr ?? '')
             .split('\n')
             .map((line) => line.trim())
             .filter((line) => line.length > 0)
             .pop() ?? error.message;
-        throw new Error(`unable to fetch ${what}: ${reason}`);
+        throw new Error(`unable to ${action} ${what}: ${reason}`);
     }
 }
+exports.git = git;
 //# sourceMappingURL=git-checkout.js.map