hammerkit 1.27.0-next.8 → 2.0.0-next.1

package/dist/remote/remote-work-commands.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.registerWorkCommands = exports.describePlan = exports.describeApproval = void 0;
+exports.registerWorkCommands = exports.describePlan = exports.describeApproval = exports.keepRuns = exports.describeListedRun = exports.describeRunEnd = void 0;
 const commander_1 = require("commander");
 const colors_1 = __importDefault(require("colors"));
 const token_source_1 = require("./token-source");
@@ -13,8 +13,41 @@ const annotations_1 = require("./annotations");
 const server_client_1 = require("./server-client");
 const bundle_text_1 = require("./bundle-text");
 const sandbox_client_1 = require("../harness/sandbox-client");
+const units_1 = require("../utils/units");
+const service_tools_1 = require("../harness/service-tools");
 const path_1 = require("path");
+const check_keys_1 = require("../optimizer/check-keys");
+const remote_status_1 = require("./remote-status");
+const timings_1 = require("../server/timings");
+const terminal_prompt_1 = require("../utils/terminal-prompt");
+function describeOutcome(planned) {
+    switch (planned.outcome) {
+        case 'published':
+            return 'publishing it would publish it';
+        case 'already published':
+            return planned.yanked ? 'it is published already, and yanked' : 'it is published already';
+        case 'proposed':
+            return planned.missing?.length
+                ? `publishing it would propose it, it names accounts you do not use: ${planned.missing
+                    .map(server_client_1.printable)
+                    .join(', ')}`
+                : 'publishing it would propose it, somebody who may publish approves it';
+        default:
+            return 'the server accepts it, publishing it needs builds: propose or publish';
+    }
+}
 const RUN_ID = /^run_[A-Za-z0-9_-]{16}$/;
+// how a check passed, one line
+function describePassed(passed) {
+    const from = passed.source.type === 'ref'
+        ? `${passed.source.ref} at ${passed.source.commit.slice(0, 12)}`
+        : `a check of sandbox ${passed.source.sandbox}, fetched at ${passed.source.commit.slice(0, 12)}`;
+    return `passed ${passed.finishedAt}${passed.status === 'cached'
+        ? ' from the cache'
+        : passed.durationMs !== undefined
+            ? ` in ${(0, units_1.formatDuration)(passed.durationMs)}`
+            : ''}, ${passed.run} by ${passed.by.sub}, from ${from}`;
+}
 // what the publisher said about where a version comes from, one line
 // What a publisher said about where a version comes from: the server takes it as said,
 // so anybody else is told it is not checked.
@@ -37,6 +70,46 @@ function printableLine(text) {
     return text.split('\t').map(server_client_1.printableWhole).join('\t');
 }
 const collectOption = (value, previous = []) => [...previous, value];
+/**
+ * Why a run that did not succeed ended: its error, then each task that failed with its own
+ * error or exit code, where the error of the run does not say it already.
+ */
+function describeRunEnd(run) {
+    const lines = [`${(0, server_client_1.printable)(run.id)} ${(0, server_client_1.printable)(run.state)}${run.error ? `: ${(0, server_client_1.printableWhole)(run.error)}` : ''}`];
+    for (const [task, result] of Object.entries(run.results ?? {})) {
+        if (result.status !== 'failed') {
+            continue;
+        }
+        const why = result.error ?? (result.exitCode !== undefined ? `exit code ${result.exitCode}` : undefined);
+        if (why && !run.error?.includes(`${task}: ${why}`)) {
+            lines.push(`  ${(0, server_client_1.printable)(task)}: ${(0, server_client_1.printableWhole)(why)}`);
+        }
+    }
+    return lines;
+}
+exports.describeRunEnd = describeRunEnd;
+// What `remote runs` shows of a run, on one line: how long it waited for a place and ran.
+function describeListedRun(run, now = Date.now()) {
+    const what = run.up ? `up ${run.up.services.join(', ')}` : run.tasks.join(', ') || 'by labels';
+    const until = run.up && (run.state === 'up' || run.state === 'asleep') && run.expiresAt ? ` until ${run.expiresAt}` : '';
+    const queue = run.queue
+        ? ` (${run.queue.position}. in the queue, waits: ${(0, server_client_1.printable)((0, remote_status_1.describeWait)(run.queue.waitsFor, !run.owner))})`
+        : '';
+    const waited = (0, timings_1.waitedMs)(run, now);
+    const ran = (0, timings_1.ranMs)(run, now);
+    const times = [
+        waited !== undefined ? `waited ${(0, timings_1.roughly)(waited)}` : '',
+        ran !== undefined ? `${run.up ? 'up after' : 'ran'} ${(0, timings_1.roughly)(ran)}` : '',
+    ].filter(Boolean);
+    return `${(0, server_client_1.printable)(run.id)}  ${(0, server_client_1.printable)(run.createdAt)}  ${(0, server_client_1.printable)(run.state)}${(0, server_client_1.printable)(until)}${queue}  ${(0, server_client_1.printable)(what)}${run.owner ? ` of ${(0, server_client_1.printable)(run.owner.sub)}` : ''}${times.length > 0 ? `  ${times.join(', ')}` : ''}${run.error ? `: ${(0, server_client_1.printable)(run.error)}` : ''}`;
+}
+exports.describeListedRun = describeListedRun;
+// What a server from before the filters of GET /v1/runs answers is filtered here, so an
+// older server never shows task runs as service runs.
+function keepRuns(runs, states, kind) {
+    return runs.filter((run) => (!states || states.includes(run.state)) && (!kind || (kind === 'services') === !!run.up));
+}
+exports.keepRuns = keepRuns;
 // What a person who approves is shown of a run: everything the decision rests on.
 function describeApproval(run) {
     const source = run.source.type === 'ref'
@@ -92,33 +165,51 @@ function describePlan(plan) {
         ...(entry.mounts.length > 0 ? [`    mounts: ${entry.mounts.map(server_client_1.printable).join(', ')}`] : []),
         ...(entry.ports.length > 0 ? [`    ports: ${entry.ports.map(server_client_1.printable).join(', ')}`] : []),
         ...(entry.generates.length > 0 ? [`    generates: ${entry.generates.map(server_client_1.printable).join(', ')}`] : []),
+        ...(entry.capAdd?.length ? [`    capAdd: ${entry.capAdd.map(server_client_1.printable).join(', ')}`] : []),
     ]);
 }
 exports.describePlan = describePlan;
 function registerWorkCommands(remote, context) {
     const { environment, buildFile, getServer, guarded, passthrough, fail } = context;
+    // the ref a run starts from: what the options name, else the checkout here
+    const refSource = async (options) => {
+        const checkout = options.repo && options.ref ? {} : await (0, annotations_1.readCheckout)((0, path_1.dirname)(buildFile));
+        const repo = options.repo ?? checkout.repo;
+        const ref = options.ref ?? checkout.branch;
+        // the commit checked out here is the code meant, when the ref is the branch it is on
+        const commit = options.commit ?? (options.ref ? undefined : checkout.commit);
+        if (!repo) {
+            throw new Error('name the repository with --repo, the checkout here has no origin it can be told by');
+        }
+        if (!ref) {
+            throw new Error('name the branch or tag with --ref, the checkout here is on no branch');
+        }
+        return { type: 'ref', repo, ref, ...(commit ? { commit } : {}) };
+    };
     const connect = async (name) => {
         const chosen = await getServer(name);
         // a CI job or an agent brings its own token, a person signs in
         const token = await (0, token_source_1.getTokenSource)(environment, chosen.name, chosen.server).token();
         return { ...chosen, token };
     };
-    remote
-        .command('publish [server]')
-        .description('publish the build file and the files it includes to a server, runs refer to it by its hash')
-        .addOption(new commander_1.Option('--annotation <key=value>', 'say more about where it comes from, or leave one out with key=; path, repo, commit, machine and user are filled in').argParser(collectOption))
-        .action(guarded('Remote publish was not successful', async (name, options) => {
-        const bundle = await (0, collect_bundle_1.collectBundle)(buildFile, environment);
-        const annotations = (0, annotations_1.applyAnnotations)(await (0, annotations_1.collectAnnotations)(buildFile, Object.keys(bundle.files)), options.annotation ?? []);
-        const { name: serverName, server, token } = await connect(name);
+    // publishes the bundle with where it comes from and says what it is now: published,
+    // already published, or only proposed
+    const publishBundle = async (serverName, server, token, bundle, annotations) => {
         const answer = await (0, server_client_1.callServer)(server, token, 'POST', '/v1/definitions', {
             ...bundle,
             annotations,
         });
         const version = answer.body;
         const files = Object.keys(bundle.files).length;
-        const proposed = answer.status === 202;
-        const verb = proposed ? 'proposed' : answer.status === 201 ? 'published' : 'already published';
+        // a proposal that already waits answers 200 like a published version, with its state
+        const proposed = answer.status === 202 || version.state !== undefined;
+        const verb = answer.status === 202
+            ? 'proposed'
+            : proposed
+                ? 'already proposed'
+                : answer.status === 201
+                    ? 'published'
+                    : 'already published';
         environment.stdout.write(`${verb} to ${serverName}: ${(0, server_client_1.printable)(version.hash)} (${files} ${files === 1 ? 'file' : 'files'})\n`);
         const names = [...(version.tasks ?? []), ...(version.services ?? [])];
         environment.stdout.write(`tasks and services: ${names.map(server_client_1.printable).join(', ') || 'none'}\n`);
@@ -128,6 +219,123 @@ function registerWorkCommands(remote, context) {
         if (proposed) {
             environment.stdout.write(`it can not run before somebody who may publish approves it, until ${(0, server_client_1.printable)(version.expiresAt)}\n`);
         }
+        return { version, proposed };
+    };
+    // what publishing the bundle would do, undefined from a server that can't say (before 1.27)
+    const planBundle = async (server, token, bundle) => {
+        try {
+            return (await (0, server_client_1.callServer)(server, token, 'POST', '/v1/definitions/plan', bundle)).body;
+        }
+        catch (e) {
+            if (e instanceof server_client_1.ServerError && e.status === 404) {
+                return undefined;
+            }
+            throw e;
+        }
+    };
+    // The build file a run or up names, and what `--publish` may publish: the hash of the
+    // build file here, unless `--build` or HAMMERKIT_BUILD names a version.
+    const runDefinition = async (options) => {
+        const named = options.build ?? environment.processEnvs.HAMMERKIT_BUILD;
+        if (named !== undefined) {
+            if (options.publish) {
+                throw new Error(`--publish publishes the build file here, it does not go with ${options.build !== undefined ? '--build' : 'HAMMERKIT_BUILD'}`);
+            }
+            if (!bundle_1.HASH_PATTERN.test(named)) {
+                throw new Error(`${(0, server_client_1.printable)(named)} is not the hash of a published version (sha256:...)`);
+            }
+            return { definition: named };
+        }
+        const local = await (0, collect_bundle_1.collectBundle)(buildFile, environment);
+        return { definition: (0, bundle_1.hashBundle)(local), local };
+    };
+    // Submits a run of a published version. When the server does not have the build file
+    // here, it is published first: with --publish at once, else when the person at the
+    // terminal says so. Without a terminal and without --publish the run fails and says how
+    // to publish it, so CI never waits on a question.
+    const submitPublishing = async (target, request, local, publish) => {
+        const { serverName, server, connection } = target;
+        try {
+            return await (0, sandbox_client_1.submitRun)(connection, request);
+        }
+        catch (e) {
+            const status = e.status;
+            if (!local || status !== 400 || !/no published build file/.test(String(e))) {
+                throw e;
+            }
+        }
+        const definition = request.definition;
+        const notPublished = `the build file here is not published on ${serverName} (${definition})`;
+        const token = await connection.token();
+        if (!publish) {
+            if (!environment.prompt) {
+                throw new Error(`${notPublished}, publish it first with: hammerkit remote publish ${serverName}, or pass --publish`);
+            }
+            // a server before 1.27 can't say what publishing would do: it is asked all the same,
+            // and refuses then what the caller may not do
+            const planned = await planBundle(server, token, local);
+            if (planned && planned.outcome === null) {
+                throw new Error(`${notPublished}, and you may only run published build files (builds: run): ` +
+                    `somebody who may publish publishes it with: hammerkit remote publish ${serverName}`);
+            }
+            if (planned?.outcome !== 'already published') {
+                const files = Object.keys(local.files).length;
+                environment.stdout.write(`${notPublished}, ${files} ${files === 1 ? 'file' : 'files'}\n`);
+                if (planned) {
+                    const names = [...(planned.tasks ?? []), ...(planned.services ?? [])];
+                    environment.stdout.write(`tasks and services: ${names.map(server_client_1.printable).join(', ') || 'none'}\n`);
+                }
+                const proposing = planned?.outcome === 'proposed';
+                if (proposing) {
+                    environment.stdout.write(`${describeOutcome(planned)}\n`);
+                }
+                const answer = await environment.prompt(proposing ? `propose it to ${serverName} now? [y/N] ` : `publish it to ${serverName} now and go on? [y/N] `);
+                if (environment.abortCtrl.signal.aborted) {
+                    throw new Error(`stopped, nothing was published`);
+                }
+                if (!(0, terminal_prompt_1.isYes)(answer)) {
+                    throw new Error(`${notPublished}, nothing was published`);
+                }
+            }
+        }
+        const annotations = await (0, annotations_1.collectAnnotations)(buildFile, Object.keys(local.files));
+        const { version, proposed } = await publishBundle(serverName, server, token, local, annotations).catch((e) => {
+            throw e instanceof server_client_1.ServerError && e.status === 403 ? new Error(`${notPublished}, ${e.message}`) : e;
+        });
+        if (version.hash !== definition) {
+            throw new Error(`${serverName} published ${(0, server_client_1.printable)(version.hash)}, not ${definition} as computed here, nothing was started`);
+        }
+        if (proposed) {
+            throw new Error(`${definition} is only proposed on ${serverName}, nothing was started: ` +
+                'run this again once somebody who may publish approves it');
+        }
+        return (0, sandbox_client_1.submitRun)(connection, request);
+    };
+    remote
+        .command('publish [server]')
+        .description('publish the build file and the files it includes to a server, runs refer to it by its hash')
+        .addOption(new commander_1.Option('--annotation <key=value>', 'say more about where it comes from, or leave one out with key=; path, repo, commit, machine and user are filled in').argParser(collectOption))
+        .addOption(new commander_1.Option('--dry-run', "check it against the server's policy and say what publishing would do, without publishing; builds: run is enough"))
+        .action(guarded('Remote publish was not successful', async (name, options) => {
+        const bundle = await (0, collect_bundle_1.collectBundle)(buildFile, environment);
+        if (options.dryRun) {
+            const { name: serverName, server, token } = await connect(name);
+            const answer = await (0, server_client_1.callServer)(server, token, 'POST', '/v1/definitions/plan', bundle).catch((e) => {
+                throw e instanceof server_client_1.ServerError && e.status === 404
+                    ? new Error(`${serverName} can't check a build file without publishing it, it needs hammerkit 1.27`)
+                    : e;
+            });
+            const planned = answer.body;
+            const files = Object.keys(bundle.files).length;
+            environment.stdout.write(`checked with the policy of ${serverName}: ${(0, server_client_1.printable)(planned.hash)} (${files} ${files === 1 ? 'file' : 'files'}), nothing was published\n`);
+            const names = [...(planned.tasks ?? []), ...(planned.services ?? [])];
+            environment.stdout.write(`tasks and services: ${names.map(server_client_1.printable).join(', ') || 'none'}\n`);
+            environment.stdout.write(`${describeOutcome(planned)}\n`);
+            return;
+        }
+        const annotations = (0, annotations_1.applyAnnotations)(await (0, annotations_1.collectAnnotations)(buildFile, Object.keys(bundle.files)), options.annotation ?? []);
+        const { name: serverName, server, token } = await connect(name);
+        await publishBundle(serverName, server, token, bundle, annotations);
     }));
     remote
         .command('run <tasks...>')
@@ -139,28 +347,18 @@ function registerWorkCommands(remote, context) {
         .addOption(new commander_1.Option('--commit <sha>', 'the commit the ref has to be at, else the run is refused; the commit checked out here when --ref is left out'))
         .addOption(new commander_1.Option('--env <name>', 'the environment of the build file, as --env of run'))
         .addOption(new commander_1.Option('--cache <method>', 'as --cache of run').choices(['checksum', 'modify-date', 'none']))
-        .addOption(new commander_1.Option('--timeout <duration>', 'stop the run after this long, such as 30m'))
+        .addOption(new commander_1.Option('--timeout <duration>', 'as --timeout of run: for tasks without their own, such as 30m'))
+        .addOption(new commander_1.Option('--max-timeout <duration>', 'as --max-timeout of run: for every task, also one with its own timeout'))
+        .addOption(new commander_1.Option('--detach', 'submit the run and exit, without following it: for a check that CI verifies later'))
+        .addOption(new commander_1.Option('--publish', 'publish the build file here first when the server does not have it, without asking; for CI and scripts'))
         .action(guarded('Remote run was not successful', async (tasks, options) => {
-        const named = options.build ?? environment.processEnvs.HAMMERKIT_BUILD;
-        const definition = named ?? (0, bundle_1.hashBundle)(await (0, collect_bundle_1.collectBundle)(buildFile, environment));
-        if (!bundle_1.HASH_PATTERN.test(definition)) {
-            throw new Error(`${(0, server_client_1.printable)(definition)} is not the hash of a published version (sha256:...)`);
-        }
-        const checkout = options.repo && options.ref ? {} : await (0, annotations_1.readCheckout)((0, path_1.dirname)(buildFile));
-        const repo = options.repo ?? checkout.repo;
-        const ref = options.ref ?? checkout.branch;
-        // the commit checked out here is the code meant, when the ref is the branch it is on
-        const commit = options.commit ?? (options.ref ? undefined : checkout.commit);
-        if (!repo) {
-            throw new Error('name the repository with --repo, the checkout here has no origin it can be told by');
-        }
-        if (!ref) {
-            throw new Error('name the branch or tag with --ref, the checkout here is on no branch');
-        }
+        const { definition, local } = await runDefinition(options);
+        const source = await refSource(options);
+        const ref = source.ref;
         const { name: serverName, server } = await getServer(options.server);
         const connection = { url: server.url, ...(0, token_source_1.getTokenSource)(environment, serverName, server) };
         const runOptions = {};
-        for (const key of ['env', 'cache', 'timeout']) {
+        for (const key of ['env', 'cache', 'timeout', 'maxTimeout']) {
             if (options[key] !== undefined) {
                 runOptions[key] = options[key];
             }
@@ -168,23 +366,13 @@ function registerWorkCommands(remote, context) {
         if (passthrough.length > 0) {
             runOptions.args = passthrough;
         }
-        let submitted;
-        try {
-            submitted = await (0, sandbox_client_1.submitRun)(connection, {
-                definition,
-                tasks,
-                source: { type: 'ref', repo, ref, ...(commit ? { commit } : {}) },
-                options: runOptions,
-            });
-        }
-        catch (e) {
-            if (e.status === 400 && !named && /no published build file/.test(String(e))) {
-                throw new Error(`the build file here is not published on ${serverName} (${definition}), publish it first`);
-            }
-            throw e;
-        }
+        const submitted = await submitPublishing({ serverName, server, connection }, { definition, tasks, source, options: runOptions }, local, !!options.publish);
         const at = submitted.source.type === 'ref' ? ` at ${(0, server_client_1.printable)(submitted.source.commit)}` : '';
         environment.stdout.write(`${(0, server_client_1.printable)(submitted.id)} on ${serverName}: ${tasks.map(server_client_1.printable).join(', ')} from ${(0, server_client_1.printable)(ref)}${at}\n`);
+        if (options.detach) {
+            environment.stdout.write(`follow it with: hammerkit remote logs ${(0, server_client_1.printable)(submitted.id)} --follow\n`);
+            return;
+        }
         const prefixed = tasks.length > 1;
         const followed = await (0, sandbox_client_1.followRun)(connection, submitted, {
             signal: environment.abortCtrl.signal,
@@ -208,7 +396,336 @@ function registerWorkCommands(remote, context) {
             .concat(Object.values(run.results ?? {}))
             .find((result) => result?.status === 'failed');
         const exitCode = failed?.exitCode && failed.exitCode > 0 ? failed.exitCode : 1;
-        fail(`${(0, server_client_1.printable)(run.id)} ${run.state}${run.error ? `: ${(0, server_client_1.printable)(run.error)}` : ''}`, exitCode);
+        const [said, ...why] = describeRunEnd(run);
+        for (const line of why) {
+            environment.stdout.write(`${line}\n`);
+        }
+        environment.stdout.write(`read its log again with: hammerkit remote logs ${(0, server_client_1.printable)(run.id)}\n`);
+        fail(said, exitCode);
+    }));
+    remote
+        .command('up <services...>')
+        .description('start services of a published build file on a server from a git ref and keep them up until their ttl ends, ' +
+        'with their endpoints at addresses of their own behind the sign-in of the server')
+        .addOption(new commander_1.Option('--server <name>', 'the server to run on; the default of remote use when left out'))
+        .addOption(new commander_1.Option('--endpoint <service:port>', 'publish this port of a service, such as web:8080; may be given more than once').argParser(collectOption))
+        .addOption(new commander_1.Option('--ttl <duration>', 'how long they stay once ready, such as 2h, then they are removed; the server default when left out'))
+        .addOption(new commander_1.Option('--sleep-ttl <duration>', 'how long they stay up without a request, such as 30m, then they sleep until the next request wakes them; ' +
+        'the server default when left out'))
+        .addOption(new commander_1.Option('--name <name>', 'a later remote up of yours with the same name moves this run to its commit, or replaces it when it ' +
+        'starts other services or another build file; the addresses stay; the branch when left out'))
+        .addOption(new commander_1.Option('--build <hash>', 'the published version to run (sha256:...), or HAMMERKIT_BUILD; the hash of the build file here when left out'))
+        .addOption(new commander_1.Option('--repo <host/path>', 'the repository; origin of the checkout here when left out'))
+        .addOption(new commander_1.Option('--ref <ref>', 'the branch or tag to start; the branch checked out here when left out'))
+        .addOption(new commander_1.Option('--commit <sha>', 'the commit the ref has to be at; the commit checked out here when --ref is left out'))
+        .addOption(new commander_1.Option('--env <name>', 'the environment of the build file, as --env of run'))
+        .addOption(new commander_1.Option('--publish', 'publish the build file here first when the server does not have it, without asking; for CI and scripts'))
+        .action(guarded('Remote up was not successful', async (services, options) => {
+        const { definition, local } = await runDefinition(options);
+        const endpoints = (options.endpoint ?? []).map(service_tools_1.parseEndpoint);
+        const source = await refSource(options);
+        const { name: serverName, server } = await getServer(options.server);
+        const connection = { url: server.url, ...(0, token_source_1.getTokenSource)(environment, serverName, server) };
+        const submitted = await submitPublishing({ serverName, server, connection }, {
+            definition,
+            services,
+            endpoints,
+            ...(options.ttl ? { ttl: options.ttl } : {}),
+            ...(options.sleepTtl ? { sleepTtl: options.sleepTtl } : {}),
+            ...(options.name ? { name: options.name } : {}),
+            source,
+            options: options.env ? { env: options.env } : {},
+        }, local, !!options.publish);
+        if (submitted.moved) {
+            // a run of this name runs the same: it moves to the commit, with its id, addresses and volumes
+            environment.stdout.write(`${(0, server_client_1.printable)(submitted.id)} on ${serverName} (${(0, server_client_1.printable)(submitted.state)}) moves to ${(0, server_client_1.printable)(source.ref)} at ${(0, server_client_1.printable)(submitted.moving?.commit ?? (submitted.source.type === 'ref' ? submitted.source.commit : '-'))}\n`);
+        }
+        else {
+            environment.stdout.write(`${(0, server_client_1.printable)(submitted.id)} on ${serverName}: ${services.map(server_client_1.printable).join(', ')} from ${(0, server_client_1.printable)(source.ref)}${submitted.source.type === 'ref' ? ` at ${(0, server_client_1.printable)(submitted.source.commit)}` : ''}\n`);
+        }
+        const followed = await (0, sandbox_client_1.followRun)(connection, submitted, {
+            signal: environment.abortCtrl.signal,
+            untilUp: true,
+            onLine: (line) => environment.stdout.write(`[${(0, server_client_1.printable)(line.task)}] ${printableLine(line.text)}\n`),
+            onWaiting: (run) => {
+                for (const approval of run.approvals ?? []) {
+                    environment.stdout.write(`waiting until somebody who uses ${(0, server_client_1.printable)(approval.account)} approves it, before ${(0, server_client_1.printable)(approval.expiresAt)}: hammerkit remote approve ${(0, server_client_1.printable)(run.id)}\n`);
+                }
+            },
+        });
+        const run = followed.run;
+        if (run.state !== 'up' && !(submitted.moved && run.state === 'asleep')) {
+            const [said, ...why] = describeRunEnd(run);
+            for (const line of why) {
+                environment.stdout.write(`${line}\n`);
+            }
+            environment.stdout.write(`read its log again with: hammerkit remote logs ${(0, server_client_1.printable)(run.id)}\n`);
+            fail(said, 1);
+        }
+        // the outcome of this move, not of one that came after it
+        const target = (0, sandbox_client_1.getMoveTarget)(submitted);
+        const lastMove = [...(run.moves ?? [])].reverse().find((move) => move.commit === target);
+        if (submitted.moved && lastMove?.error) {
+            fail(`${(0, server_client_1.printable)(run.id)} did not move to ${(0, server_client_1.printable)(lastMove.commit)}: ${(0, server_client_1.printable)(lastMove.error)}; it stays at ${(0, server_client_1.printable)(run.source.type === 'ref' ? run.source.commit : '-')}`, 1);
+        }
+        if (run.state === 'asleep') {
+            environment.stdout.write(`${(0, server_client_1.printable)(run.id)} sleeps, at ${(0, server_client_1.printable)(run.source.type === 'ref' ? run.source.commit : '-')} when it wakes, until ${(0, server_client_1.printable)(run.expiresAt ?? '-')}\n`);
+            return;
+        }
+        environment.stdout.write(`${(0, server_client_1.printable)(run.id)} is up until ${(0, server_client_1.printable)(run.expiresAt ?? '-')}${run.up?.sleepTtl ? `, asleep after ${(0, units_1.formatDuration)(run.up.sleepTtl)} without a request` : ''}\n`);
+        for (const endpoint of run.up?.endpoints ?? []) {
+            environment.stdout.write(`  ${(0, server_client_1.printable)(endpoint.service)}:${endpoint.port}  ${(0, server_client_1.printable)(endpoint.url)}\n`);
+        }
+        environment.stdout.write(`extend it with: hammerkit remote extend ${(0, server_client_1.printable)(run.id)} 8h, stop it with: hammerkit remote stop ${(0, server_client_1.printable)(run.id)}\n`);
+    }));
+    remote
+        .command('status [server]')
+        .description('show how a server stands for you: the places that run, the queue and why each run in it waits, your limits and the runs that count against them, your service runs')
+        .addOption(new commander_1.Option('--json', 'print the overview as the server answers it, for scripts and agents'))
+        .action(guarded('Remote status was not successful', async (name, options) => {
+        const { name: serverName, server, token } = await connect(name);
+        const overview = (await (0, server_client_1.callServer)(server, token, 'GET', '/v1/overview')).body;
+        if (options.json) {
+            environment.stdout.write(`${JSON.stringify(overview, null, 2)}\n`);
+            return;
+        }
+        for (const line of (0, remote_status_1.describeOverview)(serverName, overview)) {
+            environment.stdout.write(`${line}\n`);
+        }
+        // a server from before /v1/capacity, or one that can not read it, prints without
+        const capacity = await (0, server_client_1.callServer)(server, token, 'GET', '/v1/capacity').catch(() => undefined);
+        if (capacity) {
+            for (const line of (0, remote_status_1.describeCapacity)(capacity.body, true)) {
+                environment.stdout.write(`${line}\n`);
+            }
+        }
+    }));
+    remote
+        .command('capacity [server]')
+        .description('show what the machines of a server have and what its runs take: CPU and memory requested and in use, per namespace on Kubernetes')
+        .addOption(new commander_1.Option('--json', 'print it as the server answers it'))
+        .action(guarded('Remote capacity was not successful', async (name, options) => {
+        const { server, token } = await connect(name);
+        const capacity = (await (0, server_client_1.callServer)(server, token, 'GET', '/v1/capacity')).body;
+        if (options.json) {
+            environment.stdout.write(`${JSON.stringify(capacity, null, 2)}\n`);
+            return;
+        }
+        for (const line of (0, remote_status_1.describeCapacity)(capacity)) {
+            environment.stdout.write(`${line}\n`);
+        }
+    }));
+    remote
+        .command('tasks [server]')
+        .description('show what the past runs of each task took, over the runs the server keeps: how often it ran, failed and came from the cache, the median and 90th percentile of its time, and how long its runs waited for a place')
+        .addOption(new commander_1.Option('--mine', 'only your runs'))
+        .addOption(new commander_1.Option('--json', 'print it as the server answers it'))
+        .action(guarded('Remote tasks was not successful', async (name, options) => {
+        const { name: serverName, server, token } = await connect(name);
+        const { tasks } = (await (0, server_client_1.callServer)(server, token, 'GET', `/v1/tasks?scope=${options.mine ? 'mine' : 'visible'}`)).body;
+        if (options.json) {
+            environment.stdout.write(`${JSON.stringify(tasks, null, 2)}\n`);
+            return;
+        }
+        if (tasks.length === 0) {
+            environment.stdout.write(`${serverName}: no task ran in the runs it keeps\n`);
+            return;
+        }
+        for (const line of (0, remote_status_1.describeTaskStats)(tasks)) {
+            environment.stdout.write(`${line}\n`);
+        }
+    }));
+    remote
+        .command('dashboard [server]')
+        .description('print the address of the dashboard of a server, the pages of remote status, services, runs and logs')
+        .action(guarded('Remote dashboard was not successful', async (name) => {
+        const { name: serverName, server } = await getServer(name);
+        const discovery = await (0, server_client_1.getDiscovery)(server);
+        if (!discovery.dashboard) {
+            throw new Error(`${serverName} has no dashboard: it needs endpoints with a login in its config, see https://hammerkit.dev/docs/remote-compute/status#dashboard`);
+        }
+        environment.stdout.write(`${(0, server_client_1.printable)(discovery.dashboard)}\n`);
+    }));
+    remote
+        .command('services [server]')
+        .description('list the service runs that are starting, up or asleep: yours, and those whose endpoints you may open, with their addresses and when they end or sleep')
+        .addOption(new commander_1.Option('--ended', 'also the ones that ended, as long as the server keeps them'))
+        .action(guarded('Remote services was not successful', async (name, options) => {
+        const { name: serverName, server, token } = await connect(name);
+        const open = ['awaiting-approval', 'queued', 'running', 'up', 'asleep'];
+        const states = options.ended ? '' : `&state=${open.join(',')}`;
+        const runs = keepRuns((await (0, server_client_1.callServer)(server, token, 'GET', `/v1/runs?scope=visible&kind=services${states}`)).body.runs, options.ended ? undefined : open, 'services');
+        if (runs.length === 0) {
+            environment.stdout.write(`${serverName}: no service runs${options.ended ? '' : ' are starting, up or asleep'}\n`);
+            return;
+        }
+        for (const run of runs) {
+            for (const line of (0, remote_status_1.describeServiceRun)(run)) {
+                environment.stdout.write(`${line}\n`);
+            }
+        }
+    }));
+    remote
+        .command('runs [server]')
+        .description('list your runs on a server, newest first: their state, where a queued one stands, and why one failed')
+        .addOption(new commander_1.Option('--limit <count>', 'show at most this many').default('20'))
+        .addOption(new commander_1.Option('--state <states>', 'only runs in these states, comma separated: awaiting-approval, queued, running, up, asleep, succeeded, failed, cancelled, rejected'))
+        .addOption(new commander_1.Option('--services', 'only service runs').conflicts('tasks'))
+        .addOption(new commander_1.Option('--tasks', 'only runs of tasks'))
+        .addOption(new commander_1.Option('--all', 'also the runs of others you may see: those you may approve, service runs you may open, every run with observe'))
+        .action(guarded('Remote runs was not successful', async (name, options) => {
+        const limit = Number(options.limit);
+        if (!Number.isInteger(limit) || limit < 1) {
+            throw new Error('--limit is a number of runs, 1 or more');
+        }
+        const query = new URLSearchParams();
+        if (options.all) {
+            query.set('scope', 'visible');
+        }
+        if (options.state) {
+            query.set('state', options.state);
+        }
+        if (options.services || options.tasks) {
+            query.set('kind', options.services ? 'services' : 'tasks');
+        }
+        const { name: serverName, server, token } = await connect(name);
+        const path = `/v1/runs${query.toString() ? `?${query}` : ''}`;
+        const runs = keepRuns((await (0, server_client_1.callServer)(server, token, 'GET', path)).body.runs, options.state ? String(options.state).split(',') : undefined, options.services ? 'services' : options.tasks ? 'tasks' : undefined);
+        if (runs.length === 0) {
+            environment.stdout.write(`${serverName}: ${query.toString() ? 'no runs it keeps are like that' : 'you have no runs it still keeps'}\n`);
+            return;
+        }
+        for (const run of runs.slice(0, limit)) {
+            environment.stdout.write(`${describeListedRun(run)}\n`);
+        }
+        if (runs.length > limit) {
+            environment.stdout.write(`… and ${runs.length - limit} older, see them with --limit ${runs.length}\n`);
+        }
+    }));
+    remote
+        .command('logs <run> [server]')
+        .description('print the log of a run, also of one that ended: what its tasks and services printed, and what the server and the cluster said')
+        .addOption(new commander_1.Option('--task <name>', 'only the lines of this task or service'))
+        .addOption(new commander_1.Option('--tail <lines>', 'only the last lines, at most 1000').argParser((value) => {
+        if (!/^\d+$/.test(value) || Number(value) < 1 || Number(value) > 1000) {
+            throw new commander_1.InvalidArgumentError('a number of lines from 1 to 1000');
+        }
+        return Number(value);
+    }))
+        .addOption(new commander_1.Option('-f, --follow', 'keep printing what comes until the run ends; ctrl-c stops following, not the run'))
+        .action(guarded('Remote logs was not successful', async (id, name, options) => {
+        if (!RUN_ID.test(id)) {
+            throw new Error(`${(0, server_client_1.printable)(id)} is not a run id (run_...), list them with: hammerkit remote runs`);
+        }
+        if (options.follow && options.tail !== undefined) {
+            throw new Error('--follow prints the whole log, leave --tail out');
+        }
+        const { name: serverName, server } = await getServer(name);
+        const connection = { url: server.url, ...(0, token_source_1.getTokenSource)(environment, serverName, server) };
+        const found = await (0, sandbox_client_1.getRun)(connection, id);
+        if (!found) {
+            throw new Error(`${serverName} keeps no run ${(0, server_client_1.printable)(id)} you may see, list them with: hammerkit remote runs`);
+        }
+        let run = found;
+        const task = options.task;
+        const print = (line) => environment.stdout.write(`${task ? '' : `[${(0, server_client_1.printable)(line.task)}] `}${printableLine(line.text)}\n`);
+        if (options.follow) {
+            run = (await (0, sandbox_client_1.followRun)(connection, run, {
+                task,
+                signal: environment.abortCtrl.signal,
+                detach: true,
+                onLine: print,
+            })).run;
+        }
+        else {
+            await (0, sandbox_client_1.readRunLog)(connection, run, { task, tail: options.tail }, print);
+        }
+        if (run.state === 'succeeded' || run.state === 'up' || run.state === 'asleep') {
+            environment.stdout.write(`${(0, server_client_1.printable)(run.id)} ${(0, server_client_1.printable)(run.state)}\n`);
+        }
+        else if (sandbox_client_1.FINAL_RUN_STATES.includes(run.state)) {
+            for (const line of describeRunEnd(run)) {
+                environment.stdout.write(`${line}\n`);
+            }
+        }
+        else {
+            environment.stdout.write(`${(0, server_client_1.printable)(run.id)} is ${(0, server_client_1.printable)(run.state)}, follow it with --follow\n`);
+        }
+    }));
+    remote
+        .command('extend <run> <ttl> [server]')
+        .description('keep a service run that is up for this long from now, such as 8h, at most the maxTtl of the server')
+        .action(guarded('Remote extend was not successful', async (id, ttl, name) => {
+        if (!RUN_ID.test(id)) {
+            throw new Error(`${(0, server_client_1.printable)(id)} is not a run id (run_...)`);
+        }
+        const { name: serverName, server, token } = await connect(name);
+        const run = (await (0, server_client_1.callServer)(server, token, 'POST', `/v1/runs/${id}/extend`, { ttl })).body;
+        environment.stdout.write(`${(0, server_client_1.printable)(run.id)} on ${serverName} stays up until ${(0, server_client_1.printable)(run.expiresAt ?? '-')}\n`);
+    }));
+    remote
+        .command('stop <run> [server]')
+        .description('stop a run: a service run takes its services and endpoints down')
+        .action(guarded('Remote stop was not successful', async (id, name) => {
+        if (!RUN_ID.test(id)) {
+            throw new Error(`${(0, server_client_1.printable)(id)} is not a run id (run_...)`);
+        }
+        const { name: serverName, server, token } = await connect(name);
+        const run = (await (0, server_client_1.callServer)(server, token, 'POST', `/v1/runs/${id}/cancel`)).body;
+        environment.stdout.write(`${(0, server_client_1.printable)(run.id)} on ${serverName}: ${run.state === 'up' || run.state === 'running' ? 'stopping' : (0, server_client_1.printable)(run.state)}\n`);
+    }));
+    remote
+        .command('verify <tasks...>')
+        .description('check that the server already passed tasks for the files checked out here, without running them; ' +
+        'exits 1 at once for any it did not')
+        .addOption(new commander_1.Option('--server <name>', 'the server to ask; the default of remote use when left out'))
+        .addOption(new commander_1.Option('--repo <host/path>', 'the repository, such as github.com/corp/payments; origin of the checkout here when left out'))
+        .addOption(new commander_1.Option('--env <name>', 'the environment of the build file, as --env of run'))
+        .action(guarded('Remote verify was not successful', async (tasks, options) => {
+        const repo = options.repo ?? (await (0, annotations_1.readCheckout)((0, path_1.dirname)(buildFile))).repo;
+        if (!repo) {
+            throw new Error('name the repository with --repo, the checkout here has no origin it can be told by');
+        }
+        const keys = [];
+        const unkeyed = [];
+        for (const task of tasks) {
+            const found = (await (0, check_keys_1.getCheckKeys)(buildFile, environment, { taskName: task, environmentName: options.env ?? null }))[task];
+            if (!found) {
+                throw new Error(`the build file has no task ${(0, server_client_1.printable)(task)}`);
+            }
+            if (found.key === null) {
+                unkeyed.push({ task, reason: found.reason });
+            }
+            else {
+                keys.push({ task, key: found.key });
+            }
+        }
+        const { name: serverName, server, token } = await connect(options.server);
+        const answer = keys.length > 0
+            ? await (0, server_client_1.callServer)(server, token, 'POST', '/v1/checks/lookup', { repo, keys })
+            : { body: { checks: [] } };
+        const missing = [];
+        // by what was asked, never by what was answered: a task the answer leaves out, or
+        // answers for another key, is not verified
+        for (const { task, key } of keys) {
+            const check = answer.body.checks.find((answered) => answered.task === task && answered.key === key);
+            if (check?.passed) {
+                environment.stdout.write(`${(0, server_client_1.printable)(task)}: ${(0, server_client_1.printable)(describePassed(check.passed))}\n`);
+            }
+            else {
+                missing.push(task);
+                environment.stdout.write(`${(0, server_client_1.printable)(task)}: no run on ${serverName} passed it for these files${check?.reason ? `, ${(0, server_client_1.printable)(check.reason)}` : ''}\n`);
+            }
+        }
+        for (const { task, reason } of unkeyed) {
+            environment.stdout.write(`${(0, server_client_1.printable)(task)}: can not be verified, ${(0, server_client_1.printable)(reason)}\n`);
+        }
+        if (missing.length > 0 || unkeyed.length > 0) {
+            const run = missing.length > 0 ? `; run on the server: hammerkit remote run ${missing.map(server_client_1.printable).join(' ')}` : '';
+            // whole names: a task of an include is named like build:all
+            const names = [...missing, ...unkeyed.map(({ task }) => task)].map(server_client_1.printable);
+            fail(`not verified: ${names.join(', ')}${run}`, 1);
+        }
     }));
     remote
         .command('versions [server]')