hammerkit 2.0.0-next.5 → 2.0.0-next.6

package/dist/server/capacity.js

@@ -4,6 +4,9 @@ exports.formatMemory = exports.formatCpu = exports.HostCapacity = exports.Kubern
 const os_1 = require("os");
 const log_1 = require("../log");
 const apply_1 = require("../kubernetes/apply");
+const job_placement_1 = require("../kubernetes/job-placement");
+const volumes_1 = require("../kubernetes/volumes");
+const network_policy_1 = require("../kubernetes/network-policy");
 const UNITS = {
     n: 1e-9,
     u: 1e-6,
@@ -90,6 +93,17 @@ function why(what, e, rule) {
     }
     return `${what}: ${(0, log_1.getErrorMessage)(e)}`;
 }
+// `why`, in a few words
+function briefly(e, what) {
+    const code = (0, apply_1.statusCodeOf)(e);
+    if (code === 403) {
+        return what === 'nodes' ? 'may not list nodes' : 'may not read metrics';
+    }
+    if (code === 404 && what === 'metrics') {
+        return 'no metrics API';
+    }
+    return what === 'nodes' ? 'nodes not readable' : 'metrics not readable';
+}
 const ENDED = ['Succeeded', 'Failed'];
 // On Kubernetes: per namespace of the server, its pods' requests, its quota and its usage.
 class KubernetesCapacity {
@@ -107,6 +121,37 @@ class KubernetesCapacity {
         catch (e) {
             problems.push(why('the namespaces of the server', e));
         }
+        const unknown = {};
+        // the runs by the value of their label, and what the pods of each run or sandbox take
+        const byLabel = new Map(runs.map((run) => [(0, job_placement_1.getRunLabelValue)(run.id), run]));
+        const uses = new Map();
+        const useOf = (namespace, labels, owner) => {
+            const run = byLabel.get(labels?.[job_placement_1.RUN_LABEL] ?? '');
+            const sandbox = labels?.[volumes_1.SANDBOX_LABEL] ?? labels?.[network_policy_1.PROXY_OF_SANDBOX_LABEL];
+            const key = run ? `run ${run.id}` : sandbox ? `sandbox ${sandbox}` : `other ${namespace}`;
+            let use = uses.get(key);
+            if (!use) {
+                use = run
+                    ? {
+                        kind: 'run',
+                        run: run.id,
+                        tasks: run.tasks,
+                        ...(run.up ? { up: { name: run.up.name, services: run.up.services } } : {}),
+                        repo: run.source.type === 'upload' ? 'an upload' : run.source.repo,
+                        owner: { iss: run.owner.iss, sub: run.owner.sub, ...(run.owner.name ? { name: run.owner.name } : {}) },
+                        pods: 0,
+                        requested: { cpu: 0, memory: 0 },
+                    }
+                    : {
+                        kind: sandbox ? 'sandbox' : 'other',
+                        ...(owner ? { owner } : {}),
+                        pods: 0,
+                        requested: { cpu: 0, memory: 0 },
+                    };
+                uses.set(key, use);
+            }
+            return use;
+        };
         const said = new Set();
         const once = (text) => {
             if (!said.has(text)) {
@@ -123,6 +168,8 @@ class KubernetesCapacity {
                 // a namespace whose pods could be listed and that has none asks for nothing
                 requested: { cpu: 0, memory: 0 },
             };
+            // the use of each pod that runs, by its name
+            const usesOfPods = new Map();
             try {
                 const pods = await this.instance.coreApi.listNamespacedPod({ namespace });
                 for (const pod of pods.items) {
@@ -130,9 +177,13 @@ class KubernetesCapacity {
                         continue;
                     }
                     group.pods++;
+                    const use = useOf(namespace, pod.metadata?.labels, group.owner);
+                    usesOfPods.set(pod.metadata?.name ?? '', use);
+                    use.pods++;
                     for (const container of pod.spec?.containers ?? []) {
                         const requests = container.resources?.requests ?? {};
                         add(group.requested, parseQuantity(requests.cpu) ?? 0, parseQuantity(requests.memory) ?? 0);
+                        add(use.requested, parseQuantity(requests.cpu) ?? 0, parseQuantity(requests.memory) ?? 0);
                     }
                 }
             }
@@ -162,13 +213,19 @@ class KubernetesCapacity {
                     const metrics = await this.instance.metrics.getPodMetrics(namespace);
                     const used = { cpu: 0, memory: 0 };
                     for (const pod of metrics.items) {
+                        const use = usesOfPods.get(pod.metadata?.name ?? '');
                         for (const container of pod.containers) {
                             add(used, parseQuantity(container.usage.cpu), parseQuantity(container.usage.memory));
+                            if (use) {
+                                use.used ?? (use.used = { cpu: 0, memory: 0 });
+                                add(use.used, parseQuantity(container.usage.cpu), parseQuantity(container.usage.memory));
+                            }
                         }
                     }
                     group.used = used;
                 }
                 catch (e) {
+                    unknown.used ?? (unknown.used = briefly(e, 'metrics'));
                     once(why('what the pods use (metrics)', e, 'list on pods.metrics.k8s.io in them (runRole, see the backend docs)'));
                 }
             }
@@ -204,11 +261,13 @@ class KubernetesCapacity {
                     machines.used = nodesUsed;
                 }
                 catch (e) {
+                    unknown.used ?? (unknown.used = briefly(e, 'metrics'));
                     once(why('what the nodes use (metrics)', e, 'list on nodes.metrics.k8s.io (see the backend docs)'));
                 }
             }
         }
         catch (e) {
+            unknown.total = briefly(e, 'nodes');
             // listing nodes is optional, and many clusters do not let a workload do it
             if ((0, apply_1.statusCodeOf)(e) === 403) {
                 once('the nodes: not shown, the server may not list nodes (optional, see the backend docs)');
@@ -223,7 +282,11 @@ class KubernetesCapacity {
             ...(machines ? { machines } : {}),
             runs: total,
             groups,
+            uses: [...uses.values()].sort((a, b) => (b.used?.cpu ?? 0) - (a.used?.cpu ?? 0) ||
+                (b.requested.cpu ?? 0) - (a.requested.cpu ?? 0) ||
+                (b.requested.memory ?? 0) - (a.requested.memory ?? 0)),
             problems,
+            ...(Object.keys(unknown).length > 0 ? { unknown } : {}),
         };
     }
 }

package/dist/server/dashboard.js

@@ -64,8 +64,8 @@ const TYPES = {
 };
 async function handleDashboard(context, principal, req, res, url) {
     const now = context.now ?? Date.now;
-    const grant = (0, policy_1.getGrant)(context.config.entitlements, principal);
-    const viewer = { principal, grant, activity: context.config.runs.activity === 'everyone' };
+    const viewer = (0, overview_1.getViewer)(context.config, principal);
+    const { grant } = viewer;
     const pathname = url.pathname;
     for (const [name, value] of Object.entries(exports.HEADERS)) {
         res.setHeader(name, value);

package/dist/server/executor.js

@@ -220,6 +220,7 @@ class HostExecutor {
             let claim;
             // the files of the sandbox the tasks work on
             let work;
+            const sourceStarted = new Date();
             if (source.type === 'sandbox' && source.check) {
                 // a check works on the files a push of the sandbox commits, copied out of it: never
                 // on the sandbox, whose ignored files (node_modules, outputs) its owner writes
@@ -303,12 +304,17 @@ class HostExecutor {
                     sandboxNetwork: network,
                 };
             }
+            emitPhase(context, 'source', sourceStarted);
             const home = (0, path_1.join)(root, 'home');
             await (0, promises_1.mkdir)(home, { recursive: true });
             const cache = (0, path_1.join)(this.options.cache, cacheNamespace(run));
             await (0, promises_1.mkdir)(cache, { recursive: true });
             const placement = this.options.placement;
+            const namespaceStarted = new Date();
             const placed = placement ? await placement.prepare(run, context, { root: work ?? cwd, claim }) : undefined;
+            if (placed) {
+                emitPhase(context, 'namespace', namespaceStarted);
+            }
             try {
                 await this.runInProcess(worker, {
                     ...input,
@@ -504,4 +510,8 @@ class HostExecutor {
     }
 }
 exports.HostExecutor = HostExecutor;
+// A step of a run before its tasks, for its timeline (RunPhase)
+function emitPhase(context, phase, startedAt) {
+    context.emit({ type: 'phase', phase, startedAt: startedAt.toISOString(), finishedAt: new Date().toISOString() });
+}
 //# sourceMappingURL=executor.js.map

package/dist/server/overview.js

@@ -13,6 +13,7 @@ function getViewer(config, principal) {
         principal,
         grant: (0, policy_1.getGrant)(config.entitlements, principal),
         activity: config.runs.activity === 'everyone',
+        nameOf: (owner) => (0, policy_1.subjectName)(config.entitlements, owner),
     };
 }
 exports.getViewer = getViewer;
@@ -91,7 +92,8 @@ function mayRead(viewer, run) {
     return owns(viewer, run) || viewer.grant.observe || decides(viewer, run);
 }
 exports.mayRead = mayRead;
-function toListedRun(viewer, run, queue) {
+// `entryOf` names where in its repository the build file of a version is.
+function toListedRun(viewer, run, queue, entryOf) {
     const { owner, results, plan, options, up, error, ...rest } = run;
     const mine = owns(viewer, run);
     const may = runActions(viewer, run);
@@ -106,11 +108,11 @@ function toListedRun(viewer, run, queue) {
             ...(result.cache ? { cached: result.cache.hit } : {}),
         },
     ]));
+    const entry = entryOf?.(run.definition);
+    const name = owner.name ?? viewer.nameOf?.(owner);
     return {
         ...rest,
-        ...(mine
-            ? { plan, options }
-            : { owner: { iss: owner.iss, sub: owner.sub, ...(owner.name ? { name: owner.name } : {}) } }),
+        ...(mine ? { plan, options } : { owner: { iss: owner.iss, sub: owner.sub, ...(name ? { name } : {}) } }),
         ...(up
             ? {
                 up: {
@@ -124,6 +126,7 @@ function toListedRun(viewer, run, queue) {
         ...(queue ? { queue } : {}),
         ...(may ? { may } : {}),
         ...(!mine && mayRead(viewer, run) ? { readable: true } : {}),
+        ...(entry ? { entry } : {}),
     };
 }
 exports.toListedRun = toListedRun;
@@ -149,7 +152,8 @@ function queuePlace(registry, viewer, run) {
     return { position: queue.position, waitsFor: visibleReason(queue.waitsFor, visible) };
 }
 exports.queuePlace = queuePlace;
-function getOverview(snapshot, usage, viewer) {
+function getOverview(snapshot, usage, viewer, entryOf) {
+    const list = (run, place) => toListedRun(viewer, run, place, entryOf);
     const runs = new Map();
     for (const run of [
         ...snapshot.running,
@@ -168,7 +172,7 @@ function getOverview(snapshot, usage, viewer) {
         return {
             ...place,
             mine: owns(viewer, entry.run),
-            ...(maySee(viewer, entry.run) ? { run: toListedRun(viewer, entry.run, place) } : {}),
+            ...(maySee(viewer, entry.run) ? { run: list(entry.run, place) } : {}),
         };
     });
     const running = snapshot.running.filter((run) => maySee(viewer, run));
@@ -178,19 +182,19 @@ function getOverview(snapshot, usage, viewer) {
         places: {
             max: snapshot.places.max,
             used: snapshot.places.used,
-            runs: running.map((run) => toListedRun(viewer, run)),
+            runs: running.map((run) => list(run)),
             others: snapshot.running.length - running.length,
         },
         queue,
         approvals: {
             waiting: snapshot.awaiting.runs.length,
             max: snapshot.awaiting.max,
-            runs: snapshot.awaiting.runs.filter((run) => maySee(viewer, run)).map((run) => toListedRun(viewer, run)),
+            runs: snapshot.awaiting.runs.filter((run) => maySee(viewer, run)).map((run) => list(run)),
         },
         services: {
             open: snapshot.services.taken,
             ...(snapshot.services.max !== undefined ? { max: snapshot.services.max } : {}),
-            runs: snapshot.services.runs.filter((run) => maySee(viewer, run)).map((run) => toListedRun(viewer, run)),
+            runs: snapshot.services.runs.filter((run) => maySee(viewer, run)).map((run) => list(run)),
         },
         limits: {
             waiting: meter(usage.waiting, limits?.waiting),
@@ -291,6 +295,35 @@ function visibleCapacity(capacity, viewer) {
         ...capacity,
         // what the cluster answered is for operators; others learn that something is missing
         problems: viewer.grant.observe || capacity.problems.length === 0 ? capacity.problems : ['some of it could not be read'],
+        ...(capacity.unknown
+            ? {
+                unknown: everyone
+                    ? capacity.unknown
+                    : {
+                        ...(capacity.unknown.total ? { total: 'not readable' } : {}),
+                        ...(capacity.unknown.used ? { used: 'not readable' } : {}),
+                    },
+            }
+            : {}),
+        // what others' runs and sandboxes take shows, not what they are
+        ...(capacity.uses
+            ? {
+                uses: capacity.uses.map((use) => {
+                    const mine = use.owner?.iss === viewer.principal.iss && use.owner?.sub === viewer.principal.sub;
+                    const name = use.owner && !use.owner.name ? viewer.nameOf?.(use.owner) : undefined;
+                    return everyone || mine || !use.owner
+                        ? name && use.owner
+                            ? { ...use, owner: { ...use.owner, name } }
+                            : use
+                        : {
+                            kind: use.kind,
+                            pods: use.pods,
+                            requested: use.requested,
+                            ...(use.used ? { used: use.used } : {}),
+                        };
+                }),
+            }
+            : {}),
         groups: capacity.groups.map((group) => {
             const mine = group.owner?.iss === viewer.principal.iss && group.owner?.sub === viewer.principal.sub;
             if (everyone || mine || !group.owner) {

package/dist/server/plan.js

@@ -294,6 +294,13 @@ function describeMountSource(mount) {
     const home = (0, normalize_path_1.getHomeReference)(mount.mount.split(':')[0]);
     return home ? `${mount.localPath} (${home} is the home directory)` : mount.localPath;
 }
+// where the build file of an item is: in the bundle, or in the repository of a git include
+function declaredIn(root, scope) {
+    if (scope.remote) {
+        return `${scope.remote.git} ${(0, path_1.relative)(scope.remote.root, scope.fileName).split(path_1.sep).join('/')}`;
+    }
+    return (0, path_1.relative)(root, scope.fileName).split(path_1.sep).join('/');
+}
 function toPlan(root, item) {
     const data = item.data;
     const secrets = secretsOf(item).map(describeSecret);
@@ -322,6 +329,7 @@ function toPlan(root, item) {
     const isTask = data.type === 'local-task' || data.type === 'container-task';
     return {
         kind: isTask ? 'task' : 'service',
+        file: declaredIn(root, data.scope),
         ...(data.description ? { description: data.description } : {}),
         runtime: (0, work_task_1.isAggregateTask)(data) ? 'none' : data.type === 'local-task' ? 'local' : 'docker',
         ...((0, work_task_1.isContainerWorkTask)(data) || (0, work_service_1.isContainerWorkService)(data) ? { image: data.image } : {}),
@@ -330,13 +338,18 @@ function toPlan(root, item) {
         network: (0, work_network_1.toNetworkSchema)('network' in data ? data.network : { type: 'open' }),
         ...('args' in data && data.args !== null ? { args: data.args } : {}),
         deps: item.deps.map((dep) => dep.name).sort(),
-        needs: item.needs.map((need) => need.name).sort(),
+        // by their names in the plan, not the names the task reaches them at (`postgres` of an include)
+        needs: item.needs.map((need) => need.service.name).sort(),
         accounts,
         secrets,
         serverEnvs: 'envs' in data ? data.envs.processNames ?? [] : [],
         mounts,
         ports,
-        generates: 'generates' in data ? data.generates.map((generate) => generate.path) : [],
+        // where in the bundle, as the build file names them: the directory the server read it in is
+        // its own
+        generates: 'generates' in data
+            ? data.generates.map((generate) => insideBundle(root, data.cwd, generate.path)[0] ?? generate.path)
+            : [],
         ...(capAdd.length > 0 ? { capAdd: [...capAdd] } : {}),
         flags,
     };
@@ -400,7 +413,7 @@ async function planBundle(bundle, environment, options = {}) {
                         ? item.data.mounts.flatMap((mount) => insideBundle(directory, item.data.cwd, mount.localPath))
                         : [],
                     deps: item.deps.map((dep) => dep.name).sort(),
-                    needs: item.needs.map((need) => need.name).sort(),
+                    needs: item.needs.map((need) => need.service.name).sort(),
                 };
             }
             const plan = {};

package/dist/server/policy.js

@@ -1,8 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.decideRun = exports.sandboxRepos = exports.decideSandbox = exports.sandboxPushBranches = exports.sandboxBranchType = exports.endpointLevel = exports.getGrant = exports.usedAccounts = exports.mayYank = exports.canUse = exports.maySeeDefinitions = exports.mayPublish = exports.mayPropose = exports.mayReadChecks = exports.mayRun = exports.matches = void 0;
+exports.decideRun = exports.sandboxRepos = exports.decideSandbox = exports.sandboxPushBranches = exports.sandboxBranchType = exports.endpointLevel = exports.getGrant = exports.usedAccounts = exports.mayYank = exports.canUse = exports.maySeeDefinitions = exports.mayPublish = exports.mayPropose = exports.mayReadChecks = exports.mayRun = exports.matches = exports.subjectName = void 0;
 const repo_1 = require("./repo");
 const BUILDS_ORDER = ['run', 'propose', 'publish'];
+// The name an entitlement gives a subject, for the pages to name its runs by when its tokens
+// name nobody.
+function subjectName(entitlements, who) {
+    return entitlements.find((entitlement) => entitlement.subject?.name && entitlement.subject.iss === who.iss && entitlement.subject.sub === who.sub)?.subject?.name;
+}
+exports.subjectName = subjectName;
 function matches(entitlement, principal) {
     if (entitlement.subject) {
         return entitlement.subject.iss === principal.iss && entitlement.subject.sub === principal.sub;
@@ -34,9 +40,9 @@ const mayPublish = (grant) => atLeast(grant, 'publish');
 exports.mayPublish = mayPublish;
 // A caller who can neither run nor publish anything sees nothing of the server's
 // build files either. Someone who uses an account sees them: they have to know what
-// they approve.
+// they approve. Who observes sees them too, as they see what every run ran.
 function maySeeDefinitions(grant) {
-    return grant.builds !== null || Object.keys(grant.accounts).length > 0;
+    return grant.builds !== null || Object.keys(grant.accounts).length > 0 || grant.observe;
 }
 exports.maySeeDefinitions = maySeeDefinitions;
 // `use` includes deciding about the runs others request on the account.

package/dist/server/run-worker.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.workerMain = exports.readInput = exports.taskCauses = exports.describeFailures = exports.runTasks = exports.getEndpointTargets = exports.publishEndpoints = exports.getRunNetwork = exports.workerInputSchema = void 0;
+exports.workerMain = exports.readInput = exports.taskCommands = exports.taskCauses = exports.describeFailures = exports.runTasks = exports.getEndpointTargets = exports.publishEndpoints = exports.getRunNetwork = exports.workerInputSchema = void 0;
 const readline_1 = require("readline");
 const zod_1 = require("zod");
 const cli_1 = require("../cli");
@@ -10,6 +10,7 @@ const schema_parser_1 = require("../schema/schema-parser");
 const reference_parser_1 = require("../schema/reference-parser");
 const work_scope_parser_1 = require("../schema/work-scope-parser");
 const work_item_status_1 = require("../planner/work-item-status");
+const work_item_executions_1 = require("../planner/work-item-executions");
 const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
 const assert_item_paths_1 = require("../planner/utils/assert-item-paths");
 const hosted_volumes_1 = require("../planner/utils/hosted-volumes");
@@ -218,6 +219,7 @@ up) {
                 }
             }
         }
+        let planned = false;
         for (const { name, scope } of scopes) {
             const { ctx, scope: parsed } = await (0, schema_parser_1.createParseContext)(input.entry, environment, {
                 directory: input.directory,
@@ -234,6 +236,16 @@ up) {
             if (cli.tasks().length === 0 && (!input.up || cli.services().length === 0)) {
                 throw new Error(`no tasks found for ${name}`);
             }
+            if (!planned) {
+                // from the start of this process, which the executor started for the run
+                planned = true;
+                emit({
+                    type: 'phase',
+                    phase: 'plan',
+                    startedAt: new Date(performance.timeOrigin).toISOString(),
+                    finishedAt: new Date().toISOString(),
+                });
+            }
             const execution = input.up
                 ? cli.up({ ...execOptions, workers: cli.defaultWorkers(), daemon: false, wait: 'ready' })
                 : cli.exec({ ...execOptions, workers: cli.defaultWorkers() });
@@ -392,6 +404,19 @@ function taskCauses(causes, redact) {
         : {};
 }
 exports.taskCauses = taskCauses;
+// When the commands of a task that is over ran, as its runtime saw them.
+function taskCommands(task) {
+    const executions = (0, work_item_executions_1.readExecutions)(task);
+    return executions.length
+        ? {
+            commands: executions.slice(0, runs_1.MAX_COMMANDS).map((e) => ({
+                startedAt: e.startedAt.toISOString(),
+                finishedAt: e.finishedAt.toISOString(),
+            })),
+        }
+        : {};
+}
+exports.taskCommands = taskCommands;
 // A task event when a task has moved on since the last look, with what its result needs
 // once it is over.
 function reportTasks(emit, workTree, seen, 
@@ -414,17 +439,25 @@ redact) {
                 state: status,
                 ...(status === 'skipped' ? {} : { durationMs: state.duration }),
                 ...taskCauses(state.missCauses, redact),
+                ...(status === 'succeeded' ? taskCommands(task.data) : {}),
             };
         }
         else if (state.type === 'crash') {
-            event = { type: 'task', task: task.name, state: 'failed', exitCode: state.exitCode, ...since() };
+            event = {
+                type: 'task',
+                task: task.name,
+                state: 'failed',
+                exitCode: state.exitCode,
+                ...since(),
+                ...taskCommands(task.data),
+            };
         }
         else if (state.type === 'error') {
             const error = redact(state.errorMessage).slice(0, runs_1.MAX_TASK_ERROR);
-            event = { type: 'task', task: task.name, state: 'failed', error, ...since() };
+            event = { type: 'task', task: task.name, state: 'failed', error, ...since(), ...taskCommands(task.data) };
         }
         else if (state.type === 'canceled') {
-            event = { type: 'task', task: task.name, state: 'cancelled' };
+            event = { type: 'task', task: task.name, state: 'cancelled', ...taskCommands(task.data) };
         }
         if (event && before?.status !== event.state) {
             seen.set(task.name, { status: event.state, started });

package/dist/server/runs.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.unavailableExecutor = exports.RunRegistry = exports.MAX_TASK_ERROR = exports.MAX_CAUSES = exports.SANDBOX_ID_PATTERN = exports.RUN_ID_PATTERN = exports.QuotaError = exports.RunError = exports.isFinal = void 0;
+exports.unavailableExecutor = exports.RunRegistry = exports.MAX_COMMANDS = exports.MAX_TASK_ERROR = exports.MAX_CAUSES = exports.SANDBOX_ID_PATTERN = exports.RUN_ID_PATTERN = exports.QuotaError = exports.RunError = exports.RUN_PHASES = exports.isFinal = void 0;
 const crypto_1 = require("crypto");
 const oauth_1 = require("./oauth");
 const run_logs_1 = require("./run-logs");
@@ -10,6 +10,11 @@ function isFinal(state) {
     return FINAL_STATES.includes(state);
 }
 exports.isFinal = isFinal;
+// What an executor reports while it works. `seq` and `time` are added by the registry.
+// The steps of a run before its tasks: `source` gets its files (a checkout of the commit, or
+// those of the sandbox), `namespace` prepares its place on the cluster, `plan` starts the
+// process of the run and reads its build files.
+exports.RUN_PHASES = ['source', 'namespace', 'plan'];
 // The server can not take the run now; `status` is what the caller is told
 class RunError extends Error {
     constructor() {
@@ -61,6 +66,11 @@ const TAIL_LINE_LENGTH = 1000;
 // does not can still not make a result of any size.
 exports.MAX_CAUSES = 20;
 exports.MAX_TASK_ERROR = 2 * 1024;
+// the command times a task result keeps
+exports.MAX_COMMANDS = 50;
+function isTimestamp(value) {
+    return typeof value === 'string' && value.length <= 40 && !Number.isNaN(Date.parse(value));
+}
 function bounded(event) {
     const causes = event.causes;
     return {
@@ -74,12 +84,20 @@ function bounded(event) {
             }
             : {}),
         ...(event.error !== undefined ? { error: event.error.slice(0, exports.MAX_TASK_ERROR) } : {}),
+        ...(event.commands
+            ? {
+                commands: event.commands
+                    .filter((c) => isTimestamp(c?.startedAt) && isTimestamp(c?.finishedAt))
+                    .slice(0, exports.MAX_COMMANDS)
+                    .map(({ startedAt, finishedAt }) => ({ startedAt, finishedAt })),
+            }
+            : {}),
     };
 }
 // what a task event weighs against the log limits of a run
 function taskEventSize(event) {
     const causes = (event.causes ?? []).reduce((sum, cause) => sum + Buffer.byteLength(cause) + 4, 0);
-    return 64 + causes + Buffer.byteLength(event.error ?? '');
+    return 64 + causes + Buffer.byteLength(event.error ?? '') + (event.commands?.length ?? 0) * 64;
 }
 // Where runs are kept while the server is up. Everything is scoped to the
 // caller that submitted the run: the same id asked for by someone else is
@@ -95,6 +113,8 @@ class RunRegistry {
         this.waiters = new Set();
         // the moves of service runs that execute beside their services
         this.moving = new Set();
+        // the records of evicted runs still being removed from the store, waited for by close
+        this.removing = new Set();
         this.maxConcurrent = options.maxConcurrent ?? 4;
         this.keep = options.keep ?? 200;
         this.maxEvents = options.maxEvents ?? 5000;
@@ -598,8 +618,10 @@ class RunRegistry {
             [...this.entries.values()].some((entry) => entry.run.state === 'running' || entry.run.state === 'up')) {
             await new Promise((resolve) => setTimeout(resolve, 10));
         }
-        // the runs that were stopped are written down before the server is gone
+        // the runs that were stopped are written down before the server is gone, and the records
+        // of evicted runs are gone from the store
         await Promise.all([...this.entries.values()].map((entry) => entry.saving));
+        await Promise.all([...this.removing]);
     }
     find(owner, id, canUse) {
         const entry = this.entries.get(id);
@@ -635,6 +657,7 @@ class RunRegistry {
             accounts: [...run.accounts],
             ...(run.approvals ? { approvals: JSON.parse(JSON.stringify(run.approvals)) } : {}),
             ...(run.results ? { results: JSON.parse(JSON.stringify(run.results)) } : {}),
+            ...(run.phases ? { phases: run.phases.map((phase) => ({ ...phase })) } : {}),
             ...(run.moves ? { moves: run.moves.map((move) => ({ ...move })) } : {}),
             ...(run.moving ? { moving: { ...run.moving } } : {}),
         };
@@ -1447,6 +1470,10 @@ class RunRegistry {
             }
             return;
         }
+        if (recorded.type === 'phase') {
+            this.keepPhase(entry, recorded);
+            return;
+        }
         const event = recorded.type === 'task' ? bounded(recorded) : recorded;
         if (event.type === 'log') {
             this.keepTail(entry, event.task, event.text);
@@ -1469,6 +1496,19 @@ class RunRegistry {
         }
         this.push(entry, event);
     }
+    // once each, from the executor: a phase it reports again (a service run that wakes) is the
+    // newest
+    keepPhase(entry, event) {
+        if (!exports.RUN_PHASES.includes(event.phase) ||
+            !isTimestamp(event.startedAt) ||
+            !isTimestamp(event.finishedAt) ||
+            Date.parse(event.finishedAt) < Date.parse(event.startedAt)) {
+            return;
+        }
+        const phases = (entry.run.phases ?? []).filter((phase) => phase.phase !== event.phase);
+        phases.push({ phase: event.phase, startedAt: event.startedAt, finishedAt: event.finishedAt });
+        entry.run.phases = phases.sort((a, b) => exports.RUN_PHASES.indexOf(a.phase) - exports.RUN_PHASES.indexOf(b.phase));
+    }
     keepTail(entry, task, text) {
         let tail = entry.tails.get(task);
         if (!tail) {
@@ -1510,6 +1550,7 @@ class RunRegistry {
             ...(event.durationMs !== undefined ? { durationMs: event.durationMs } : {}),
             ...(event.exitCode !== undefined ? { exitCode: event.exitCode } : {}),
             ...(event.error ? { error: event.error } : {}),
+            ...(event.commands?.length ? { commands: event.commands } : {}),
             ...(event.state === 'cached'
                 ? { cache: { hit: true } }
                 : event.state === 'succeeded' || event.state === 'failed'
@@ -1547,7 +1588,11 @@ class RunRegistry {
         const finished = [...this.entries.values()].filter((entry) => FINAL_STATES.includes(entry.run.state));
         for (const entry of finished.slice(0, Math.max(0, finished.length - this.keep))) {
             this.entries.delete(entry.run.id);
-            void entry.saving.then(() => this.options.store?.remove(entry.run.id)).catch(() => undefined);
+            const removal = entry.saving
+                .then(() => this.options.store?.remove(entry.run.id))
+                .catch(() => undefined)
+                .finally(() => this.removing.delete(removal));
+            this.removing.add(removal);
         }
     }
 }

package/dist/server/server-config.js

@@ -45,7 +45,10 @@ exports.issuerSchema = zod_1.z
     scope: zod_1.z.string().min(1).default('openid'),
 })
     .strict();
-const subjectSchema = zod_1.z.object({ iss: zod_1.z.string().min(1), sub: zod_1.z.string().min(1) }).strict();
+// `name`: how pages name a subject whose tokens name nobody, such as a service account's
+const subjectSchema = zod_1.z
+    .object({ iss: zod_1.z.string().min(1), sub: zod_1.z.string().min(1), name: zod_1.z.string().trim().min(1).max(100).optional() })
+    .strict();
 const groupSchema = zod_1.z.object({ iss: zod_1.z.string().min(1), name: zod_1.z.string().min(1) }).strict();
 const claimsSchema = zod_1.z.object({ iss: zod_1.z.string().min(1), match: zod_1.z.record(zod_1.z.string()) }).strict();
 // Who: the person or client (`subject`), everyone in a group of an issuer