hammerkit 1.27.0-next.9 → 2.0.0-next.2

package/dist/server/runs.js

@@ -27,9 +27,31 @@ class QuotaError extends RunError {
 exports.QuotaError = QuotaError;
 exports.RUN_ID_PATTERN = /^run_[A-Za-z0-9_-]{16}$/;
 exports.SANDBOX_ID_PATTERN = /^sbx_[A-Za-z0-9_-]{16}$/;
+// A service run that is starting, up or asleep is listed as open.
+function isOpenService(run) {
+    return !!run.up && !isFinal(run.state);
+}
+// Whether a run executes, for the limit of its caller: a move beside its services counts too
+function executes(entry) {
+    return entry.run.state === 'running' || !!entry.mover?.counted;
+}
+// The places of runs.up.limits that service runs take: one per caller and name of the runs
+// that are starting or up, an asleep one takes none (as checkUpLimits counts). Per place the
+// newest run, which is the one that stays.
+function servicePlaces(runs, asleep = false) {
+    const places = new Map();
+    for (const run of runs) {
+        if (run.up && !isFinal(run.state) && (run.state === 'asleep') === asleep) {
+            places.set(JSON.stringify([run.owner.iss, run.owner.sub, run.up.name]), run);
+        }
+    }
+    return [...places.values()];
+}
 function onSandbox(run, sandbox) {
     return run.source.type === 'sandbox' && run.source.id === sandbox;
 }
+// the tasks of a run whose check keys are kept, a build file has fewer
+const MAX_KEYED_TASKS = 1000;
 // how much of the end of a task's output its result carries
 const TAIL_LINES = 20;
 const TAIL_LINE_LENGTH = 1000;
@@ -70,6 +92,8 @@ class RunRegistry {
         this.closed = false;
         // callers that wait for a run to finish, let go when the server stops
         this.waiters = new Set();
+        // the moves of service runs that execute beside their services
+        this.moving = new Set();
         this.maxConcurrent = options.maxConcurrent ?? 4;
         this.keep = options.keep ?? 200;
         this.maxEvents = options.maxEvents ?? 5000;
@@ -86,11 +110,15 @@ class RunRegistry {
     async restore() {
         const store = this.options.store;
         if (!store) {
+            // what a server before this one kept for its runs is nobody's now
+            this.options.executor.restored?.([]);
             return;
         }
         const { records, skipped } = await store.load();
         skipped.forEach((name) => this.options.log?.(`skip run record ${name}: it can not be read`));
         for (const { run, events } of records) {
+            // a move that executed went with the server
+            delete run.moving;
             const entry = {
                 run,
                 events,
@@ -100,6 +128,7 @@ class RunRegistry {
                 bytes: 0,
                 tails: new Map(Object.entries(run.results ?? {}).map(([task, result]) => [task, result.lastLog])),
                 saving: Promise.resolve(),
+                keys: {},
             };
             this.entries.set(run.id, entry);
             if (run.state === 'awaiting-approval') {
@@ -112,11 +141,41 @@ class RunRegistry {
                     this.finish(entry, 'rejected', 'nobody approved it in time');
                 }
             }
+            else if (this.maySleep(entry) && ['queued', 'running', 'up', 'asleep'].includes(run.state)) {
+                // the services went with the process that held them, the next request wakes them
+                if (run.state === 'queued' || run.state === 'running' || run.state === 'up') {
+                    // a server that stopped without putting the run to sleep left its volumes where they
+                    // were, for the wake to mount
+                    if (this.keepsVolumes()) {
+                        run.volumesKept = true;
+                    }
+                    this.sleep(entry);
+                }
+                else {
+                    this.expireLater(entry);
+                }
+                if (!this.keepsVolumes()) {
+                    // the backend no longer keeps them, the sweep below removes them
+                    delete run.volumesKept;
+                }
+            }
+            else if (run.state === 'asleep') {
+                // its time ran out while the server was down: it ends now, and gives its place back
+                run.stoppedBy ?? (run.stoppedBy = 'ttl');
+                this.finish(entry, 'succeeded');
+            }
             else if (run.state === 'queued' || run.state === 'running') {
                 this.finish(entry, 'failed', `the server stopped while the run was ${run.state}, submit it again`);
             }
+            else if (run.state === 'up') {
+                // the services went with the process that held them
+                this.finish(entry, 'failed', 'the server stopped while the services were up, start them again');
+            }
         }
         this.evict();
+        this.options.executor.restored?.([...this.entries.values()]
+            .filter((entry) => entry.run.volumesKept && !isFinal(entry.run.state))
+            .map((entry) => this.copy(entry.run)));
     }
     // A run that needs approvals (for the accounts named) waits for them, the others
     // are queued at once.
@@ -129,13 +188,24 @@ class RunRegistry {
             throw new RunError('too many runs are waiting for an approval');
         }
         this.checkLimits(owner, waiting);
+        if (spec.up) {
+            this.checkUpLimits(owner, spec.up.name);
+            this.checkHosts(owner, spec.up);
+        }
         const run = {
             ...spec,
             id: `run_${(0, crypto_1.randomBytes)(12).toString('base64url')}`,
-            owner: { iss: owner.iss, sub: owner.sub },
+            owner: {
+                iss: owner.iss,
+                sub: owner.sub,
+                ...(spec.up?.sleepTtl ? { groups: [...(owner.groups ?? [])], claims: { ...(owner.claims ?? {}) } } : {}),
+            },
             state: waiting ? 'awaiting-approval' : 'queued',
             createdAt: this.timestamp(),
         };
+        if (!waiting) {
+            run.queuedAt = run.createdAt;
+        }
         const entry = {
             run,
             events: [],
@@ -145,6 +215,7 @@ class RunRegistry {
             bytes: 0,
             tails: new Map(),
             saving: Promise.resolve(),
+            keys: {},
         };
         this.entries.set(run.id, entry);
         if (waiting) {
@@ -179,30 +250,133 @@ class RunRegistry {
             .map((entry) => this.copy(entry.run))
             .reverse();
     }
-    // A queued run is cancelled at once; a running one when its executor stops.
+    // one run of any caller: who may see it is for the caller to decide
+    peek(id) {
+        const entry = this.entries.get(id);
+        return entry ? this.copy(entry.run) : undefined;
+    }
+    // every run it keeps, of every caller, newest first: who may see which is for the caller
+    all() {
+        return [...this.entries.values()].map((entry) => this.copy(entry.run)).reverse();
+    }
+    // What the caller has against each of their limits.
+    usage(owner) {
+        const mine = [...this.entries.values()].filter((entry) => this.owns(owner, entry));
+        const ids = (keep) => mine.filter(keep).map((entry) => entry.run.id);
+        return {
+            waiting: ids((entry) => entry.run.state === 'awaiting-approval'),
+            queued: ids((entry) => entry.run.state === 'queued'),
+            running: ids(executes),
+            services: servicePlaces(mine.map((entry) => entry.run)).map((run) => run.id),
+            asleep: servicePlaces(mine.map((entry) => entry.run), true).map((run) => run.id),
+        };
+    }
+    snapshot() {
+        const all = [...this.entries.values()];
+        const up = this.options.up;
+        const executing = new Map();
+        const key = (owner) => `${owner.iss}\0${owner.sub}`;
+        for (const entry of all) {
+            if (executes(entry)) {
+                executing.set(key(entry.run.owner), (executing.get(key(entry.run.owner)) ?? 0) + 1);
+            }
+        }
+        const runningOf = (owner) => executing.get(key(owner)) ?? 0;
+        return {
+            places: { max: this.maxConcurrent, used: this.running },
+            running: all.filter((entry) => entry.run.state === 'running').map((entry) => this.copy(entry.run)),
+            queue: this.queue.map((entry) => ({ run: this.copy(entry.run), waitsFor: this.waitReason(entry, runningOf) })),
+            awaiting: {
+                runs: all.filter((entry) => entry.run.state === 'awaiting-approval').map((entry) => this.copy(entry.run)),
+                max: this.maxWaiting,
+            },
+            services: {
+                runs: all.filter((entry) => isOpenService(entry.run)).map((entry) => this.copy(entry.run)),
+                taken: servicePlaces(all.map((entry) => entry.run)).length,
+                ...(up ? { max: up.total, perCaller: up.perCaller } : {}),
+                ...(up?.asleep !== undefined ? { asleepPerCaller: up.asleep } : {}),
+            },
+            ...(this.options.limits ? { limits: { ...this.options.limits } } : {}),
+        };
+    }
+    // Where a queued run stands in the queue (from 1), and why it waits.
+    queued(id) {
+        const index = this.queue.findIndex((entry) => entry.run.id === id);
+        return index < 0 ? undefined : { position: index + 1, waitsFor: this.waitReason(this.queue[index]) };
+    }
+    // A queued run is cancelled at once; a running one when its executor stops. A service run
+    // that is up stops its services, and says who stopped it.
     cancel(owner, id, canUse) {
         const entry = this.find(owner, id, canUse);
         if (!entry) {
             return undefined;
         }
-        this.stop(entry);
+        this.stop(entry, undefined, { iss: owner.iss, sub: owner.sub });
         return this.copy(entry.run);
     }
+    /**
+     * Moves the end of a service run that is up (or asleep) to `ttl` (ms) from now, at most
+     * the server's maxTtl, and never sooner than it was. Its owner may, and whoever `mayOpen`
+     * says may open its endpoints.
+     */
+    extend(caller, id, ttl, mayOpen) {
+        const entry = this.entries.get(id);
+        if (!entry || !(this.owns(caller, entry) || (entry.run.up && mayOpen?.(this.copy(entry.run))))) {
+            return { status: 'not-found' };
+        }
+        // up, asleep, or woken and starting again
+        if (!entry.run.upAt || isFinal(entry.run.state)) {
+            return { status: 'not-up', state: entry.run.state };
+        }
+        // never sooner than it was: who wants it gone stops it
+        const ms = Math.min(ttl, this.options.up?.maxTtl ?? ttl);
+        const expiresAt = Math.max(Date.parse(entry.run.expiresAt ?? '') || 0, this.now() + ms);
+        entry.run.expiresAt = new Date(expiresAt).toISOString();
+        this.record(entry, { type: 'extended', by: { iss: caller.iss, sub: caller.sub }, expiresAt: entry.run.expiresAt });
+        this.expireLater(entry);
+        return { status: 'ok', run: this.copy(entry.run) };
+    }
+    /**
+     * The service run an endpoint host belongs to: the newest that is up, so one that replaces
+     * a run of the same name takes its hosts over as soon as it is up; else the newest that
+     * had it, for a page that says how it stands. With the address of the endpoint when up.
+     */
+    endpoint(host) {
+        let found;
+        for (const entry of this.entries.values()) {
+            const endpoint = entry.run.up?.endpoints.find((candidate) => candidate.host === host);
+            if (endpoint && (entry.run.state === 'up' || found?.entry.run.state !== 'up')) {
+                found = { entry, endpoint };
+            }
+        }
+        if (!found) {
+            return undefined;
+        }
+        const { entry, endpoint } = found;
+        const target = entry.run.state === 'up'
+            ? entry.targets?.find((candidate) => candidate.service === endpoint.service && candidate.port === endpoint.port)
+            : undefined;
+        return { run: this.copy(entry.run), endpoint: { ...endpoint }, ...(target ? { target } : {}) };
+    }
     // The run that holds a sandbox: the oldest on it that is not over. It holds it from
-    // when it is submitted, so the files are not moved under what an approver saw.
+    // when it is submitted, so the files are not moved under what an approver saw. A service
+    // run that is up works on a copy of the files, and holds it no longer.
     holder(sandbox) {
-        return [...this.entries.values()].find((entry) => onSandbox(entry.run, sandbox) && !isFinal(entry.run.state))?.run
-            .id;
+        return [...this.entries.values()].find((entry) => onSandbox(entry.run, sandbox) &&
+            !isFinal(entry.run.state) &&
+            entry.run.state !== 'up' &&
+            entry.run.state !== 'asleep')?.run.id;
     }
     // Cancels every run on a sandbox and resolves when none of them runs anymore.
     async stopSandbox(sandbox, reason) {
         const on = () => [...this.entries.values()].filter((entry) => onSandbox(entry.run, sandbox));
         on().forEach((entry) => this.stop(entry, reason));
-        while (on().some((entry) => entry.run.state === 'running')) {
+        while (on().some((entry) => entry.run.state === 'running' || entry.run.state === 'up')) {
             await new Promise((resolve) => setTimeout(resolve, 20));
         }
     }
-    stop(entry, reason) {
+    stop(entry, reason, by) {
+        var _a, _b;
         if (entry.run.state === 'queued') {
             this.queue.splice(this.queue.indexOf(entry), 1);
             this.finish(entry, 'cancelled', reason);
@@ -213,6 +387,17 @@ class RunRegistry {
         else if (entry.run.state === 'running') {
             entry.abort.abort();
         }
+        else if (entry.run.state === 'up') {
+            (_a = entry.run).stoppedBy ?? (_a.stoppedBy = by ?? 'replaced');
+            entry.sleeping = false;
+            entry.restartTo = undefined;
+            entry.mover?.abort.abort();
+            entry.abort.abort();
+        }
+        else if (entry.run.state === 'asleep') {
+            (_b = entry.run).stoppedBy ?? (_b.stoppedBy = by ?? 'replaced');
+            this.finish(entry, 'succeeded');
+        }
     }
     // The runs that wait for an approval, of anybody, oldest first. Who may see
     // them is for the caller to decide.
@@ -268,6 +453,7 @@ class RunRegistry {
         }
         this.clearTimer(entry);
         entry.run.state = 'queued';
+        entry.run.queuedAt = this.timestamp();
         this.record(entry, { type: 'state', state: 'queued' });
         this.queue.push(entry);
         const approved = this.copy(entry.run);
@@ -289,18 +475,21 @@ class RunRegistry {
         return { events, finished, stop: () => entry.listeners.delete(listener) };
     }
     /**
-     * The run once it is over, or as it is after `ms` or when `signal` aborts, for clients
-     * that can not hold a stream of events open (a tool call of an agent).
+     * The run once it is over (a service run: once it is up), or as it is after `ms` or when
+     * `signal` aborts, for clients that can not hold a stream of events open (a tool call of
+     * an agent).
      */
     async wait(owner, id, ms, canUse, signal) {
         const entry = this.find(owner, id, canUse);
         if (!entry) {
             return undefined;
         }
-        if (!isFinal(entry.run.state) && ms > 0 && !this.closed && !signal?.aborted) {
+        // a service run is there once it is up, and at the commit it moves to
+        const there = () => isFinal(entry.run.state) || (entry.run.state === 'up' && !entry.run.moving);
+        if (!there() && ms > 0 && !this.closed && !signal?.aborted) {
             await new Promise((resolve) => {
                 const listener = (event) => {
-                    if (event.type === 'state' && isFinal(event.state)) {
+                    if ((event.type === 'state' || event.type === 'moved') && there()) {
                         done();
                     }
                 };
@@ -331,14 +520,34 @@ class RunRegistry {
         entry.index.update(entry.events);
         return { events: entry.events, index: entry.index, truncated: entry.truncated };
     }
+    // The log of any run, for a caller whose right to read it was checked already (an
+    // entitlement with `observe`).
+    logsOf(id) {
+        const entry = this.entries.get(id);
+        if (!entry) {
+            return undefined;
+        }
+        entry.index ?? (entry.index = new run_logs_1.LogIndex());
+        entry.index.update(entry.events);
+        return { events: entry.events, index: entry.index, truncated: entry.truncated };
+    }
     // Stops accepting runs and aborts the ones in flight.
     async close() {
         this.closed = true;
         for (const entry of this.queue.splice(0)) {
-            this.finish(entry, 'cancelled');
+            if (this.maySleep(entry) && this.options.store) {
+                // a woken service run sleeps again, the next start keeps it
+                this.sleep(entry);
+            }
+            else {
+                this.finish(entry, 'cancelled');
+            }
         }
         for (const entry of this.entries.values()) {
-            if (entry.run.state === 'awaiting-approval') {
+            if (entry.run.state === 'asleep') {
+                this.clearTimer(entry);
+            }
+            else if (entry.run.state === 'awaiting-approval') {
                 if (this.options.store) {
                     // the record keeps it waiting, the next start takes it back
                     this.clearTimer(entry);
@@ -353,11 +562,17 @@ class RunRegistry {
             done();
         }
         for (const entry of this.entries.values()) {
-            if (entry.run.state === 'running') {
+            if (entry.run.state === 'running' || entry.run.state === 'up') {
+                // a service run that may sleep is asleep in its record, the next request after the
+                // start wakes it
+                entry.sleeping = this.maySleep(entry) && !!this.options.store;
+                entry.mover?.abort.abort();
                 entry.abort.abort();
             }
         }
-        while (this.running > 0) {
+        while (this.running > 0 ||
+            this.moving.size > 0 ||
+            [...this.entries.values()].some((entry) => entry.run.state === 'running' || entry.run.state === 'up')) {
             await new Promise((resolve) => setTimeout(resolve, 10));
         }
         // the runs that were stopped are written down before the server is gone
@@ -388,6 +603,8 @@ class RunRegistry {
     copy(run) {
         return {
             ...run,
+            ...(run.up ? { up: JSON.parse(JSON.stringify(run.up)) } : {}),
+            ...(run.stoppedBy && typeof run.stoppedBy === 'object' ? { stoppedBy: { ...run.stoppedBy } } : {}),
             tasks: [...run.tasks],
             source: { ...run.source },
             options: JSON.parse(JSON.stringify(run.options)),
@@ -395,6 +612,8 @@ class RunRegistry {
             accounts: [...run.accounts],
             ...(run.approvals ? { approvals: JSON.parse(JSON.stringify(run.approvals)) } : {}),
             ...(run.results ? { results: JSON.parse(JSON.stringify(run.results)) } : {}),
+            ...(run.moves ? { moves: run.moves.map((move) => ({ ...move })) } : {}),
+            ...(run.moving ? { moving: { ...run.moving } } : {}),
         };
     }
     timestamp() {
@@ -406,30 +625,128 @@ class RunRegistry {
         if (!limits) {
             return;
         }
-        const mine = [...this.entries.values()].filter((entry) => this.owns(owner, entry));
-        const count = (state) => mine.filter((entry) => entry.run.state === state).length;
-        if (waiting && count('awaiting-approval') >= limits.waiting) {
+        const usage = this.usage(owner);
+        if (waiting && usage.waiting.length >= limits.waiting) {
             throw new QuotaError(`you have ${limits.waiting} runs waiting for an approval, wait for a decision or cancel one`);
         }
-        if (!waiting && count('queued') >= limits.queued) {
+        if (!waiting && usage.queued.length >= limits.queued) {
             throw new QuotaError(`you have ${limits.queued} runs in the queue, wait for one to start or cancel one`);
         }
     }
+    // Service runs that are starting or up, of the caller and of everybody, against what may
+    // be. One that is asleep does not count: it asks for a place when it wakes (visit). The runs
+    // of a caller with one name take one place: the newest replaces the others once it is up,
+    // and an older one that gets up after it goes (markUp), so a new run of a name the caller
+    // has does not need another. Several of one name that are still starting share it too:
+    // what they take while they start is held by runs.limits.running and maxConcurrent.
+    checkUpLimits(owner, name) {
+        // what a caller keeps asleep is held by a limit of its own: a run that sleeps may keep the
+        // volumes of its services. A run of a name the caller has a run of (asleep, up or
+        // starting) replaces it, it is not one more.
+        const asleep = this.options.up?.asleep;
+        if (asleep !== undefined) {
+            const open = [...this.entries.values()].filter((entry) => entry.run.up && !isFinal(entry.run.state) && this.owns(owner, entry));
+            const sleeping = new Set(open.filter((entry) => entry.run.state === 'asleep').map((entry) => entry.run.up?.name));
+            if (!open.some((entry) => entry.run.up?.name === name) && sleeping.size >= asleep) {
+                throw new QuotaError(`you have ${asleep} service runs asleep, stop one or wait for its end`);
+            }
+        }
+        const full = this.upLimitReached(owner, name);
+        if (full === 'caller') {
+            throw new QuotaError(`you have ${this.options.up?.perCaller} service runs starting or up, stop one or wait for its end`);
+        }
+        if (full === 'server') {
+            throw new RunError(`the server has ${this.options.up?.total} service runs starting or up, try again later`);
+        }
+    }
+    // Whose limit a service run of the caller with this name would go over, if any
+    upLimitReached(owner, name) {
+        const limits = this.options.up;
+        if (!limits) {
+            return undefined;
+        }
+        const places = new Set();
+        const mine = new Set();
+        for (const entry of this.entries.values()) {
+            if (!entry.run.up || isFinal(entry.run.state) || entry.run.state === 'asleep') {
+                continue;
+            }
+            places.add(JSON.stringify([entry.run.owner.iss, entry.run.owner.sub, entry.run.up.name]));
+            if (this.owns(owner, entry)) {
+                mine.add(entry.run.up.name);
+            }
+        }
+        if (mine.has(name)) {
+            return undefined;
+        }
+        if (mine.size >= limits.perCaller) {
+            return 'caller';
+        }
+        if (places.size >= limits.total) {
+            return 'server';
+        }
+        return undefined;
+    }
+    // A host is one run's, or a run's that replaces it (same owner and name) once it is up:
+    // two labels that come out the same are refused, they would reach each other's services.
+    checkHosts(owner, up) {
+        const hosts = new Set(up.endpoints.map((endpoint) => endpoint.host));
+        for (const entry of this.entries.values()) {
+            if (isFinal(entry.run.state) || !entry.run.up) {
+                continue;
+            }
+            const taken = entry.run.up.endpoints.find((endpoint) => hosts.has(endpoint.host));
+            if (taken && !(this.owns(owner, entry) && entry.run.up.name === up.name)) {
+                const error = new RunError(`the address ${taken.host} is another service run's, pick another name`);
+                error.status = 409;
+                throw error;
+            }
+        }
+    }
     runningOf(owner) {
-        return [...this.entries.values()].filter((entry) => entry.run.state === 'running' && this.owns(owner, entry)).length;
+        return this.usage(owner).running.length;
     }
-    // Whether a run on the same sandbox executes: a sandbox runs one at a time.
+    // The run on the same sandbox that executes: a sandbox runs one at a time.
     sandboxBusy(run) {
         const source = run.source;
-        return (source.type === 'sandbox' &&
-            [...this.entries.values()].some((entry) => entry.run.state === 'running' && onSandbox(entry.run, source.id)));
+        if (source.type !== 'sandbox') {
+            return undefined;
+        }
+        return [...this.entries.values()].find((entry) => entry.run.state === 'running' && onSandbox(entry.run, source.id))
+            ?.run.id;
     }
-    // Whether a run that executes holds what the run would (RunExecutor.exclusive)
+    // The run that executes and holds what the run would (RunExecutor.exclusive)
     held(run) {
         const exclusive = this.options.executor.exclusive?.bind(this.options.executor);
         const key = exclusive?.(run);
-        return (key !== undefined &&
-            [...this.entries.values()].some((entry) => entry.run.state === 'running' && exclusive?.(entry.run) === key));
+        if (key === undefined) {
+            return undefined;
+        }
+        return [...this.entries.values()].find((entry) => entry.run.state === 'running' && exclusive?.(entry.run) === key)
+            ?.run.id;
+    }
+    // What keeps a queued run from starting, checked as dispatch checks it: what holds it
+    // even when a place frees up is named first.
+    // `runningOf` counts the runs of the owner that execute; a snapshot passes one that
+    // counted all owners once, so a look at the queue costs the runs once and not per entry
+    waitReason(entry, runningOf = (owner) => this.runningOf(owner)) {
+        const limit = this.options.limits?.running ?? Infinity;
+        const running = runningOf(entry.run.owner);
+        if (running >= limit) {
+            return { type: 'own-limit', running, limit };
+        }
+        const sandbox = this.sandboxBusy(entry.run);
+        if (sandbox) {
+            return { type: 'sandbox', run: sandbox };
+        }
+        const holder = this.held(entry.run);
+        if (holder) {
+            return { type: 'held', run: holder };
+        }
+        if (this.running >= this.maxConcurrent) {
+            return { type: 'place', used: this.running, max: this.maxConcurrent };
+        }
+        return { type: 'starting' };
     }
     // The first run in the queue whose caller has a place left: one caller with many runs
     // does not keep the others from the machine. A run on a sandbox another run works on
@@ -439,12 +756,21 @@ class RunRegistry {
             const perCaller = this.options.limits?.running ?? Infinity;
             const index = this.queue.findIndex((entry) => this.runningOf(entry.run.owner) < perCaller && !this.sandboxBusy(entry.run) && !this.held(entry.run));
             if (index < 0) {
+                // the moves that wait for a place
+                for (const entry of this.entries.values()) {
+                    if (entry.pendingMove && entry.run.state === 'up' && !entry.mover) {
+                        this.beginMove(entry);
+                    }
+                }
                 return;
             }
             const [entry] = this.queue.splice(index, 1);
             this.running++;
             this.execute(entry).finally(() => {
-                this.running--;
+                // a service run that is up gave its place back already
+                if (!entry.released) {
+                    this.running--;
+                }
                 this.dispatch();
             });
         }
@@ -453,27 +779,597 @@ class RunRegistry {
         entry.run.state = 'running';
         entry.run.startedAt = this.timestamp();
         this.record(entry, { type: 'state', state: 'running' });
+        // a service run gets this long to be up, then its ttl counts
         const limit = this.options.maxDuration;
-        const timer = limit
+        entry.durationTimer = limit
             ? setTimeout(() => {
                 entry.timedOut = true;
                 entry.abort.abort();
             }, limit)
             : undefined;
-        timer?.unref();
+        entry.durationTimer?.unref();
+        const token = {};
+        entry.holder = token;
+        // the commit of a move whose services are not up there yet
+        const landing = entry.landing && !entry.landing.recorded ? entry.landing.change : undefined;
+        const run = landing
+            ? { ...this.copy(entry.run), source: { ...landing.source }, ...(landing.plan ? { plan: landing.plan } : {}) }
+            : this.copy(entry.run);
         try {
-            await this.options.executor.execute(this.copy(entry.run), {
+            await this.options.executor.execute(run, {
                 emit: (event) => this.record(entry, event),
                 signal: entry.abort.signal,
+                ...(entry.run.up
+                    ? {
+                        up: (targets) => this.markUp(entry, targets),
+                        keep: () => (entry.holder === token ? this.keepOf(entry) : 'everything'),
+                    }
+                    : {}),
             });
-            this.finish(entry, entry.timedOut ? 'failed' : entry.abort.signal.aborted ? 'cancelled' : 'succeeded', entry.timedOut ? this.tooLong() : undefined);
+            if (entry.holder !== token) {
+                // a move took the services over
+                return;
+            }
+            if (this.fallBack(entry, entry.timedOut ? this.tooLong() : 'its services stopped before they were up')) {
+                return;
+            }
+            if (entry.restartTo || entry.sleeping) {
+                this.stopped(entry);
+            }
+            else if (entry.run.upAt) {
+                // services stay until they are stopped: ones that stopped on their own failed
+                this.endServices(entry, 'the services stopped on their own');
+            }
+            else {
+                this.finish(entry, entry.timedOut ? 'failed' : entry.abort.signal.aborted ? 'cancelled' : 'succeeded', entry.timedOut ? this.tooLong() : undefined);
+            }
         }
         catch (e) {
-            const cancelled = entry.abort.signal.aborted && !entry.timedOut;
-            this.finish(entry, cancelled ? 'cancelled' : 'failed', entry.timedOut ? this.tooLong() : cancelled ? undefined : e instanceof Error ? e.message : String(e));
+            const message = e instanceof Error ? e.message : String(e);
+            if (entry.holder !== token) {
+                return;
+            }
+            if (this.fallBack(entry, entry.timedOut ? this.tooLong() : message)) {
+                return;
+            }
+            if (entry.restartTo || entry.sleeping) {
+                this.stopped(entry);
+            }
+            else if (entry.run.upAt) {
+                this.endServices(entry, message);
+            }
+            else {
+                const cancelled = entry.abort.signal.aborted && !entry.timedOut;
+                this.finish(entry, cancelled ? 'cancelled' : 'failed', entry.timedOut ? this.tooLong() : cancelled ? undefined : message);
+            }
         }
         finally {
-            clearTimeout(timer);
+            clearTimeout(entry.durationTimer);
+        }
+    }
+    // The services of a service run are ready: its ttl counts from now, its place of
+    // maxConcurrent goes to the next run, and an older run of its owner with the same name
+    // stops, so its hosts are this run's.
+    markUp(entry, targets) {
+        const spec = entry.run.up;
+        if (!spec || entry.run.state !== 'running' || entry.abort.signal.aborted) {
+            return;
+        }
+        clearTimeout(entry.durationTimer);
+        entry.durationTimer = undefined;
+        entry.targets = targets.map((target) => ({ ...target, address: { ...target.address } }));
+        // the services got up at the commit of a move: the run is there now
+        const landing = entry.landing;
+        entry.landing = undefined;
+        if (landing && !landing.recorded) {
+            this.moveTo(entry, landing.change);
+        }
+        entry.run.state = 'up';
+        // one reading of the clock, so the ttl is exactly expiresAt - upAt
+        const now = this.now();
+        // the ttl counts from the first time, not from a wake
+        if (!entry.run.upAt) {
+            entry.run.upAt = new Date(now).toISOString();
+            entry.run.expiresAt = new Date(now + spec.ttl).toISOString();
+        }
+        if (spec.sleepTtl) {
+            entry.run.sleepsAt = new Date(now + spec.sleepTtl).toISOString();
+        }
+        this.record(entry, { type: 'state', state: 'up' });
+        this.expireLater(entry);
+        entry.released = true;
+        this.running--;
+        const entries = [...this.entries.values()];
+        for (const other of entries.slice(0, entries.indexOf(entry))) {
+            if (other.run.up?.name === spec.name &&
+                this.owns(entry.run.owner, other) &&
+                !isFinal(other.run.state) &&
+                other.run.state !== 'awaiting-approval') {
+                other.run.stoppedBy = 'replaced';
+                this.stop(other, `replaced by ${entry.run.id}`);
+            }
+        }
+        // A newer run of the name that is up or asleep already (this one waited for an approval
+        // while it got up) is what the owner started last: this one goes, so the runs of a name,
+        // which take one place of runs.up.limits, never hold more than one set of services.
+        const newer = entries
+            .slice(entries.indexOf(entry) + 1)
+            .find((other) => other.run.up?.name === spec.name &&
+            this.owns(entry.run.owner, other) &&
+            (other.run.state === 'up' || other.run.state === 'asleep'));
+        if (newer) {
+            entry.run.stoppedBy = 'replaced';
+            this.stop(entry, `replaced by ${newer.run.id}`);
+        }
+        this.dispatch();
+        if (!newer) {
+            // a move that came while it started or woke
+            this.beginMove(entry);
+        }
+    }
+    /**
+     * The service run of the caller of this name that a run of theirs would replace: the newest
+     * that is not over and waits for no approval. RunRegistry.move moves it instead, when it
+     * runs the same.
+     */
+    named(owner, name) {
+        const entry = [...this.entries.values()]
+            .reverse()
+            .find((candidate) => candidate.run.up?.name === name &&
+            this.owns(owner, candidate) &&
+            !isFinal(candidate.run.state) &&
+            candidate.run.state !== 'awaiting-approval');
+        return entry ? this.copy(entry.run) : undefined;
+    }
+    /**
+     * Moves a service run of the caller to another commit of its repository, in place of a new
+     * run that replaces it: it keeps its id, its addresses, its volumes and its place. Queued or
+     * asleep, it starts the new commit when it starts; up, it moves there now (see
+     * RunExecutor.movesBeside), starting or waking, once it is up. A later move takes the place
+     * of one that did not begin yet, and stops one that executes. `ttl` moves its end to that
+     * long from now, if that is later. Undefined when the run is not the caller's to move.
+     */
+    move(owner, id, change) {
+        const entry = this.entries.get(id);
+        const state = entry?.run.state;
+        if (!entry ||
+            !this.owns(owner, entry) ||
+            !entry.run.up ||
+            entry.run.source.type !== 'ref' ||
+            !state ||
+            isFinal(state) ||
+            state === 'awaiting-approval' ||
+            this.closed) {
+            return undefined;
+        }
+        const by = { iss: owner.iss, sub: owner.sub };
+        if (change.owner) {
+            entry.run.owner = { ...change.owner };
+        }
+        if (change.ttl !== undefined) {
+            if (entry.run.upAt) {
+                const ms = Math.min(change.ttl, this.options.up?.maxTtl ?? change.ttl);
+                const expiresAt = Math.max(Date.parse(entry.run.expiresAt ?? '') || 0, this.now() + ms);
+                if (new Date(expiresAt).toISOString() !== entry.run.expiresAt) {
+                    entry.run.expiresAt = new Date(expiresAt).toISOString();
+                    this.record(entry, { type: 'extended', by, expiresAt: entry.run.expiresAt });
+                }
+            }
+            else {
+                entry.run.up.ttl = Math.max(entry.run.up.ttl, change.ttl);
+            }
+        }
+        if (state === 'queued' && entry.landing && !entry.landing.recorded) {
+            // it starts again at the commit of a move: at this one instead
+            entry.landing.change = change;
+            entry.run.moving = { commit: change.source.commit, ref: change.source.ref, since: this.timestamp() };
+            return this.copy(entry.run);
+        }
+        if (state === 'queued' || state === 'asleep') {
+            if (entry.run.upAt) {
+                // it was up at the commit it is at: a wake that fails at the new one goes back there
+                entry.landing = {
+                    change,
+                    from: entry.landing?.from ?? this.landingFrom(entry),
+                    recorded: true,
+                };
+            }
+            this.moveTo(entry, change);
+            if (state === 'asleep') {
+                this.expireLater(entry);
+            }
+            return this.copy(entry.run);
+        }
+        // running or up: once it is up, beside its services
+        entry.pendingMove = change;
+        entry.run.moving = { commit: change.source.commit, ref: change.source.ref, since: this.timestamp() };
+        // one that executes goes, the newer commit is what is wanted
+        entry.mover?.abort.abort();
+        if (state === 'up') {
+            // a move counts as a request: the run does not sleep under it
+            this.keepAwake(entry);
+            this.beginMove(entry);
+        }
+        return this.copy(entry.run);
+    }
+    // The run is at the commit of `change` now
+    moveTo(entry, change, error) {
+        const move = {
+            commit: change.source.commit,
+            ref: change.source.ref,
+            at: this.timestamp(),
+            ...(error ? { error } : {}),
+        };
+        if (!error) {
+            entry.run.source = { ...change.source };
+            if (change.plan) {
+                entry.run.plan = change.plan;
+            }
+        }
+        entry.run.moves = [...(entry.run.moves ?? []), move];
+        if (!entry.pendingMove && !entry.mover) {
+            delete entry.run.moving;
+        }
+        this.record(entry, {
+            type: 'moved',
+            commit: move.commit,
+            ref: move.ref,
+            ...(error ? { error } : {}),
+        });
+    }
+    keepAwake(entry) {
+        const sleepTtl = entry.run.up?.sleepTtl;
+        if (sleepTtl && entry.run.state === 'up') {
+            entry.run.sleepsAt = new Date(this.now() + sleepTtl).toISOString();
+            this.expireLater(entry);
+        }
+    }
+    // The move that waits, once the run is up and no other move executes
+    beginMove(entry) {
+        const change = entry.pendingMove;
+        if (!change || entry.run.state !== 'up' || entry.mover || entry.restartTo || entry.abort.signal.aborted) {
+            return;
+        }
+        const beside = !!this.options.executor.movesBeside?.(this.copy(entry.run));
+        if (beside &&
+            (this.running >= this.maxConcurrent ||
+                this.runningOf(entry.run.owner) >= (this.options.limits?.running ?? Infinity))) {
+            // it executes as a run does: it waits for a place (dispatch)
+            return;
+        }
+        entry.pendingMove = undefined;
+        entry.run.moving = { commit: change.source.commit, ref: change.source.ref, since: this.timestamp() };
+        this.record(entry, {
+            type: 'log',
+            task: 'hammerkit',
+            stream: 'stdout',
+            text: `moving ${entry.run.source.type === 'ref' ? `from ${entry.run.source.commit} ` : ''}to ${change.source.commit}`,
+        });
+        if (!beside) {
+            // its services stop, their volumes kept, and it starts again at the commit
+            entry.restartTo = change;
+            entry.abort.abort();
+            return;
+        }
+        const token = {};
+        const abort = new AbortController();
+        const mover = { token, abort, change, counted: true };
+        this.running++;
+        // it gets as long as a run gets to be up
+        const limit = this.options.maxDuration;
+        if (limit) {
+            mover.timer = setTimeout(() => {
+                mover.timedOut = true;
+                abort.abort();
+            }, limit);
+            mover.timer.unref();
+        }
+        entry.mover = mover;
+        const run = {
+            ...this.copy(entry.run),
+            source: { ...change.source },
+            ...(change.plan ? { plan: change.plan } : {}),
+        };
+        const execution = this.executeMove(entry, token, abort, run);
+        this.moving.add(execution);
+        void execution.finally(() => this.moving.delete(execution));
+    }
+    // A move beside the services of a run that is up: once its services are ready it holds them
+    // (moved), and its end is the run's. One that ends before leaves the run where it was.
+    async executeMove(entry, token, abort, run) {
+        let failure;
+        try {
+            await this.options.executor.execute(run, {
+                emit: (event) => this.record(entry, event),
+                signal: abort.signal,
+                up: (targets) => this.moved(entry, token, targets),
+                keep: () => (entry.holder === token ? this.keepOf(entry) : 'everything'),
+            });
+            failure = abort.signal.aborted ? 'the move was stopped' : 'its services stopped before they were up';
+        }
+        catch (e) {
+            failure = e instanceof Error ? e.message : String(e);
+        }
+        const mover = entry.mover?.token === token ? entry.mover : undefined;
+        if (mover?.timedOut) {
+            failure = `the move took longer than ${(0, units_1.formatDuration)(this.options.maxDuration)} and was stopped`;
+        }
+        this.releaseMover(mover);
+        if (entry.holder === token) {
+            // the run's services were this execution's
+            if (entry.restartTo || entry.sleeping) {
+                this.stopped(entry);
+            }
+            else {
+                this.endServices(entry, failure === 'the move was stopped' ? 'the services stopped on their own' : failure);
+            }
+            return;
+        }
+        if (entry.mover?.token !== token) {
+            return;
+        }
+        entry.mover = undefined;
+        if (isFinal(entry.run.state)) {
+            return;
+        }
+        this.moveTo(entry, { source: run.source }, failure);
+        this.beginMove(entry);
+        this.dispatch();
+    }
+    // A move that took the services over or ended gives its place back
+    releaseMover(mover) {
+        if (!mover?.counted) {
+            return;
+        }
+        mover.counted = false;
+        clearTimeout(mover.timer);
+        this.running--;
+    }
+    // The services of a move are ready: they are the run's from now, at its commit, and the
+    // execution that held the services until now leaves them to it.
+    moved(entry, token, targets) {
+        const mover = entry.mover;
+        if (!mover || mover.token !== token || mover.abort.signal.aborted || entry.run.state !== 'up') {
+            return;
+        }
+        entry.mover = undefined;
+        this.releaseMover(mover);
+        const previous = entry.abort;
+        entry.abort = mover.abort;
+        entry.holder = token;
+        entry.targets = targets.map((target) => ({ ...target, address: { ...target.address } }));
+        this.moveTo(entry, mover.change);
+        this.keepAwake(entry);
+        previous.abort();
+        this.beginMove(entry);
+        this.dispatch();
+    }
+    // The services of a run stopped to sleep, or to start again at the commit of a move
+    stopped(entry) {
+        if (!entry.restartTo) {
+            if (entry.run.stoppedBy) {
+                // a stop came while its services stopped to sleep: it ends
+                this.finish(entry, 'succeeded');
+                return;
+            }
+            this.sleep(entry);
+            return;
+        }
+        // the newest move, one that came while the services stopped
+        const restart = entry.pendingMove ?? entry.restartTo;
+        entry.restartTo = undefined;
+        entry.pendingMove = undefined;
+        if (this.closed) {
+            // the record keeps it asleep, at the commit it was up at
+            this.moveTo(entry, restart, 'the server stopped before the services were up at the commit');
+            if (this.maySleep(entry) && this.options.store) {
+                this.sleep(entry);
+            }
+            else {
+                this.finish(entry, 'failed', 'the server stopped while the services were up, start them again');
+            }
+            return;
+        }
+        // up at its commit, the run is there (markUp); else it goes back (fallBack)
+        entry.landing = {
+            change: restart,
+            from: this.landingFrom(entry),
+            recorded: false,
+        };
+        entry.run.moving = { commit: restart.source.commit, ref: restart.source.ref, since: this.timestamp() };
+        this.requeue(entry);
+    }
+    /**
+     * The execution that started the services at the commit of a move ended before they were
+     * up: the run goes back to the commit it was up at, with its volumes, and starts there
+     * again. The move says why it did not get there. False when no move was landing, or the
+     * run was stopped.
+     */
+    fallBack(entry, error) {
+        const landing = entry.landing;
+        if (!landing || entry.run.stoppedBy) {
+            return false;
+        }
+        entry.landing = undefined;
+        const from = landing.from.source;
+        entry.run.source = { ...from };
+        if (landing.from.plan) {
+            entry.run.plan = landing.from.plan;
+        }
+        this.moveTo(entry, landing.change, error);
+        this.record(entry, {
+            type: 'log',
+            task: 'hammerkit',
+            stream: 'stderr',
+            text: `did not get up at ${landing.change.source.commit}, starting again at ${from.commit}: ${error}`,
+        });
+        if (this.closed) {
+            if (this.maySleep(entry) && this.options.store) {
+                this.sleep(entry);
+            }
+            else {
+                this.finish(entry, 'failed', 'the server stopped while the services were up, start them again');
+            }
+            return true;
+        }
+        this.requeue(entry);
+        return true;
+    }
+    // where a run that moves goes back to: the commit it is at (service runs start from a ref)
+    landingFrom(entry) {
+        return {
+            source: { ...entry.run.source },
+            ...(entry.run.plan ? { plan: entry.run.plan } : {}),
+        };
+    }
+    // The run starts again, its volumes kept
+    requeue(entry) {
+        entry.sleeping = false;
+        entry.targets = undefined;
+        entry.abort = new AbortController();
+        entry.released = false;
+        entry.timedOut = false;
+        entry.run.state = 'queued';
+        this.record(entry, { type: 'state', state: 'queued' });
+        this.queue.push(entry);
+        this.dispatch();
+    }
+    // Stops a service run at its expiry, and puts one that is up to sleep at its sleepsAt. A
+    // timer can't wait longer than about 24 days, so a longer one looks again then.
+    expireLater(entry) {
+        this.clearTimer(entry);
+        const expiresAt = Date.parse(entry.run.expiresAt ?? '');
+        const sleepsAt = entry.run.state === 'up' ? Date.parse(entry.run.sleepsAt ?? '') : NaN;
+        const next = Number.isFinite(sleepsAt) ? Math.min(expiresAt, sleepsAt) : expiresAt;
+        entry.timer = setTimeout(() => {
+            var _a;
+            if (isFinal(entry.run.state)) {
+                return;
+            }
+            if (Date.parse(entry.run.expiresAt ?? '') <= this.now()) {
+                (_a = entry.run).stoppedBy ?? (_a.stoppedBy = 'ttl');
+                entry.sleeping = false;
+                if (entry.run.state === 'asleep') {
+                    this.finish(entry, 'succeeded');
+                }
+                else {
+                    this.stop(entry);
+                }
+                return;
+            }
+            if (entry.run.state === 'up' && Date.parse(entry.run.sleepsAt ?? '') <= this.now()) {
+                if (entry.run.stoppedBy || entry.abort.signal.aborted) {
+                    // it stops already: it ends, it does not sleep
+                    return;
+                }
+                if (entry.mover || entry.pendingMove || entry.restartTo) {
+                    // it moves: it sleeps once its services are at the new commit
+                    this.keepAwake(entry);
+                    return;
+                }
+                entry.sleeping = true;
+                entry.abort.abort();
+                return;
+            }
+            this.expireLater(entry);
+        }, Math.max(0, Math.min(next - this.now(), 2 ** 31 - 1)));
+        entry.timer.unref();
+    }
+    keepsVolumes() {
+        return this.options.up?.keepVolumes !== false;
+    }
+    // What the executor of a service run keeps of its services once they stopped, as it asks:
+    // the volumes of a run that sleeps, on a server that keeps them. What it was told stays in
+    // the record, so the volumes are removed (RunExecutor.forget) when the run ends asleep, or
+    // when a stop came after the sleep began.
+    keepOf(entry) {
+        // a run that starts again at another commit keeps them on any server
+        if (entry.restartTo || entry.landing || (entry.sleeping && this.keepsVolumes())) {
+            entry.run.volumesKept = true;
+            return 'volumes';
+        }
+        delete entry.run.volumesKept;
+        return undefined;
+    }
+    // Whether a service run sleeps instead of ending: it has a sleepTtl, its ttl has not
+    // ended, and it starts from a commit (the files of a sandbox change under it).
+    maySleep(entry) {
+        return (!!entry.run.up?.sleepTtl &&
+            !!entry.run.upAt &&
+            entry.run.source.type === 'ref' &&
+            Date.parse(entry.run.expiresAt ?? '') > this.now());
+    }
+    // The services stopped, the run waits for a request to wake it, or for its end.
+    sleep(entry) {
+        entry.sleeping = false;
+        entry.targets = undefined;
+        entry.run.state = 'asleep';
+        delete entry.run.sleepsAt;
+        this.record(entry, { type: 'state', state: 'asleep' });
+        this.expireLater(entry);
+    }
+    /**
+     * A request to an endpoint of a service run: one that is up stays up for its sleepTtl from
+     * now, one that is asleep is started again, if `mayWake` finds its owner may still run it
+     * (else it ends) and there is a place of runs.up.limits for it (else it sleeps on, and the
+     * next request asks again). Answers why it was not woken, or undefined.
+     */
+    visit(id, mayWake) {
+        const entry = this.entries.get(id);
+        if (!entry?.run.up) {
+            return undefined;
+        }
+        const sleepTtl = entry.run.up.sleepTtl;
+        if (entry.run.state === 'up' && sleepTtl) {
+            // only kept in memory: a server that stops puts the run to sleep anyway
+            entry.run.sleepsAt = new Date(this.now() + sleepTtl).toISOString();
+            return undefined;
+        }
+        if (entry.run.state !== 'asleep' || this.closed) {
+            return undefined;
+        }
+        if (!this.maySleep(entry)) {
+            this.expireLater(entry);
+            return 'its time ran out';
+        }
+        const refused = mayWake?.(this.copy(entry.run));
+        if (refused) {
+            this.finish(entry, 'failed', refused);
+            return refused;
+        }
+        const full = this.upLimitReached(entry.run.owner, entry.run.up.name);
+        if (full === 'caller') {
+            return 'its owner has as many service runs starting or up as they may, it wakes once one of them stops or sleeps';
+        }
+        if (full === 'server') {
+            return 'the server has as many service runs starting or up as it may, it wakes once one of them stops or sleeps';
+        }
+        this.clearTimer(entry);
+        entry.abort = new AbortController();
+        entry.released = false;
+        entry.timedOut = false;
+        entry.run.wakes = (entry.run.wakes ?? 0) + 1;
+        entry.run.state = 'queued';
+        entry.run.queuedAt = this.timestamp();
+        this.record(entry, { type: 'state', state: 'queued' });
+        this.queue.push(entry);
+        this.dispatch();
+        return undefined;
+    }
+    // The end of a run that was up: stopped by somebody, its ttl or a newer run it succeeded,
+    // stopped by the server going away it failed, as its record says after a restart.
+    endServices(entry, failure) {
+        if (entry.timedOut) {
+            this.finish(entry, 'failed', this.tooLong());
+        }
+        else if (!entry.abort.signal.aborted) {
+            this.finish(entry, 'failed', failure);
+        }
+        else if (entry.run.stoppedBy) {
+            this.finish(entry, 'succeeded');
+        }
+        else {
+            this.finish(entry, 'failed', 'the server stopped while the services were up, start them again');
         }
     }
     tooLong() {
@@ -481,6 +1377,22 @@ class RunRegistry {
     }
     finish(entry, state, error) {
         this.clearTimer(entry);
+        clearTimeout(entry.durationTimer);
+        entry.targets = undefined;
+        entry.pendingMove = undefined;
+        entry.restartTo = undefined;
+        entry.landing = undefined;
+        entry.mover?.abort.abort();
+        entry.mover = undefined;
+        delete entry.run.moving;
+        if (entry.run.volumesKept) {
+            // it ends without waking: what was kept for it goes
+            delete entry.run.volumesKept;
+            const run = this.copy(entry.run);
+            Promise.resolve()
+                .then(() => this.options.executor.forget?.(run))
+                .catch((e) => this.options.log?.(`the volumes run ${run.id} kept could not be removed: ${e instanceof Error ? e.message : e}`));
+        }
         entry.run.state = state;
         entry.run.finishedAt = this.timestamp();
         if (error) {
@@ -495,9 +1407,23 @@ class RunRegistry {
         }
         this.record(entry, { type: 'state', state, ...(error ? { error } : {}) }, true);
         entry.listeners.clear();
+        try {
+            this.options.onFinished?.(this.copy(entry.run), { ...entry.keys });
+        }
+        catch (e) {
+            this.options.log?.(`run ${entry.run.id} ended, but what it passed was not kept: ${e instanceof Error ? e.message : e}`);
+        }
         this.evict();
     }
     record(entry, recorded, force = false) {
+        if (recorded.type === 'key') {
+            // a task name is the executor's; the first key it reports for a task is the one
+            // computed before anything of the run had run
+            if (!(recorded.task in entry.keys) && Object.keys(entry.keys).length < MAX_KEYED_TASKS) {
+                entry.keys[recorded.task] = recorded.key;
+            }
+            return;
+        }
         const event = recorded.type === 'task' ? bounded(recorded) : recorded;
         if (event.type === 'log') {
             this.keepTail(entry, event.task, event.text);
@@ -579,7 +1505,7 @@ class RunRegistry {
         }
         // what a restart must not lose: the runs that wait, and how a run ended. A running
         // run is not kept up to date, it fails with the server.
-        if (event.type === 'state' || event.type === 'approval') {
+        if (event.type === 'state' || event.type === 'approval' || event.type === 'extended' || event.type === 'moved') {
             this.persist(entry);
         }
     }