hammerkit 1.27.0-next.13 → 1.27.0-next.15

package/dist/server/runs.js

@@ -112,9 +112,22 @@ class RunRegistry {
                     this.finish(entry, 'rejected', 'nobody approved it in time');
                 }
             }
+            else if (this.maySleep(entry) && ['queued', 'running', 'up', 'asleep'].includes(run.state)) {
+                // the services went with the process that held them, the next request wakes them
+                if (run.state === 'queued' || run.state === 'running' || run.state === 'up') {
+                    this.sleep(entry);
+                }
+                else {
+                    this.expireLater(entry);
+                }
+            }
             else if (run.state === 'queued' || run.state === 'running') {
                 this.finish(entry, 'failed', `the server stopped while the run was ${run.state}, submit it again`);
             }
+            else if (run.state === 'up') {
+                // the services went with the process that held them
+                this.finish(entry, 'failed', 'the server stopped while the services were up, start them again');
+            }
         }
         this.evict();
     }
@@ -129,10 +142,18 @@ class RunRegistry {
             throw new RunError('too many runs are waiting for an approval');
         }
         this.checkLimits(owner, waiting);
+        if (spec.up) {
+            this.checkUpLimits(owner);
+            this.checkHosts(owner, spec.up);
+        }
         const run = {
             ...spec,
             id: `run_${(0, crypto_1.randomBytes)(12).toString('base64url')}`,
-            owner: { iss: owner.iss, sub: owner.sub },
+            owner: {
+                iss: owner.iss,
+                sub: owner.sub,
+                ...(spec.up?.sleepTtl ? { groups: [...(owner.groups ?? [])], claims: { ...(owner.claims ?? {}) } } : {}),
+            },
             state: waiting ? 'awaiting-approval' : 'queued',
             createdAt: this.timestamp(),
         };
@@ -179,30 +200,79 @@ class RunRegistry {
             .map((entry) => this.copy(entry.run))
             .reverse();
     }
-    // A queued run is cancelled at once; a running one when its executor stops.
+    // A queued run is cancelled at once; a running one when its executor stops. A service run
+    // that is up stops its services, and says who stopped it.
     cancel(owner, id, canUse) {
         const entry = this.find(owner, id, canUse);
         if (!entry) {
             return undefined;
         }
-        this.stop(entry);
+        this.stop(entry, undefined, { iss: owner.iss, sub: owner.sub });
         return this.copy(entry.run);
     }
+    /**
+     * Moves the end of a service run that is up (or asleep) to `ttl` (ms) from now, at most
+     * the server's maxTtl, and never sooner than it was. Its owner may, and whoever `mayOpen`
+     * says may open its endpoints.
+     */
+    extend(caller, id, ttl, mayOpen) {
+        const entry = this.entries.get(id);
+        if (!entry || !(this.owns(caller, entry) || (entry.run.up && mayOpen?.(this.copy(entry.run))))) {
+            return { status: 'not-found' };
+        }
+        // up, asleep, or woken and starting again
+        if (!entry.run.upAt || isFinal(entry.run.state)) {
+            return { status: 'not-up', state: entry.run.state };
+        }
+        // never sooner than it was: who wants it gone stops it
+        const ms = Math.min(ttl, this.options.up?.maxTtl ?? ttl);
+        const expiresAt = Math.max(Date.parse(entry.run.expiresAt ?? '') || 0, this.now() + ms);
+        entry.run.expiresAt = new Date(expiresAt).toISOString();
+        this.record(entry, { type: 'extended', by: { iss: caller.iss, sub: caller.sub }, expiresAt: entry.run.expiresAt });
+        this.expireLater(entry);
+        return { status: 'ok', run: this.copy(entry.run) };
+    }
+    /**
+     * The service run an endpoint host belongs to: the newest that is up, so one that replaces
+     * a run of the same name takes its hosts over as soon as it is up; else the newest that
+     * had it, for a page that says how it stands. With the address of the endpoint when up.
+     */
+    endpoint(host) {
+        let found;
+        for (const entry of this.entries.values()) {
+            const endpoint = entry.run.up?.endpoints.find((candidate) => candidate.host === host);
+            if (endpoint && (entry.run.state === 'up' || found?.entry.run.state !== 'up')) {
+                found = { entry, endpoint };
+            }
+        }
+        if (!found) {
+            return undefined;
+        }
+        const { entry, endpoint } = found;
+        const target = entry.run.state === 'up'
+            ? entry.targets?.find((candidate) => candidate.service === endpoint.service && candidate.port === endpoint.port)
+            : undefined;
+        return { run: this.copy(entry.run), endpoint: { ...endpoint }, ...(target ? { target } : {}) };
+    }
     // The run that holds a sandbox: the oldest on it that is not over. It holds it from
-    // when it is submitted, so the files are not moved under what an approver saw.
+    // when it is submitted, so the files are not moved under what an approver saw. A service
+    // run that is up works on a copy of the files, and holds it no longer.
     holder(sandbox) {
-        return [...this.entries.values()].find((entry) => onSandbox(entry.run, sandbox) && !isFinal(entry.run.state))?.run
-            .id;
+        return [...this.entries.values()].find((entry) => onSandbox(entry.run, sandbox) &&
+            !isFinal(entry.run.state) &&
+            entry.run.state !== 'up' &&
+            entry.run.state !== 'asleep')?.run.id;
     }
     // Cancels every run on a sandbox and resolves when none of them runs anymore.
     async stopSandbox(sandbox, reason) {
         const on = () => [...this.entries.values()].filter((entry) => onSandbox(entry.run, sandbox));
         on().forEach((entry) => this.stop(entry, reason));
-        while (on().some((entry) => entry.run.state === 'running')) {
+        while (on().some((entry) => entry.run.state === 'running' || entry.run.state === 'up')) {
             await new Promise((resolve) => setTimeout(resolve, 20));
         }
     }
-    stop(entry, reason) {
+    stop(entry, reason, by) {
+        var _a, _b;
         if (entry.run.state === 'queued') {
             this.queue.splice(this.queue.indexOf(entry), 1);
             this.finish(entry, 'cancelled', reason);
@@ -213,6 +283,15 @@ class RunRegistry {
         else if (entry.run.state === 'running') {
             entry.abort.abort();
         }
+        else if (entry.run.state === 'up') {
+            (_a = entry.run).stoppedBy ?? (_a.stoppedBy = by ?? 'replaced');
+            entry.sleeping = false;
+            entry.abort.abort();
+        }
+        else if (entry.run.state === 'asleep') {
+            (_b = entry.run).stoppedBy ?? (_b.stoppedBy = by ?? 'replaced');
+            this.finish(entry, 'succeeded');
+        }
     }
     // The runs that wait for an approval, of anybody, oldest first. Who may see
     // them is for the caller to decide.
@@ -289,18 +368,20 @@ class RunRegistry {
         return { events, finished, stop: () => entry.listeners.delete(listener) };
     }
     /**
-     * The run once it is over, or as it is after `ms` or when `signal` aborts, for clients
-     * that can not hold a stream of events open (a tool call of an agent).
+     * The run once it is over (a service run: once it is up), or as it is after `ms` or when
+     * `signal` aborts, for clients that can not hold a stream of events open (a tool call of
+     * an agent).
      */
     async wait(owner, id, ms, canUse, signal) {
         const entry = this.find(owner, id, canUse);
         if (!entry) {
             return undefined;
         }
-        if (!isFinal(entry.run.state) && ms > 0 && !this.closed && !signal?.aborted) {
+        if (!isFinal(entry.run.state) && entry.run.state !== 'up' && ms > 0 && !this.closed && !signal?.aborted) {
             await new Promise((resolve) => {
                 const listener = (event) => {
-                    if (event.type === 'state' && isFinal(event.state)) {
+                    // a service run is there once it is up
+                    if (event.type === 'state' && (isFinal(event.state) || event.state === 'up')) {
                         done();
                     }
                 };
@@ -335,10 +416,19 @@ class RunRegistry {
     async close() {
         this.closed = true;
         for (const entry of this.queue.splice(0)) {
-            this.finish(entry, 'cancelled');
+            if (this.maySleep(entry) && this.options.store) {
+                // a woken service run sleeps again, the next start keeps it
+                this.sleep(entry);
+            }
+            else {
+                this.finish(entry, 'cancelled');
+            }
         }
         for (const entry of this.entries.values()) {
-            if (entry.run.state === 'awaiting-approval') {
+            if (entry.run.state === 'asleep') {
+                this.clearTimer(entry);
+            }
+            else if (entry.run.state === 'awaiting-approval') {
                 if (this.options.store) {
                     // the record keeps it waiting, the next start takes it back
                     this.clearTimer(entry);
@@ -353,11 +443,15 @@ class RunRegistry {
             done();
         }
         for (const entry of this.entries.values()) {
-            if (entry.run.state === 'running') {
+            if (entry.run.state === 'running' || entry.run.state === 'up') {
+                // a service run that may sleep is asleep in its record, the next request after the
+                // start wakes it
+                entry.sleeping = this.maySleep(entry) && !!this.options.store;
                 entry.abort.abort();
             }
         }
-        while (this.running > 0) {
+        while (this.running > 0 ||
+            [...this.entries.values()].some((entry) => entry.run.state === 'running' || entry.run.state === 'up')) {
             await new Promise((resolve) => setTimeout(resolve, 10));
         }
         // the runs that were stopped are written down before the server is gone
@@ -388,6 +482,8 @@ class RunRegistry {
     copy(run) {
         return {
             ...run,
+            ...(run.up ? { up: JSON.parse(JSON.stringify(run.up)) } : {}),
+            ...(run.stoppedBy && typeof run.stoppedBy === 'object' ? { stoppedBy: { ...run.stoppedBy } } : {}),
             tasks: [...run.tasks],
             source: { ...run.source },
             options: JSON.parse(JSON.stringify(run.options)),
@@ -415,6 +511,37 @@ class RunRegistry {
             throw new QuotaError(`you have ${limits.queued} runs in the queue, wait for one to start or cancel one`);
         }
     }
+    // Service runs that are not over, of the caller and of everybody, against what may be:
+    // one that is asleep counts, it wakes without asking for a place.
+    checkUpLimits(owner) {
+        const limits = this.options.up;
+        if (!limits) {
+            return;
+        }
+        const open = [...this.entries.values()].filter((entry) => entry.run.up && !isFinal(entry.run.state));
+        if (open.filter((entry) => this.owns(owner, entry)).length >= limits.perCaller) {
+            throw new QuotaError(`you have ${limits.perCaller} service runs starting, up or asleep, stop one or wait for its end`);
+        }
+        if (open.length >= limits.total) {
+            throw new RunError(`the server has ${limits.total} service runs starting, up or asleep, try again later`);
+        }
+    }
+    // A host is one run's, or a run's that replaces it (same owner and name) once it is up:
+    // two labels that come out the same are refused, they would reach each other's services.
+    checkHosts(owner, up) {
+        const hosts = new Set(up.endpoints.map((endpoint) => endpoint.host));
+        for (const entry of this.entries.values()) {
+            if (isFinal(entry.run.state) || !entry.run.up) {
+                continue;
+            }
+            const taken = entry.run.up.endpoints.find((endpoint) => hosts.has(endpoint.host));
+            if (taken && !(this.owns(owner, entry) && entry.run.up.name === up.name)) {
+                const error = new RunError(`the address ${taken.host} is another service run's, pick another name`);
+                error.status = 409;
+                throw error;
+            }
+        }
+    }
     runningOf(owner) {
         return [...this.entries.values()].filter((entry) => entry.run.state === 'running' && this.owns(owner, entry)).length;
     }
@@ -444,7 +571,10 @@ class RunRegistry {
             const [entry] = this.queue.splice(index, 1);
             this.running++;
             this.execute(entry).finally(() => {
-                this.running--;
+                // a service run that is up gave its place back already
+                if (!entry.released) {
+                    this.running--;
+                }
                 this.dispatch();
             });
         }
@@ -453,27 +583,188 @@ class RunRegistry {
         entry.run.state = 'running';
         entry.run.startedAt = this.timestamp();
         this.record(entry, { type: 'state', state: 'running' });
+        // a service run gets this long to be up, then its ttl counts
         const limit = this.options.maxDuration;
-        const timer = limit
+        entry.durationTimer = limit
             ? setTimeout(() => {
                 entry.timedOut = true;
                 entry.abort.abort();
             }, limit)
             : undefined;
-        timer?.unref();
+        entry.durationTimer?.unref();
         try {
             await this.options.executor.execute(this.copy(entry.run), {
                 emit: (event) => this.record(entry, event),
                 signal: entry.abort.signal,
+                ...(entry.run.up ? { up: (targets) => this.markUp(entry, targets) } : {}),
             });
-            this.finish(entry, entry.timedOut ? 'failed' : entry.abort.signal.aborted ? 'cancelled' : 'succeeded', entry.timedOut ? this.tooLong() : undefined);
+            if (entry.sleeping) {
+                this.sleep(entry);
+            }
+            else if (entry.run.upAt) {
+                // services stay until they are stopped: ones that stopped on their own failed
+                this.endServices(entry, 'the services stopped on their own');
+            }
+            else {
+                this.finish(entry, entry.timedOut ? 'failed' : entry.abort.signal.aborted ? 'cancelled' : 'succeeded', entry.timedOut ? this.tooLong() : undefined);
+            }
         }
         catch (e) {
-            const cancelled = entry.abort.signal.aborted && !entry.timedOut;
-            this.finish(entry, cancelled ? 'cancelled' : 'failed', entry.timedOut ? this.tooLong() : cancelled ? undefined : e instanceof Error ? e.message : String(e));
+            const message = e instanceof Error ? e.message : String(e);
+            if (entry.sleeping) {
+                this.sleep(entry);
+            }
+            else if (entry.run.upAt) {
+                this.endServices(entry, message);
+            }
+            else {
+                const cancelled = entry.abort.signal.aborted && !entry.timedOut;
+                this.finish(entry, cancelled ? 'cancelled' : 'failed', entry.timedOut ? this.tooLong() : cancelled ? undefined : message);
+            }
         }
         finally {
-            clearTimeout(timer);
+            clearTimeout(entry.durationTimer);
+        }
+    }
+    // The services of a service run are ready: its ttl counts from now, its place of
+    // maxConcurrent goes to the next run, and an older run of its owner with the same name
+    // stops, so its hosts are this run's.
+    markUp(entry, targets) {
+        const spec = entry.run.up;
+        if (!spec || entry.run.state !== 'running' || entry.abort.signal.aborted) {
+            return;
+        }
+        clearTimeout(entry.durationTimer);
+        entry.durationTimer = undefined;
+        entry.targets = targets.map((target) => ({ ...target, address: { ...target.address } }));
+        entry.run.state = 'up';
+        // the ttl counts from the first time, not from a wake
+        if (!entry.run.upAt) {
+            entry.run.upAt = this.timestamp();
+            entry.run.expiresAt = new Date(this.now() + spec.ttl).toISOString();
+        }
+        if (spec.sleepTtl) {
+            entry.run.sleepsAt = new Date(this.now() + spec.sleepTtl).toISOString();
+        }
+        this.record(entry, { type: 'state', state: 'up' });
+        this.expireLater(entry);
+        entry.released = true;
+        this.running--;
+        const entries = [...this.entries.values()];
+        for (const other of entries.slice(0, entries.indexOf(entry))) {
+            if (other.run.up?.name === spec.name &&
+                this.owns(entry.run.owner, other) &&
+                !isFinal(other.run.state) &&
+                other.run.state !== 'awaiting-approval') {
+                other.run.stoppedBy = 'replaced';
+                this.stop(other, `replaced by ${entry.run.id}`);
+            }
+        }
+        this.dispatch();
+    }
+    // Stops a service run at its expiry, and puts one that is up to sleep at its sleepsAt. A
+    // timer can't wait longer than about 24 days, so a longer one looks again then.
+    expireLater(entry) {
+        this.clearTimer(entry);
+        const expiresAt = Date.parse(entry.run.expiresAt ?? '');
+        const sleepsAt = entry.run.state === 'up' ? Date.parse(entry.run.sleepsAt ?? '') : NaN;
+        const next = Number.isFinite(sleepsAt) ? Math.min(expiresAt, sleepsAt) : expiresAt;
+        entry.timer = setTimeout(() => {
+            var _a;
+            if (isFinal(entry.run.state)) {
+                return;
+            }
+            if (Date.parse(entry.run.expiresAt ?? '') <= this.now()) {
+                (_a = entry.run).stoppedBy ?? (_a.stoppedBy = 'ttl');
+                entry.sleeping = false;
+                if (entry.run.state === 'asleep') {
+                    this.finish(entry, 'succeeded');
+                }
+                else {
+                    this.stop(entry);
+                }
+                return;
+            }
+            if (entry.run.state === 'up' && Date.parse(entry.run.sleepsAt ?? '') <= this.now()) {
+                entry.sleeping = true;
+                entry.abort.abort();
+                return;
+            }
+            this.expireLater(entry);
+        }, Math.max(0, Math.min(next - this.now(), 2 ** 31 - 1)));
+        entry.timer.unref();
+    }
+    // Whether a service run sleeps instead of ending: it has a sleepTtl, its ttl has not
+    // ended, and it starts from a commit (the files of a sandbox change under it).
+    maySleep(entry) {
+        return (!!entry.run.up?.sleepTtl &&
+            !!entry.run.upAt &&
+            entry.run.source.type === 'ref' &&
+            Date.parse(entry.run.expiresAt ?? '') > this.now());
+    }
+    // The services stopped, the run waits for a request to wake it, or for its end.
+    sleep(entry) {
+        entry.sleeping = false;
+        entry.targets = undefined;
+        entry.run.state = 'asleep';
+        delete entry.run.sleepsAt;
+        this.record(entry, { type: 'state', state: 'asleep' });
+        this.expireLater(entry);
+    }
+    /**
+     * A request to an endpoint of a service run: one that is up stays up for its sleepTtl from
+     * now, one that is asleep is started again (it holds its place of runs.up.limits while it
+     * sleeps), if `mayWake` finds its owner may still run it: else it ends. Answers why it was
+     * not woken, or undefined.
+     */
+    visit(id, mayWake) {
+        const entry = this.entries.get(id);
+        if (!entry?.run.up) {
+            return undefined;
+        }
+        const sleepTtl = entry.run.up.sleepTtl;
+        if (entry.run.state === 'up' && sleepTtl) {
+            // only kept in memory: a server that stops puts the run to sleep anyway
+            entry.run.sleepsAt = new Date(this.now() + sleepTtl).toISOString();
+            return undefined;
+        }
+        if (entry.run.state !== 'asleep' || this.closed) {
+            return undefined;
+        }
+        if (!this.maySleep(entry)) {
+            this.expireLater(entry);
+            return 'its time ran out';
+        }
+        const refused = mayWake?.(this.copy(entry.run));
+        if (refused) {
+            this.finish(entry, 'failed', refused);
+            return refused;
+        }
+        this.clearTimer(entry);
+        entry.abort = new AbortController();
+        entry.released = false;
+        entry.timedOut = false;
+        entry.run.wakes = (entry.run.wakes ?? 0) + 1;
+        entry.run.state = 'queued';
+        this.record(entry, { type: 'state', state: 'queued' });
+        this.queue.push(entry);
+        this.dispatch();
+        return undefined;
+    }
+    // The end of a run that was up: stopped by somebody, its ttl or a newer run it succeeded,
+    // stopped by the server going away it failed, as its record says after a restart.
+    endServices(entry, failure) {
+        if (entry.timedOut) {
+            this.finish(entry, 'failed', this.tooLong());
+        }
+        else if (!entry.abort.signal.aborted) {
+            this.finish(entry, 'failed', failure);
+        }
+        else if (entry.run.stoppedBy) {
+            this.finish(entry, 'succeeded');
+        }
+        else {
+            this.finish(entry, 'failed', 'the server stopped while the services were up, start them again');
         }
     }
     tooLong() {
@@ -481,6 +772,8 @@ class RunRegistry {
     }
     finish(entry, state, error) {
         this.clearTimer(entry);
+        clearTimeout(entry.durationTimer);
+        entry.targets = undefined;
         entry.run.state = state;
         entry.run.finishedAt = this.timestamp();
         if (error) {
@@ -579,7 +872,7 @@ class RunRegistry {
         }
         // what a restart must not lose: the runs that wait, and how a run ended. A running
         // run is not kept up to date, it fails with the server.
-        if (event.type === 'state' || event.type === 'approval') {
+        if (event.type === 'state' || event.type === 'approval' || event.type === 'extended') {
             this.persist(entry);
         }
     }