@zenera/server 1.2.1 → 1.2.2

package/README.md

@@ -28,10 +28,10 @@ Requires Node.js 24+ and the [Zenera CLI](../cli/README.md).
 
 ## Quickstart
 
-From a configured project:
+For local testing, run the server in the configured project's agent container:
 
 ```sh
-zen server start --insecure --user ann
+zen server start --insecure --container --user andrey
 ```
 
 Open the printed HTTP `/chat` link to list sessions, send messages, follow active
@@ -43,17 +43,22 @@ impersonate any user. API requests still need a JWT, but signatures are not chec
 The test page is available only in this mode. Its token stays in memory; after a
 reload, reopen the printed link or enter a development token again.
 
-To run the server in the project's agent image:
+To show full image build logs:
 
 ```sh
-zen server start --container --insecure --user ann
-zen server start --container --insecure --user ann --verbose # full build logs
+zen server start --insecure --container --user andrey --verbose
 ```
 
 Podman builds and caches the image. Default build output shows a spinner, elapsed
 time, and the latest line; warnings remain visible. Use `--rebuild` to rebuild.
 Press Ctrl+C to stop gracefully and save active runs.
 
+To run the server directly on the host instead, omit `--container`:
+
+```sh
+zen server start --insecure --user andrey
+```
+
 ## Secure setup
 
 A trusted caller, usually Zenera-app, authenticates users, checks project access,
@@ -95,6 +100,7 @@ it expires. The server does not perform user login or project-membership checks.
 | `zen server status`                | Inspect a running runtime; use `--token <jwt>` for secure mode. |
 | `zen server keygen --out <prefix>` | Generate private/public signing keys.                           |
 | `zen server token --user <id>`     | Print an **unsigned** development JWT for insecure mode.        |
+| `zen server build`                 | Build (and push) a release image; see below.                    |
 
 Test an API without the browser:
 
@@ -134,6 +140,61 @@ credential files. The gateway must be reachable from the container; host
 Set `sandbox.keys: false` to withhold credentials from agent-written commands,
 while allowing the engine to use them for model calls.
 
+## Release images
+
+`zen server build` bakes a project into an immutable image for Kubernetes and
+prints its digest (or the local tag when not pushed):
+
+```sh
+zen server build --project p1 --release r42 \
+  --push registry.local:5000/agent-p1:r42 [--authfile auth.json] [--insecure-registry]
+```
+
+The image is the agent image plus the project at `/project`, its memory seed at
+`/seed` (`--memory-seed <dir>` / `--no-memory-seed`), `ZEN_RELEASE` and
+`ZEN_PROJECT_ID`. The release id defaults to `r-<content hash>`; `--json` prints
+`{release, image, agentImage, base, built, ref, digest}`.
+
+Never baked in: `.env*` (except `.env.example`), `sessions/`, `.git`,
+`node_modules`, `.tmp/`, or the live memory directory. Deployment settings (JWT public key, MCP
+plugin, Bifrost URL/key, storage) are passed at deploy time via environment
+variables or a ConfigMap. Use `--add <src>:<dest>` (repeatable) to add large
+content such as plugins, e.g. `--add ./plugins:/opt/zenera/plugins`.
+
+## Many servers per project
+
+Several servers can serve one project when users' data lives in a store rather
+than on one disk. Name a store module with `ZEN_STATE_STORE` (ZeneraLite ships an
+S3 store as `state.mjs`). The server then:
+
+- loads a user's data before their first request, and a chat's files when it is opened;
+- saves them after every run, after deletions, every `ZEN_STATE_CHECKPOINT_SECONDS`
+  (300) while runs go on, when released, and on shutdown;
+- refuses a token whose `gen` (lease generation) is older than the one it holds,
+  and drops its copy when the store reports that a newer lease saved the user.
+
+The host manages users through `/api/admin` with tokens for its own subject
+(`ZEN_SERVER_SYSTEM_SUBJECT`, default `zenera-app`):
+
+| Route                                 | Use                                                       |
+| ------------------------------------- | --------------------------------------------------------- |
+| `GET /api/admin/users[?usage=1]`      | Users here: lease, runs, unsaved changes, disk use.       |
+| `POST /api/admin/users/:sub/release`  | Save and let go of one user; `409 busy` while runs go on. |
+| `POST /api/admin/drain[?timeout=600]` | No new users; release each one as their runs end.         |
+| `POST /api/admin/undrain`             | Take new users again.                                     |
+
+Soft limits for many users per pod:
+
+| Variable                   | Limit                                                      |
+| -------------------------- | ---------------------------------------------------------- |
+| `ZEN_RUNTIME_USER_DISK_MB` | Per-user disk; over it, no new runs and running ones stop. |
+| `ZEN_RUNTIME_NICE`         | Lower CPU priority for commands (default 10).              |
+| `ZEN_RUNTIME_MAX_FILE_MB`  | Largest file a command may write.                          |
+| `ZEN_RUNTIME_TMP_MB`       | Size of each command's `/tmp` (it uses memory).            |
+
+These are soft. Hard limits come from the pod: CPU, memory, and a fixed-size
+volume for `/state`.
+
 ## HTTP API
 
 Session routes operate only on the JWT subject's own sessions.

package/dist/auth.d.ts

@@ -8,6 +8,8 @@ export interface CallerClaims extends JWTPayload {
     role?: string;
     email?: string;
     name?: string;
+    /** the host's lease generation for this user on this server (userState.ts) */
+    gen?: number;
 }
 export declare function verifyCaller(req: Request): Promise<CallerClaims>;
 export declare function requireCaller(): (req: Request, res: Response, next: NextFunction) => Promise<void>;

package/dist/config.d.ts

@@ -1,5 +1,6 @@
 import type { SandboxSetup, ZenPlugin } from '@zenera/engine';
-import { type SandboxFactory } from '@zenera/sandbox';
+import { type CommandLimits, type SandboxFactory } from '@zenera/sandbox';
+import type { UserStateStore } from './userState.ts';
 export type ServerAuth = 
 /** EdDSA tokens signed per request by the host (zenera-app), checked with its public key */
 {
@@ -50,6 +51,19 @@ export interface ServerSettings {
     maxUserRuns: number;
     maxRunMinutes: number;
     runLogMaxBytes: number;
+    /**
+     * Where users' data lives beyond this disk (userState.ts): loaded before their first request,
+     * saved at safe points, released on the host's word. Unset: only here (`localStore`).
+     */
+    stateStore?: UserStateStore;
+    /** how often users with runs going (or a failed save) are saved again; 0: never */
+    checkpointSeconds: number;
+    /** the `sub` of the host's own calls, the only one /api/admin answers */
+    systemSubject: string;
+    /** soft limit on one user's folder; over it, no new runs and running ones stop; 0: none */
+    userDiskQuotaBytes: number;
+    /** what each bwrap command is held to: priority, largest file, size of its /tmp; 0: none */
+    commandLimits: CommandLimits;
 }
 /**
  * bwrap inside the agent image, which ships it at /opt/zenera/bin/bwrap; podman anywhere else,

package/dist/config.js

@@ -30,6 +30,10 @@ export const DEFAULTS = {
     maxUserRuns: 2,
     maxRunMinutes: 120,
     runLogMaxBytes: 50 * 1024 * 1024,
+    checkpointSeconds: 300,
+    systemSubject: DEFAULT_ISSUER,
+    userDiskQuotaBytes: 0,
+    commandLimits: { nice: 10, fileSizeBytes: 0, tmpBytes: 0 },
 };
 let current;
 export function configure(settings) {

package/dist/http.js

@@ -21,6 +21,10 @@ export function errorHandler(err, req, res, _next) {
     const status = err instanceof HttpError ? err.status : (err?.status ?? 500);
     if (status >= 500)
         console.error(`${req.method} ${req.originalUrl}:`, err);
-    res.status(status).json({ error: errorMessage(err) });
+    const code = err?.code;
+    res.status(status).json({
+        error: errorMessage(err),
+        ...(err instanceof HttpError && typeof code === 'string' ? { code } : {}),
+    });
 }
 //# sourceMappingURL=http.js.map

package/dist/index.d.ts

@@ -4,7 +4,7 @@ export { requireCaller, unsignedToken, verifyCaller, type CallerClaims } from '.
 export { identify, runUser, userFolder, type Identity } from './identity.ts';
 export { LAYOUT_VERSION, LayoutError, openLayout, type Layout } from './layout.ts';
 export { privateStore, memoryDir, memorySeed, userDir, userMemory, SEED_FILE, usesMemory, imageReady, openEngineFor, statusFor, errorMessage, asHttpError, type ChatStore, type ChatThread, } from './store.ts';
-export { runsRouter, runDebugRouter, activeRuns, currentRun, stopAllRuns } from './runs.ts';
+export { runsRouter, runDebugRouter, activeRuns, currentRun, ownerRuns, stopAllRuns, stopOwnerRuns, } from './runs.ts';
 export { historyRouter, toMessages } from './history.ts';
 export { traceRouter, openTrace, stepParam } from './trace.ts';
 export { memoryRouter } from './memory.ts';
@@ -13,6 +13,7 @@ export { RunLog, compact, readRunLog, threadRunLogs } from './runLog.ts';
 export * from './runRecords.ts';
 export { applyProjectEnv, withoutProjectEnv } from './projectEnv.ts';
 export { HttpError, errorHandler, notFound } from './http.ts';
-export { envRunPlugins } from './plugins.ts';
+export { envRunPlugins, envServerLimits, envStateStore, type StateStoreOptions, } from './plugins.ts';
+export { localStore, StaleGenerationError, USER_LAYOUT, userStates, userUsageBytes, type CheckpointReason, type UserInfo, type UserRef, type UserSnapshot, type UserStateStore, } from './userState.ts';
 export { serverStatus, startupErrors, type ServerStatus } from './status.ts';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -6,7 +6,7 @@ export { requireCaller, unsignedToken, verifyCaller } from "./auth.js";
 export { identify, runUser, userFolder } from "./identity.js";
 export { LAYOUT_VERSION, LayoutError, openLayout } from "./layout.js";
 export { privateStore, memoryDir, memorySeed, userDir, userMemory, SEED_FILE, usesMemory, imageReady, openEngineFor, statusFor, errorMessage, asHttpError, } from "./store.js";
-export { runsRouter, runDebugRouter, activeRuns, currentRun, stopAllRuns } from "./runs.js";
+export { runsRouter, runDebugRouter, activeRuns, currentRun, ownerRuns, stopAllRuns, stopOwnerRuns, } from "./runs.js";
 export { historyRouter, toMessages } from "./history.js";
 export { traceRouter, openTrace, stepParam } from "./trace.js";
 export { memoryRouter } from "./memory.js";
@@ -15,6 +15,7 @@ export { RunLog, compact, readRunLog, threadRunLogs } from "./runLog.js";
 export * from "./runRecords.js";
 export { applyProjectEnv, withoutProjectEnv } from "./projectEnv.js";
 export { HttpError, errorHandler, notFound } from "./http.js";
-export { envRunPlugins } from "./plugins.js";
+export { envRunPlugins, envServerLimits, envStateStore, } from "./plugins.js";
+export { localStore, StaleGenerationError, USER_LAYOUT, userStates, userUsageBytes, } from "./userState.js";
 export { serverStatus, startupErrors } from "./status.js";
 //# sourceMappingURL=index.js.map

package/dist/plugins.d.ts

@@ -1,5 +1,24 @@
 import type { ZenPlugin } from '@zenera/engine';
+import { type ServerSettings } from './config.ts';
+import { USER_LAYOUT, type UserStateStore } from './userState.ts';
 export declare function envRunPlugins(env?: NodeJS.ProcessEnv): (run: {
     mcpToken: string | null;
 }) => Promise<ZenPlugin[]>;
+/** What a state store module is given: who it serves, and the user folder's layout. */
+export interface StateStoreOptions {
+    projectId: string;
+    root: string;
+    layout: typeof USER_LAYOUT;
+    env: NodeJS.ProcessEnv;
+    log: {
+        info(message: string): void;
+        warn(message: string): void;
+    };
+}
+/** The store `ZEN_STATE_STORE` names, made and checked; a broken one stops the server starting. */
+export declare function envStateStore(env: NodeJS.ProcessEnv, at: {
+    projectId: string;
+    root: string;
+}): Promise<UserStateStore | undefined>;
+export declare function envServerLimits(env: NodeJS.ProcessEnv): Pick<ServerSettings, 'checkpointSeconds' | 'systemSubject' | 'userDiskQuotaBytes' | 'commandLimits'>;
 //# sourceMappingURL=plugins.d.ts.map

package/dist/plugins.js

@@ -7,6 +7,8 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
     return path;
 };
 import { pathToFileURL } from 'node:url';
+import { DEFAULTS } from "./config.js";
+import { USER_LAYOUT } from "./userState.js";
 export function envRunPlugins(env = process.env) {
     const plugin = (env.ZEN_PLUGINS ?? '')
         .split(',')
@@ -26,4 +28,49 @@ export function envRunPlugins(env = process.env) {
         return mcpPlugin ? [mcpPlugin({ brokerUrl, token: mcpToken })] : [];
     };
 }
+/** The store `ZEN_STATE_STORE` names, made and checked; a broken one stops the server starting. */
+export async function envStateStore(env, at) {
+    const path = env.ZEN_STATE_STORE?.trim();
+    if (!path)
+        return undefined;
+    const mod = (await import(__rewriteRelativeImportExtension(pathToFileURL(path).href)));
+    if (typeof mod.userStateStore !== 'function') {
+        throw new Error(`${path} exports no userStateStore(options)`);
+    }
+    const store = await mod.userStateStore({
+        ...at,
+        layout: USER_LAYOUT,
+        env,
+        log: {
+            info: (message) => console.log(`[state] ${message}`),
+            warn: (message) => console.warn(`[state] ${message}`),
+        },
+    });
+    const ok = store &&
+        typeof store.name === 'string' &&
+        typeof store.durable === 'boolean' &&
+        typeof store.load === 'function' &&
+        typeof store.checkpoint === 'function';
+    if (!ok)
+        throw new Error(`${path}: userStateStore() did not return a store`);
+    return store;
+}
+function whole(text, fallback, max = Number.MAX_SAFE_INTEGER) {
+    const n = Number(text?.trim());
+    return text?.trim() && Number.isInteger(n) && n >= 0 ? Math.min(n, max) : fallback;
+}
+const MB = 2 ** 20;
+export function envServerLimits(env) {
+    const limits = DEFAULTS.commandLimits;
+    return {
+        checkpointSeconds: whole(env.ZEN_STATE_CHECKPOINT_SECONDS, DEFAULTS.checkpointSeconds),
+        systemSubject: env.ZEN_SERVER_SYSTEM_SUBJECT?.trim() || DEFAULTS.systemSubject,
+        userDiskQuotaBytes: whole(env.ZEN_RUNTIME_USER_DISK_MB, 0) * MB,
+        commandLimits: {
+            nice: whole(env.ZEN_RUNTIME_NICE, limits.nice ?? 0, 19),
+            fileSizeBytes: whole(env.ZEN_RUNTIME_MAX_FILE_MB, 0) * MB,
+            tmpBytes: whole(env.ZEN_RUNTIME_TMP_MB, 0) * MB,
+        },
+    };
+}
 //# sourceMappingURL=plugins.js.map

package/dist/runs.d.ts

@@ -4,6 +4,14 @@ export declare function activeRuns(): number;
 export declare function currentRun(t: ChatThread): {
     runId: string;
 } | null;
+/** One owner's runs going now, with the thread each is on and when it ends. */
+export declare function ownerRuns(owner: string): {
+    threadId: string;
+    runId: string;
+    done: Promise<void>;
+}[];
+/** Ends one owner's runs as interrupted, saying why; resolves once they have saved. */
+export declare function stopOwnerRuns(owner: string, message: string): Promise<void>;
 /** Runs may start again: a new server in a process whose previous one stopped. */
 export declare function acceptRuns(): void;
 export declare function stopAllRuns(timeoutMs: number): Promise<void>;

package/dist/runs.js

@@ -32,6 +32,19 @@ export function currentRun(t) {
     const run = runs.get(t.id);
     return run && run.owner === t.owner ? { runId: run.runId } : null;
 }
+/** One owner's runs going now, with the thread each is on and when it ends. */
+export function ownerRuns(owner) {
+    return [...runs.entries()]
+        .filter(([, run]) => run.owner === owner)
+        .map(([threadId, run]) => ({ threadId, runId: run.runId, done: run.done }));
+}
+/** Ends one owner's runs as interrupted, saying why; resolves once they have saved. */
+export async function stopOwnerRuns(owner, message) {
+    const mine = [...runs.values()].filter((run) => run.owner === owner);
+    for (const run of mine)
+        run.controller.abort(new RunStop('interrupted', message));
+    await Promise.all(mine.map((run) => run.done));
+}
 // The newest user message of an AG-UI RunAgentInput; the thread's history is the runtime's own.
 function inputOf(body) {
     const messages = body?.messages;
@@ -236,6 +249,7 @@ async function execute(store, t, run, turn, memory, user, mcpToken) {
         }
         await log.end().catch((err) => console.error(`run ${runId}: saving its log:`, err));
         runs.delete(t.id);
+        store.changed?.(t, 'run-end');
     }
 }
 /** Runs may start again: a new server in a process whose previous one stopped. */
@@ -260,6 +274,8 @@ export function runsRouter(store) {
         const resume = resumeOf(req.body);
         const turn = resume ? { resume } : { message: inputOf(req.body) };
         const runId = ID.test(req.body?.runId ?? '') ? String(req.body.runId) : randomUUID();
+        // Over its storage quota, a user gets no new run (checked before the slot is taken).
+        await store.admit?.(t);
         if (stopping) {
             res.set('Retry-After', '5');
             throw new HttpError(503, 'the server is stopping, try again shortly');
@@ -345,6 +361,7 @@ export function runsRouter(store) {
             await run.done;
         }
         await store.removeThread(t);
+        store.changed?.(t, 'changed');
         res.status(204).end();
     });
     return router;

package/dist/server.d.ts

@@ -14,8 +14,11 @@ export interface AgentServer {
     app: Express;
     settings: ServerSettings;
     listen(on: ListenOn): Promise<Server>;
-    /** stops taking runs, ends the running ones as interrupted (they save), closes the listener */
-    stop(graceMs?: number): Promise<void>;
+    /**
+     * Stops taking runs, ends the running ones as interrupted (they save), saves every user
+     * (within `saveMs`), closes the listener.
+     */
+    stop(graceMs?: number, saveMs?: number): Promise<void>;
 }
 export declare function createServer(opts: ServerOptions): AgentServer;
 /** The env `zen server start --container` sets on the container it runs, port on loopback. */

package/dist/server.js

@@ -11,7 +11,11 @@
 //   GET  /api/sessions                    - the caller's sessions, newest activity first
 //   /api/sessions/:tid...                 - history, runs (start, follow, replay, cancel), delete
 //   /api/sessions/:tid/{runs,trace,memory}... with `debug` - the Test tab's views of a chat
-//   createServer  - the app, configured; `listen` starts it, `stop` ends runs and closes it
+//   /api/admin/{users,users/:sub/release,drain,undrain} - the host's view of this server's users
+// Every /api/sessions request first has the caller's data loaded (userState.ts): a store plugin
+// may keep users elsewhere and move them between servers; the token's `gen` is their lease.
+//   createServer  - the app, configured; `listen` recovers users and starts it, `stop` ends runs,
+//                   saves every user and closes it
 import express, {} from 'express';
 import { configure, DEFAULTS } from "./config.js";
 import { historyRouter } from "./history.js";
@@ -26,6 +30,7 @@ import { serverStatus, setListening } from "./status.js";
 import { privateStore } from "./store.js";
 import { sessionsRouter } from "./sessions.js";
 import { traceRouter } from "./trace.js";
+import { loadCaller, loadThread, userStateRouter, userStates } from "./userState.js";
 export function createServer(opts) {
     const { chatPage, ...runtime } = opts;
     const settings = { ...DEFAULTS, ...runtime, hidden: [...(opts.hidden ?? [])] };
@@ -58,6 +63,9 @@ export function createServer(opts) {
     app.get('/api/activity', (_req, res) => {
         res.json({ busy: activeRuns() > 0, runs: activeRuns() });
     });
+    app.use('/api/admin', userStateRouter());
+    app.use('/api/sessions', loadCaller);
+    app.use('/api/sessions/:tid', loadThread);
     app.use('/api/sessions', sessionsRouter());
     app.use('/api/sessions', runsRouter(privateStore));
     app.use('/api/sessions', historyRouter(privateStore));
@@ -72,19 +80,23 @@ export function createServer(opts) {
     return {
         app,
         settings,
-        listen: ({ port, host = '127.0.0.1' }) => new Promise((resolve, reject) => {
+        listen: async ({ port, host = '127.0.0.1' }) => {
             if (settings.auth.mode === 'insecure' && !LOOPBACK.has(host) && !insecureHere()) {
-                reject(new Error(`an insecure server listens on loopback only, not ${host}`));
-                return;
+                throw new Error(`an insecure server listens on loopback only, not ${host}`);
             }
-            setListening(`${host}:${port}`);
-            // Express 5 hands a listen error (EADDRINUSE) to this callback instead of throwing it.
-            server = app.listen(port, host, (err) => err ? reject(err) : resolve(server));
-        }),
-        stop: async (graceMs = 20_000) => {
+            // Users whose folders survived a restart are known (and saved) before anyone calls.
+            await userStates.start();
+            return new Promise((resolve, reject) => {
+                setListening(`${host}:${port}`);
+                // Express 5 hands a listen error (EADDRINUSE) to this callback instead of throwing it.
+                server = app.listen(port, host, (err) => err ? reject(err) : resolve(server));
+            });
+        },
+        stop: async (graceMs = 20_000, saveMs = 60_000) => {
             // No new connections, and no new runs on the ones still open, before the drain.
             const closed = new Promise((resolve) => server ? server.close(() => resolve()) : resolve());
             await stopAllRuns(graceMs);
+            await userStates.shutdown(saveMs);
             // Followers of runs that have ended, idle keep-alives: nothing left to answer.
             server?.closeAllConnections();
             await closed;

package/dist/status.d.ts

@@ -47,6 +47,13 @@ export interface ServerStatus {
     };
     /** what each user's memory starts as, when there is anything */
     memorySeed?: string;
+    /** where users' data is kept, and how many are on this server now */
+    state: {
+        store: string;
+        durable: boolean;
+        users: number;
+        draining: boolean;
+    };
 }
 /** Where the server listens, for the report. */
 export declare function setListening(where: string | undefined): void;

package/dist/status.js

@@ -11,6 +11,7 @@ import { createRequire } from 'node:module';
 import { settings } from "./config.js";
 import { LAYOUT_VERSION, openLayout } from "./layout.js";
 import { memorySeed } from "./store.js";
+import { userStates } from "./userState.js";
 const require = createRequire(import.meta.url);
 function versionOf(pkg) {
     try {
@@ -105,6 +106,12 @@ export async function serverStatus(env = process.env) {
         image,
         root,
         ...(memorySeed() ? { memorySeed: memorySeed() } : {}),
+        state: {
+            store: s.stateStore?.name ?? 'local',
+            durable: s.stateStore?.durable ?? false,
+            users: userStates.size,
+            draining: userStates.draining,
+        },
     };
 }
 /** Why the server must not start, if anything. An image that was never labelled is not one. */

package/dist/store.d.ts

@@ -25,6 +25,10 @@ export interface ChatStore {
     maxOwnerRuns: number;
     /** Why a run is refused when the owner has that many going. */
     ownerBusy: string;
+    /** Before a run takes its slot: throw an HttpError to refuse it (e.g. storage over quota). */
+    admit?(t: ChatThread): Promise<void>;
+    /** After a run ended or the thread was deleted: the owner's data changed. */
+    changed?(t: ChatThread, reason: 'run-end' | 'changed'): void;
 }
 /** Where one user's things are. */
 export declare function userDir(user: string): string;

package/dist/store.js

@@ -28,6 +28,7 @@ import { settings } from "./config.js";
 import { HttpError } from "./http.js";
 import { identify } from "./identity.js";
 import { applyProjectEnv } from "./projectEnv.js";
+import { userStates } from "./userState.js";
 /** Where one user's things are. */
 export function userDir(user) {
     return join(settings().root, 'users', user);
@@ -112,6 +113,8 @@ export const privateStore = {
         return settings().maxUserRuns;
     },
     ownerBusy: 'your other chats are still answering',
+    admit: (t) => userStates.admit(t.owner),
+    changed: (t, reason) => userStates.changed(t.owner, reason),
 };
 export function usesMemory() {
     try {
@@ -153,7 +156,7 @@ function sandboxFactory(s) {
         ...(existsSync(AGENT_APP_DIR) ? [AGENT_APP_DIR] : []),
         ...s.hidden,
     ];
-    return (opts) => new BwrapSandbox(opts, { hidden });
+    return (opts) => new BwrapSandbox(opts, { hidden, limits: s.commandLimits });
 }
 export async function openEngineFor(session, memoryDir, 
 // Who the run acts for; null is nobody, and stops the engine reading ZEN_USER_* from this process.

package/dist/userState.d.ts

@@ -0,0 +1,99 @@
+import { Router, type NextFunction, type Request, type Response } from 'express';
+/** Where things are in a user's folder, for stores that cannot import this package. */
+export declare const USER_LAYOUT: {
+    /** the user's memory graph; checkpoints carry a consistent copy instead (`memoryDir`) */
+    readonly memory: "memory";
+    /** one folder per chat thread */
+    readonly sessions: "sessions";
+    /** inside a session folder: its metadata, which the session list reads */
+    readonly sessionMeta: ".data/session.json";
+    /** the server's own record of the user's lease; never a store's to save */
+    readonly marker: ".zen-user.json";
+};
+export interface UserRef {
+    /** the user's folder name under `<root>/users/` */
+    id: string;
+    /** that folder: everything of theirs on this server */
+    dir: string;
+    /** the host's lease generation (the token's `gen`); undefined when the host does not lease */
+    generation?: number;
+}
+export type CheckpointReason = 'run-end' | 'changed' | 'periodic' | 'release' | 'shutdown' | 'recover';
+export interface UserSnapshot {
+    /** a consistent copy of the user's memory made for this call, removed after it; none: no memory */
+    memoryDir?: string;
+    /** threads with a run going: their files may change while they are read */
+    activeThreads: string[];
+}
+export interface UserStateStore {
+    readonly name: string;
+    /** true: the store keeps users elsewhere, so a released user's local copy is deleted */
+    readonly durable: boolean;
+    /** Before the user's first request here: fetch what they need into `user.dir`. */
+    load(user: UserRef): Promise<void>;
+    /** Before a request names one thread: fetch it if `load` left it for later. */
+    loadSession?(user: UserRef, threadId: string): Promise<void>;
+    /** Save the user's folder as it is now (memory from `snapshot.memoryDir`). */
+    checkpoint(user: UserRef, reason: CheckpointReason, snapshot: UserSnapshot): Promise<void>;
+    /** After the final checkpoint, before the local copy goes. */
+    release?(user: UserRef): Promise<void>;
+    /** On start, for users whose folders survived a restart of this process. */
+    recover?(users: UserRef[]): Promise<void>;
+    close?(): Promise<void>;
+}
+/** Thrown by a store when a newer lease generation owns the user: this copy must go. */
+export declare class StaleGenerationError extends Error {
+    readonly code = "stale_generation";
+}
+export declare const localStore: UserStateStore;
+type Status = 'loading' | 'ready' | 'releasing';
+export interface UserInfo {
+    id: string;
+    generation?: number;
+    status: Status;
+    activeRuns: number;
+    unsaved: boolean;
+    lastUsed: string;
+    lastCheckpointAt?: string;
+    lastError?: string;
+    usageBytes?: number;
+}
+declare class UserStates {
+    #private;
+    get draining(): boolean;
+    get size(): number;
+    /** The user's data, ready; their folder is theirs to use once this resolves. */
+    acquire(id: string, generation?: number): Promise<void>;
+    /** Before a request names a thread: the store fetches it if it left it for later. */
+    session(id: string, threadId: string): Promise<void>;
+    /** Something of the user's changed (a run ended, a thread went): save it. */
+    changed(id: string, reason?: CheckpointReason): void;
+    /** Saves the user now, after any save already queued; one at a time per user. */
+    checkpoint(id: string, reason: CheckpointReason): Promise<void>;
+    /** Saves the user for the last time and lets them go; refused while their runs go on. */
+    release(id: string): Promise<'released' | 'absent'>;
+    /** No new users; each one here released once their runs end. */
+    drain(timeoutMs: number): Promise<{
+        released: string[];
+        failed: Record<string, string>;
+    }>;
+    /** Takes users again: a drain that was called off. */
+    undrain(): void;
+    list(withUsage?: boolean): Promise<UserInfo[]>;
+    /** Over the soft quota: refuse new runs (checked before one starts). */
+    admit(id: string): Promise<void>;
+    /** Recovers users whose folders survived a restart, and starts the timers. */
+    start(): Promise<void>;
+    /** After the runs stopped: everyone saved once more, as far as the time allows. */
+    shutdown(timeoutMs: number): Promise<void>;
+}
+export declare const userStates: UserStates;
+/** Every user-scoped route: the caller's data loaded first, and their thread when one is named. */
+export declare function loadCaller(req: Request, _res: Response, next: NextFunction): Promise<void>;
+export declare function loadThread(req: Request, _res: Response, next: NextFunction): Promise<void>;
+/** /api/admin: only for the host's own calls (its system subject, never a user's thread token). */
+export declare function userStateRouter(): Router;
+/** Bytes under a user's folder, links not followed. */
+export declare function userUsageBytes(id: string): Promise<number>;
+export {};
+//# sourceMappingURL=userState.d.ts.map

package/dist/userState.js

@@ -0,0 +1,461 @@
+// Each user's data on a server that may be one of many: loaded before their first request, saved
+// at safe points, released when the host moves them elsewhere. The server decides *when* (it
+// alone knows when runs end and how to copy a live memory graph); a UserStateStore plugin decides
+// *where* (ZEN_STATE_STORE, e.g. ZeneraLite's S3 store). Without one, the local store keeps
+// everything on this disk, as a single server always has.
+//   UserStateStore, UserRef, UserSnapshot, USER_LAYOUT - the plugin contract
+//   StaleGenerationError - what a store throws when a newer lease owns the user
+//   localStore           - the default: nothing to load or save
+//   userStates           - the server's manager: acquire (middleware), session, changed,
+//                          release, drain, list, start (recovery, timers), shutdown
+//   userStateRouter      - /api/admin: the host's view of this server's users (system token only)
+//   userUsageBytes       - a user's local disk use, for the soft quota
+// Used by server.ts (middleware, admin routes, listen/stop), store.ts (privateStore hooks) and
+// runs.ts (quota at run start).
+import { copyMemory, isMemory } from '@zenera/neo';
+import { Router } from 'express';
+import { randomBytes } from 'node:crypto';
+import { existsSync } from 'node:fs';
+import { lstat, mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { settings } from "./config.js";
+import { HttpError } from "./http.js";
+import { caller, userFolder } from "./identity.js";
+import { ownerRuns, stopOwnerRuns } from "./runs.js";
+import { memoryDir, userDir } from "./store.js";
+/** Where things are in a user's folder, for stores that cannot import this package. */
+export const USER_LAYOUT = {
+    /** the user's memory graph; checkpoints carry a consistent copy instead (`memoryDir`) */
+    memory: 'memory',
+    /** one folder per chat thread */
+    sessions: 'sessions',
+    /** inside a session folder: its metadata, which the session list reads */
+    sessionMeta: '.data/session.json',
+    /** the server's own record of the user's lease; never a store's to save */
+    marker: '.zen-user.json',
+};
+/** Thrown by a store when a newer lease generation owns the user: this copy must go. */
+export class StaleGenerationError extends Error {
+    code = 'stale_generation';
+}
+export const localStore = {
+    name: 'local',
+    durable: false,
+    load: async () => undefined,
+    checkpoint: async () => undefined,
+};
+function store() {
+    return settings().stateStore ?? localStore;
+}
+function generationOf(req) {
+    const gen = caller(req).gen;
+    if (gen === undefined)
+        return undefined;
+    if (!Number.isSafeInteger(gen) || gen < 0) {
+        throw new HttpError(401, 'the token names no valid lease generation (gen)');
+    }
+    return gen;
+}
+function refOf(entry) {
+    return { id: entry.id, dir: userDir(entry.id), generation: entry.generation };
+}
+const markerOf = (id) => join(userDir(id), USER_LAYOUT.marker);
+async function writeMarker(entry) {
+    // Only a durable store recovers users after a restart; local users need no record, and
+    // listing sessions must not create a user's folder.
+    if (!store().durable)
+        return;
+    await mkdir(userDir(entry.id), { recursive: true });
+    await writeFile(markerOf(entry.id), `${JSON.stringify({ generation: entry.generation, store: store().name, loadedAt: new Date().toISOString(), lastCheckpointAt: entry.lastCheckpointAt }, null, 2)}\n`);
+}
+class UserStates {
+    #users = new Map();
+    #draining = false;
+    #timer;
+    #quotaTimer;
+    #usage = new Map();
+    get draining() {
+        return this.#draining;
+    }
+    get size() {
+        return this.#users.size;
+    }
+    /** The user's data, ready; their folder is theirs to use once this resolves. */
+    async acquire(id, generation) {
+        let entry = this.#users.get(id);
+        if (entry && generation !== undefined && entry.generation !== undefined) {
+            if (generation < entry.generation) {
+                throw Object.assign(new HttpError(409, 'this lease is no longer current: the user moved'), { code: 'stale_lease' });
+            }
+            if (generation > entry.generation) {
+                // Leased elsewhere and back: what is here may be older than what was saved there.
+                if (ownerRuns(id).length) {
+                    throw new HttpError(409, 'the user moved while a run is still going here');
+                }
+                await this.#forget(entry, 'a newer lease');
+                entry = undefined;
+            }
+        }
+        if (!entry) {
+            if (this.#draining)
+                throw new HttpError(503, 'this server is draining');
+            entry = this.#load(id, generation);
+        }
+        if (entry.status === 'releasing')
+            throw new HttpError(503, 'the user is being released');
+        entry.lastUsed = Date.now();
+        await entry.loaded;
+    }
+    #load(id, generation) {
+        const entry = {
+            id,
+            generation,
+            status: 'loading',
+            loaded: Promise.resolve(),
+            sessions: new Map(),
+            changes: 0,
+            saved: 0,
+            queue: Promise.resolve(),
+            lastUsed: Date.now(),
+        };
+        this.#users.set(id, entry);
+        const s = store();
+        entry.loaded = (async () => {
+            // A durable store's user is whatever it holds: never mix that with a stale local copy.
+            if (s.durable)
+                await rm(userDir(id), { recursive: true, force: true });
+            await s.load(refOf(entry));
+            entry.status = 'ready';
+            await writeMarker(entry);
+        })().catch((err) => {
+            this.#users.delete(id);
+            throw new HttpError(503, `could not load the user's data: ${messageOf(err)}`);
+        });
+        return entry;
+    }
+    /** Before a request names a thread: the store fetches it if it left it for later. */
+    async session(id, threadId) {
+        const load = store().loadSession;
+        const entry = this.#users.get(id);
+        if (!load || !entry)
+            return;
+        let pending = entry.sessions.get(threadId);
+        if (!pending) {
+            pending = load.call(store(), refOf(entry), threadId).catch((err) => {
+                entry.sessions.delete(threadId);
+                throw new HttpError(503, `could not load the chat: ${messageOf(err)}`);
+            });
+            entry.sessions.set(threadId, pending);
+        }
+        await pending;
+    }
+    /** Something of the user's changed (a run ended, a thread went): save it. */
+    changed(id, reason = 'changed') {
+        this.#usage.delete(id);
+        const entry = this.#users.get(id);
+        if (!entry)
+            return;
+        entry.changes++;
+        void this.checkpoint(id, reason).catch(() => undefined);
+    }
+    /** Saves the user now, after any save already queued; one at a time per user. */
+    checkpoint(id, reason) {
+        const entry = this.#users.get(id);
+        if (!entry || !store().durable)
+            return Promise.resolve();
+        const run = entry.queue.then(() => this.#save(entry, reason));
+        entry.queue = run.catch(() => undefined);
+        return run;
+    }
+    async #save(entry, reason) {
+        if (this.#users.get(entry.id) !== entry)
+            return;
+        const upTo = entry.changes;
+        const tmp = join(settings().root, '.checkpoints', `${entry.id}-${randomBytes(4).toString('hex')}`);
+        try {
+            const memory = memoryDir(entry.id);
+            let snapshotDir;
+            if (isMemory(memory)) {
+                await mkdir(join(settings().root, '.checkpoints'), { recursive: true });
+                await copyMemory(memory, tmp);
+                snapshotDir = tmp;
+            }
+            await store().checkpoint(refOf(entry), reason, {
+                memoryDir: snapshotDir,
+                activeThreads: ownerRuns(entry.id).map((r) => r.threadId),
+            });
+            entry.saved = Math.max(entry.saved, upTo);
+            entry.lastCheckpointAt = new Date().toISOString();
+            entry.lastError = undefined;
+            await writeMarker(entry);
+        }
+        catch (err) {
+            if (err instanceof StaleGenerationError ||
+                err?.code === 'stale_generation') {
+                console.warn(`user ${entry.id}: ${messageOf(err)}; dropping this copy`);
+                await this.#forget(entry, 'a newer lease', true);
+                return;
+            }
+            entry.lastError = messageOf(err);
+            console.error(`user ${entry.id}: checkpoint (${reason}) failed: ${entry.lastError}`);
+            throw err;
+        }
+        finally {
+            await rm(tmp, { recursive: true, force: true });
+        }
+    }
+    /** Drops the user here: their runs stopped, their local copy gone when the store keeps them. */
+    async #forget(entry, why, stopRuns = false) {
+        this.#users.delete(entry.id);
+        if (stopRuns)
+            await stopOwnerRuns(entry.id, `the run was stopped: ${why} owns this user`);
+        if (store().durable)
+            await rm(userDir(entry.id), { recursive: true, force: true });
+        this.#usage.delete(entry.id);
+    }
+    /** Saves the user for the last time and lets them go; refused while their runs go on. */
+    async release(id) {
+        const entry = this.#users.get(id);
+        if (!entry)
+            return 'absent';
+        await entry.loaded.catch(() => undefined);
+        if (!this.#users.has(id))
+            return 'absent';
+        if (ownerRuns(id).length) {
+            throw Object.assign(new HttpError(409, 'the user has runs going'), { code: 'busy' });
+        }
+        entry.status = 'releasing';
+        try {
+            await this.checkpoint(id, 'release');
+            await store().release?.(refOf(entry));
+        }
+        catch (err) {
+            entry.status = 'ready';
+            throw new HttpError(503, `could not save the user: ${messageOf(err)}`);
+        }
+        if (this.#users.get(id) === entry)
+            await this.#forget(entry, 'release');
+        return 'released';
+    }
+    /** No new users; each one here released once their runs end. */
+    async drain(timeoutMs) {
+        this.#draining = true;
+        const released = [];
+        const failed = {};
+        const deadline = Date.now() + timeoutMs;
+        await Promise.all([...this.#users.keys()].map(async (id) => {
+            const runs = ownerRuns(id).map((r) => r.done);
+            const left = Math.max(0, deadline - Date.now());
+            await Promise.race([
+                Promise.all(runs),
+                new Promise((resolve) => setTimeout(resolve, left).unref()),
+            ]);
+            try {
+                if ((await this.release(id)) === 'released')
+                    released.push(id);
+            }
+            catch (err) {
+                failed[id] = messageOf(err);
+            }
+        }));
+        return { released, failed };
+    }
+    /** Takes users again: a drain that was called off. */
+    undrain() {
+        this.#draining = false;
+    }
+    async list(withUsage = false) {
+        return Promise.all([...this.#users.values()].map(async (e) => ({
+            id: e.id,
+            generation: e.generation,
+            status: e.status,
+            activeRuns: ownerRuns(e.id).length,
+            unsaved: store().durable && e.changes > e.saved,
+            lastUsed: new Date(e.lastUsed).toISOString(),
+            ...(e.lastCheckpointAt ? { lastCheckpointAt: e.lastCheckpointAt } : {}),
+            ...(e.lastError ? { lastError: e.lastError } : {}),
+            ...(withUsage ? { usageBytes: await userUsageBytes(e.id) } : {}),
+        })));
+    }
+    /** Over the soft quota: refuse new runs (checked before one starts). */
+    async admit(id) {
+        const quota = settings().userDiskQuotaBytes;
+        if (!quota)
+            return;
+        const used = await this.#cachedUsage(id);
+        if (used >= quota) {
+            throw new HttpError(507, `your storage is full: ${mb(used)} of ${mb(quota)} MB; delete chats or files to continue`);
+        }
+    }
+    async #cachedUsage(id, maxAgeMs = 15_000) {
+        const hit = this.#usage.get(id);
+        if (hit && Date.now() - hit.at < maxAgeMs)
+            return hit.bytes;
+        const bytes = await userUsageBytes(id);
+        this.#usage.set(id, { bytes, at: Date.now() });
+        return bytes;
+    }
+    /** Recovers users whose folders survived a restart, and starts the timers. */
+    async start() {
+        this.#draining = false;
+        const s = store();
+        const recovered = [];
+        const users = join(settings().root, 'users');
+        if (s.durable && existsSync(users)) {
+            for (const id of await readdir(users)) {
+                let marker;
+                try {
+                    marker = JSON.parse(await readFile(markerOf(id), 'utf8'));
+                }
+                catch {
+                    continue;
+                }
+                const entry = {
+                    id,
+                    generation: marker.generation,
+                    status: 'ready',
+                    loaded: Promise.resolve(),
+                    sessions: new Map(),
+                    // Whatever happened before the restart may not be saved yet.
+                    changes: 1,
+                    saved: 0,
+                    queue: Promise.resolve(),
+                    lastUsed: Date.now(),
+                };
+                this.#users.set(id, entry);
+                recovered.push(entry);
+            }
+        }
+        if (recovered.length) {
+            // A store that cannot recover still lets the server start; its saves will say why.
+            await s.recover?.(recovered.map(refOf)).catch((err) => {
+                console.error(`recovering ${recovered.length} users: ${messageOf(err)}`);
+            });
+            for (const entry of recovered)
+                void this.checkpoint(entry.id, 'recover').catch(() => undefined);
+        }
+        const every = settings().checkpointSeconds * 1000;
+        if (s.durable && every > 0) {
+            this.#timer = setInterval(() => this.#periodic(), every);
+            this.#timer.unref();
+        }
+        if (settings().userDiskQuotaBytes) {
+            this.#quotaTimer = setInterval(() => void this.#enforceQuota(), 30_000);
+            this.#quotaTimer.unref();
+        }
+    }
+    // Users with runs going (their files keep changing) or a save that failed are saved again.
+    #periodic() {
+        for (const entry of this.#users.values()) {
+            if (entry.status !== 'ready')
+                continue;
+            if (ownerRuns(entry.id).length)
+                entry.changes++;
+            if (entry.changes > entry.saved) {
+                void this.checkpoint(entry.id, 'periodic').catch(() => undefined);
+            }
+        }
+    }
+    // A run that pushes its user far over the quota is stopped, before the pod's disk fills.
+    async #enforceQuota() {
+        const quota = settings().userDiskQuotaBytes;
+        for (const id of this.#users.keys()) {
+            if (!ownerRuns(id).length)
+                continue;
+            const used = await this.#cachedUsage(id, 0).catch(() => 0);
+            if (used > quota) {
+                await stopOwnerRuns(id, `the run was stopped: your storage is over its limit (${mb(used)} of ${mb(quota)} MB)`);
+            }
+        }
+    }
+    /** After the runs stopped: everyone saved once more, as far as the time allows. */
+    async shutdown(timeoutMs) {
+        clearInterval(this.#timer);
+        clearInterval(this.#quotaTimer);
+        const all = [...this.#users.keys()].map((id) => {
+            this.changed(id, 'shutdown');
+            return this.#users.get(id)?.queue ?? Promise.resolve();
+        });
+        await Promise.race([
+            Promise.all(all),
+            new Promise((resolve) => setTimeout(resolve, timeoutMs).unref()),
+        ]);
+        await store().close?.();
+        this.#users.clear();
+        this.#usage.clear();
+    }
+}
+export const userStates = new UserStates();
+/** Every user-scoped route: the caller's data loaded first, and their thread when one is named. */
+export async function loadCaller(req, _res, next) {
+    const id = userFolder(caller(req).sub);
+    await userStates.acquire(id, generationOf(req));
+    next();
+}
+export async function loadThread(req, _res, next) {
+    const tid = String(req.params.tid ?? '');
+    if (tid)
+        await userStates.session(userFolder(caller(req).sub), tid);
+    next();
+}
+/** /api/admin: only for the host's own calls (its system subject, never a user's thread token). */
+export function userStateRouter() {
+    const router = Router();
+    router.use((req, _res, next) => {
+        const claims = caller(req);
+        if (claims.sub !== settings().systemSubject || claims.thr !== undefined) {
+            throw new HttpError(403, 'for the host only');
+        }
+        next();
+    });
+    router.get('/users', async (req, res) => {
+        res.setHeader('Cache-Control', 'no-store');
+        res.json({
+            store: store().name,
+            durable: store().durable,
+            draining: userStates.draining,
+            users: await userStates.list(req.query.usage === '1'),
+        });
+    });
+    router.post('/users/:sub/release', async (req, res) => {
+        const result = await userStates.release(userFolder(String(req.params.sub)));
+        res.json({ result });
+    });
+    router.post('/drain', async (req, res) => {
+        const seconds = Number(req.query.timeout ?? 600);
+        const timeout = Number.isFinite(seconds) && seconds >= 0 ? seconds * 1000 : 600_000;
+        res.json(await userStates.drain(timeout));
+    });
+    router.post('/undrain', (_req, res) => {
+        userStates.undrain();
+        res.json({ draining: false });
+    });
+    return router;
+}
+/** Bytes under a user's folder, links not followed. */
+export async function userUsageBytes(id) {
+    let total = 0;
+    const walk = async (dir) => {
+        let entries;
+        try {
+            entries = await readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            const path = join(dir, entry.name);
+            if (entry.isDirectory())
+                await walk(path);
+            else if (entry.isFile())
+                total += (await lstat(path).catch(() => ({ size: 0 }))).size;
+        }
+    };
+    await walk(userDir(id));
+    return total;
+}
+const mb = (bytes) => Math.round(bytes / 2 ** 20);
+function messageOf(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+//# sourceMappingURL=userState.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@zenera/server",
-    "version": "1.2.1",
+    "version": "1.2.2",
     "description": "One user's chats with a project over HTTP: the server in the agent image, commands isolated with bubblewrap. Every user its host vouches for with a signed JWT, each in a folder of their own.",
     "keywords": [
         "agents",
@@ -45,9 +45,9 @@
     },
     "dependencies": {
         "@ag-ui/core": "^1.0.3",
-        "@zenera/engine": "^1.2.1",
-        "@zenera/neo": "^1.2.1",
-        "@zenera/sandbox": "^1.2.1",
+        "@zenera/engine": "^1.2.2",
+        "@zenera/neo": "^1.2.2",
+        "@zenera/sandbox": "^1.2.2",
         "express": "^5.3.0",
         "jose": "^6.2.13"
     },