@zenera/server 1.2.0 → 1.2.2

package/dist/userState.js

@@ -0,0 +1,461 @@
+// Each user's data on a server that may be one of many: loaded before their first request, saved
+// at safe points, released when the host moves them elsewhere. The server decides *when* (it
+// alone knows when runs end and how to copy a live memory graph); a UserStateStore plugin decides
+// *where* (ZEN_STATE_STORE, e.g. ZeneraLite's S3 store). Without one, the local store keeps
+// everything on this disk, as a single server always has.
+//   UserStateStore, UserRef, UserSnapshot, USER_LAYOUT - the plugin contract
+//   StaleGenerationError - what a store throws when a newer lease owns the user
+//   localStore           - the default: nothing to load or save
+//   userStates           - the server's manager: acquire (middleware), session, changed,
+//                          release, drain, list, start (recovery, timers), shutdown
+//   userStateRouter      - /api/admin: the host's view of this server's users (system token only)
+//   userUsageBytes       - a user's local disk use, for the soft quota
+// Used by server.ts (middleware, admin routes, listen/stop), store.ts (privateStore hooks) and
+// runs.ts (quota at run start).
+import { copyMemory, isMemory } from '@zenera/neo';
+import { Router } from 'express';
+import { randomBytes } from 'node:crypto';
+import { existsSync } from 'node:fs';
+import { lstat, mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { settings } from "./config.js";
+import { HttpError } from "./http.js";
+import { caller, userFolder } from "./identity.js";
+import { ownerRuns, stopOwnerRuns } from "./runs.js";
+import { memoryDir, userDir } from "./store.js";
+/** Where things are in a user's folder, for stores that cannot import this package. */
+export const USER_LAYOUT = {
+    /** the user's memory graph; checkpoints carry a consistent copy instead (`memoryDir`) */
+    memory: 'memory',
+    /** one folder per chat thread */
+    sessions: 'sessions',
+    /** inside a session folder: its metadata, which the session list reads */
+    sessionMeta: '.data/session.json',
+    /** the server's own record of the user's lease; never a store's to save */
+    marker: '.zen-user.json',
+};
+/** Thrown by a store when a newer lease generation owns the user: this copy must go. */
+export class StaleGenerationError extends Error {
+    code = 'stale_generation';
+}
+export const localStore = {
+    name: 'local',
+    durable: false,
+    load: async () => undefined,
+    checkpoint: async () => undefined,
+};
+function store() {
+    return settings().stateStore ?? localStore;
+}
+function generationOf(req) {
+    const gen = caller(req).gen;
+    if (gen === undefined)
+        return undefined;
+    if (!Number.isSafeInteger(gen) || gen < 0) {
+        throw new HttpError(401, 'the token names no valid lease generation (gen)');
+    }
+    return gen;
+}
+function refOf(entry) {
+    return { id: entry.id, dir: userDir(entry.id), generation: entry.generation };
+}
+const markerOf = (id) => join(userDir(id), USER_LAYOUT.marker);
+async function writeMarker(entry) {
+    // Only a durable store recovers users after a restart; local users need no record, and
+    // listing sessions must not create a user's folder.
+    if (!store().durable)
+        return;
+    await mkdir(userDir(entry.id), { recursive: true });
+    await writeFile(markerOf(entry.id), `${JSON.stringify({ generation: entry.generation, store: store().name, loadedAt: new Date().toISOString(), lastCheckpointAt: entry.lastCheckpointAt }, null, 2)}\n`);
+}
+class UserStates {
+    #users = new Map();
+    #draining = false;
+    #timer;
+    #quotaTimer;
+    #usage = new Map();
+    get draining() {
+        return this.#draining;
+    }
+    get size() {
+        return this.#users.size;
+    }
+    /** The user's data, ready; their folder is theirs to use once this resolves. */
+    async acquire(id, generation) {
+        let entry = this.#users.get(id);
+        if (entry && generation !== undefined && entry.generation !== undefined) {
+            if (generation < entry.generation) {
+                throw Object.assign(new HttpError(409, 'this lease is no longer current: the user moved'), { code: 'stale_lease' });
+            }
+            if (generation > entry.generation) {
+                // Leased elsewhere and back: what is here may be older than what was saved there.
+                if (ownerRuns(id).length) {
+                    throw new HttpError(409, 'the user moved while a run is still going here');
+                }
+                await this.#forget(entry, 'a newer lease');
+                entry = undefined;
+            }
+        }
+        if (!entry) {
+            if (this.#draining)
+                throw new HttpError(503, 'this server is draining');
+            entry = this.#load(id, generation);
+        }
+        if (entry.status === 'releasing')
+            throw new HttpError(503, 'the user is being released');
+        entry.lastUsed = Date.now();
+        await entry.loaded;
+    }
+    #load(id, generation) {
+        const entry = {
+            id,
+            generation,
+            status: 'loading',
+            loaded: Promise.resolve(),
+            sessions: new Map(),
+            changes: 0,
+            saved: 0,
+            queue: Promise.resolve(),
+            lastUsed: Date.now(),
+        };
+        this.#users.set(id, entry);
+        const s = store();
+        entry.loaded = (async () => {
+            // A durable store's user is whatever it holds: never mix that with a stale local copy.
+            if (s.durable)
+                await rm(userDir(id), { recursive: true, force: true });
+            await s.load(refOf(entry));
+            entry.status = 'ready';
+            await writeMarker(entry);
+        })().catch((err) => {
+            this.#users.delete(id);
+            throw new HttpError(503, `could not load the user's data: ${messageOf(err)}`);
+        });
+        return entry;
+    }
+    /** Before a request names a thread: the store fetches it if it left it for later. */
+    async session(id, threadId) {
+        const load = store().loadSession;
+        const entry = this.#users.get(id);
+        if (!load || !entry)
+            return;
+        let pending = entry.sessions.get(threadId);
+        if (!pending) {
+            pending = load.call(store(), refOf(entry), threadId).catch((err) => {
+                entry.sessions.delete(threadId);
+                throw new HttpError(503, `could not load the chat: ${messageOf(err)}`);
+            });
+            entry.sessions.set(threadId, pending);
+        }
+        await pending;
+    }
+    /** Something of the user's changed (a run ended, a thread went): save it. */
+    changed(id, reason = 'changed') {
+        this.#usage.delete(id);
+        const entry = this.#users.get(id);
+        if (!entry)
+            return;
+        entry.changes++;
+        void this.checkpoint(id, reason).catch(() => undefined);
+    }
+    /** Saves the user now, after any save already queued; one at a time per user. */
+    checkpoint(id, reason) {
+        const entry = this.#users.get(id);
+        if (!entry || !store().durable)
+            return Promise.resolve();
+        const run = entry.queue.then(() => this.#save(entry, reason));
+        entry.queue = run.catch(() => undefined);
+        return run;
+    }
+    async #save(entry, reason) {
+        if (this.#users.get(entry.id) !== entry)
+            return;
+        const upTo = entry.changes;
+        const tmp = join(settings().root, '.checkpoints', `${entry.id}-${randomBytes(4).toString('hex')}`);
+        try {
+            const memory = memoryDir(entry.id);
+            let snapshotDir;
+            if (isMemory(memory)) {
+                await mkdir(join(settings().root, '.checkpoints'), { recursive: true });
+                await copyMemory(memory, tmp);
+                snapshotDir = tmp;
+            }
+            await store().checkpoint(refOf(entry), reason, {
+                memoryDir: snapshotDir,
+                activeThreads: ownerRuns(entry.id).map((r) => r.threadId),
+            });
+            entry.saved = Math.max(entry.saved, upTo);
+            entry.lastCheckpointAt = new Date().toISOString();
+            entry.lastError = undefined;
+            await writeMarker(entry);
+        }
+        catch (err) {
+            if (err instanceof StaleGenerationError ||
+                err?.code === 'stale_generation') {
+                console.warn(`user ${entry.id}: ${messageOf(err)}; dropping this copy`);
+                await this.#forget(entry, 'a newer lease', true);
+                return;
+            }
+            entry.lastError = messageOf(err);
+            console.error(`user ${entry.id}: checkpoint (${reason}) failed: ${entry.lastError}`);
+            throw err;
+        }
+        finally {
+            await rm(tmp, { recursive: true, force: true });
+        }
+    }
+    /** Drops the user here: their runs stopped, their local copy gone when the store keeps them. */
+    async #forget(entry, why, stopRuns = false) {
+        this.#users.delete(entry.id);
+        if (stopRuns)
+            await stopOwnerRuns(entry.id, `the run was stopped: ${why} owns this user`);
+        if (store().durable)
+            await rm(userDir(entry.id), { recursive: true, force: true });
+        this.#usage.delete(entry.id);
+    }
+    /** Saves the user for the last time and lets them go; refused while their runs go on. */
+    async release(id) {
+        const entry = this.#users.get(id);
+        if (!entry)
+            return 'absent';
+        await entry.loaded.catch(() => undefined);
+        if (!this.#users.has(id))
+            return 'absent';
+        if (ownerRuns(id).length) {
+            throw Object.assign(new HttpError(409, 'the user has runs going'), { code: 'busy' });
+        }
+        entry.status = 'releasing';
+        try {
+            await this.checkpoint(id, 'release');
+            await store().release?.(refOf(entry));
+        }
+        catch (err) {
+            entry.status = 'ready';
+            throw new HttpError(503, `could not save the user: ${messageOf(err)}`);
+        }
+        if (this.#users.get(id) === entry)
+            await this.#forget(entry, 'release');
+        return 'released';
+    }
+    /** No new users; each one here released once their runs end. */
+    async drain(timeoutMs) {
+        this.#draining = true;
+        const released = [];
+        const failed = {};
+        const deadline = Date.now() + timeoutMs;
+        await Promise.all([...this.#users.keys()].map(async (id) => {
+            const runs = ownerRuns(id).map((r) => r.done);
+            const left = Math.max(0, deadline - Date.now());
+            await Promise.race([
+                Promise.all(runs),
+                new Promise((resolve) => setTimeout(resolve, left).unref()),
+            ]);
+            try {
+                if ((await this.release(id)) === 'released')
+                    released.push(id);
+            }
+            catch (err) {
+                failed[id] = messageOf(err);
+            }
+        }));
+        return { released, failed };
+    }
+    /** Takes users again: a drain that was called off. */
+    undrain() {
+        this.#draining = false;
+    }
+    async list(withUsage = false) {
+        return Promise.all([...this.#users.values()].map(async (e) => ({
+            id: e.id,
+            generation: e.generation,
+            status: e.status,
+            activeRuns: ownerRuns(e.id).length,
+            unsaved: store().durable && e.changes > e.saved,
+            lastUsed: new Date(e.lastUsed).toISOString(),
+            ...(e.lastCheckpointAt ? { lastCheckpointAt: e.lastCheckpointAt } : {}),
+            ...(e.lastError ? { lastError: e.lastError } : {}),
+            ...(withUsage ? { usageBytes: await userUsageBytes(e.id) } : {}),
+        })));
+    }
+    /** Over the soft quota: refuse new runs (checked before one starts). */
+    async admit(id) {
+        const quota = settings().userDiskQuotaBytes;
+        if (!quota)
+            return;
+        const used = await this.#cachedUsage(id);
+        if (used >= quota) {
+            throw new HttpError(507, `your storage is full: ${mb(used)} of ${mb(quota)} MB; delete chats or files to continue`);
+        }
+    }
+    async #cachedUsage(id, maxAgeMs = 15_000) {
+        const hit = this.#usage.get(id);
+        if (hit && Date.now() - hit.at < maxAgeMs)
+            return hit.bytes;
+        const bytes = await userUsageBytes(id);
+        this.#usage.set(id, { bytes, at: Date.now() });
+        return bytes;
+    }
+    /** Recovers users whose folders survived a restart, and starts the timers. */
+    async start() {
+        this.#draining = false;
+        const s = store();
+        const recovered = [];
+        const users = join(settings().root, 'users');
+        if (s.durable && existsSync(users)) {
+            for (const id of await readdir(users)) {
+                let marker;
+                try {
+                    marker = JSON.parse(await readFile(markerOf(id), 'utf8'));
+                }
+                catch {
+                    continue;
+                }
+                const entry = {
+                    id,
+                    generation: marker.generation,
+                    status: 'ready',
+                    loaded: Promise.resolve(),
+                    sessions: new Map(),
+                    // Whatever happened before the restart may not be saved yet.
+                    changes: 1,
+                    saved: 0,
+                    queue: Promise.resolve(),
+                    lastUsed: Date.now(),
+                };
+                this.#users.set(id, entry);
+                recovered.push(entry);
+            }
+        }
+        if (recovered.length) {
+            // A store that cannot recover still lets the server start; its saves will say why.
+            await s.recover?.(recovered.map(refOf)).catch((err) => {
+                console.error(`recovering ${recovered.length} users: ${messageOf(err)}`);
+            });
+            for (const entry of recovered)
+                void this.checkpoint(entry.id, 'recover').catch(() => undefined);
+        }
+        const every = settings().checkpointSeconds * 1000;
+        if (s.durable && every > 0) {
+            this.#timer = setInterval(() => this.#periodic(), every);
+            this.#timer.unref();
+        }
+        if (settings().userDiskQuotaBytes) {
+            this.#quotaTimer = setInterval(() => void this.#enforceQuota(), 30_000);
+            this.#quotaTimer.unref();
+        }
+    }
+    // Users with runs going (their files keep changing) or a save that failed are saved again.
+    #periodic() {
+        for (const entry of this.#users.values()) {
+            if (entry.status !== 'ready')
+                continue;
+            if (ownerRuns(entry.id).length)
+                entry.changes++;
+            if (entry.changes > entry.saved) {
+                void this.checkpoint(entry.id, 'periodic').catch(() => undefined);
+            }
+        }
+    }
+    // A run that pushes its user far over the quota is stopped, before the pod's disk fills.
+    async #enforceQuota() {
+        const quota = settings().userDiskQuotaBytes;
+        for (const id of this.#users.keys()) {
+            if (!ownerRuns(id).length)
+                continue;
+            const used = await this.#cachedUsage(id, 0).catch(() => 0);
+            if (used > quota) {
+                await stopOwnerRuns(id, `the run was stopped: your storage is over its limit (${mb(used)} of ${mb(quota)} MB)`);
+            }
+        }
+    }
+    /** After the runs stopped: everyone saved once more, as far as the time allows. */
+    async shutdown(timeoutMs) {
+        clearInterval(this.#timer);
+        clearInterval(this.#quotaTimer);
+        const all = [...this.#users.keys()].map((id) => {
+            this.changed(id, 'shutdown');
+            return this.#users.get(id)?.queue ?? Promise.resolve();
+        });
+        await Promise.race([
+            Promise.all(all),
+            new Promise((resolve) => setTimeout(resolve, timeoutMs).unref()),
+        ]);
+        await store().close?.();
+        this.#users.clear();
+        this.#usage.clear();
+    }
+}
+export const userStates = new UserStates();
+/** Every user-scoped route: the caller's data loaded first, and their thread when one is named. */
+export async function loadCaller(req, _res, next) {
+    const id = userFolder(caller(req).sub);
+    await userStates.acquire(id, generationOf(req));
+    next();
+}
+export async function loadThread(req, _res, next) {
+    const tid = String(req.params.tid ?? '');
+    if (tid)
+        await userStates.session(userFolder(caller(req).sub), tid);
+    next();
+}
+/** /api/admin: only for the host's own calls (its system subject, never a user's thread token). */
+export function userStateRouter() {
+    const router = Router();
+    router.use((req, _res, next) => {
+        const claims = caller(req);
+        if (claims.sub !== settings().systemSubject || claims.thr !== undefined) {
+            throw new HttpError(403, 'for the host only');
+        }
+        next();
+    });
+    router.get('/users', async (req, res) => {
+        res.setHeader('Cache-Control', 'no-store');
+        res.json({
+            store: store().name,
+            durable: store().durable,
+            draining: userStates.draining,
+            users: await userStates.list(req.query.usage === '1'),
+        });
+    });
+    router.post('/users/:sub/release', async (req, res) => {
+        const result = await userStates.release(userFolder(String(req.params.sub)));
+        res.json({ result });
+    });
+    router.post('/drain', async (req, res) => {
+        const seconds = Number(req.query.timeout ?? 600);
+        const timeout = Number.isFinite(seconds) && seconds >= 0 ? seconds * 1000 : 600_000;
+        res.json(await userStates.drain(timeout));
+    });
+    router.post('/undrain', (_req, res) => {
+        userStates.undrain();
+        res.json({ draining: false });
+    });
+    return router;
+}
+/** Bytes under a user's folder, links not followed. */
+export async function userUsageBytes(id) {
+    let total = 0;
+    const walk = async (dir) => {
+        let entries;
+        try {
+            entries = await readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            const path = join(dir, entry.name);
+            if (entry.isDirectory())
+                await walk(path);
+            else if (entry.isFile())
+                total += (await lstat(path).catch(() => ({ size: 0 }))).size;
+        }
+    };
+    await walk(userDir(id));
+    return total;
+}
+const mb = (bytes) => Math.round(bytes / 2 ** 20);
+function messageOf(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+//# sourceMappingURL=userState.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@zenera/server",
-    "version": "1.2.0",
+    "version": "1.2.2",
     "description": "One user's chats with a project over HTTP: the server in the agent image, commands isolated with bubblewrap. Every user its host vouches for with a signed JWT, each in a folder of their own.",
     "keywords": [
         "agents",
@@ -45,9 +45,9 @@
     },
     "dependencies": {
         "@ag-ui/core": "^1.0.3",
-        "@zenera/engine": "^1.2.0",
-        "@zenera/neo": "^1.2.0",
-        "@zenera/sandbox": "^1.2.0",
+        "@zenera/engine": "^1.2.2",
+        "@zenera/neo": "^1.2.2",
+        "@zenera/sandbox": "^1.2.2",
         "express": "^5.3.0",
         "jose": "^6.2.13"
     },

package/dist/state.d.ts

@@ -1,24 +0,0 @@
-/** 1: data/{sessions,runs,memory}, one user's. */
-export declare const STATE_VERSION = 1;
-export interface StateInfo {
-    version: number;
-    createdAt: string;
-    /** whose state it is */
-    userId: string;
-    projectId: string;
-}
-export declare class StateError extends Error {
-    name: string;
-}
-/**
- * The state directory's description, checked against who opens it: the same project and the
- * same user. `create: false` only reads (a check must not claim a directory): a missing file
- * is then `undefined`.
- */
-export declare function openState(dir: string, owner: {
-    userId: string;
-    projectId: string;
-}, opts?: {
-    create?: boolean;
-}): StateInfo | undefined;
-//# sourceMappingURL=state.d.ts.map

package/dist/state.js

@@ -1,55 +0,0 @@
-// The state directory's own description: which layout it is in, so a server never reads (or
-// writes over) a layout it does not know. Written on first start; a newer one is refused.
-//   STATE_VERSION   - the layout this server writes
-//   openState       - create or check <state>/state.json, answering what it says
-import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
-import { join } from 'node:path';
-/** 1: data/{sessions,runs,memory}, one user's. */
-export const STATE_VERSION = 1;
-export class StateError extends Error {
-    name = 'StateError';
-}
-/**
- * The state directory's description, checked against who opens it: the same project and the
- * same user. `create: false` only reads (a check must not claim a directory): a missing file
- * is then `undefined`.
- */
-export function openState(dir, owner, opts = {}) {
-    const file = join(dir, 'state.json');
-    if (!existsSync(file)) {
-        if (opts.create === false)
-            return undefined;
-        // The user's chats: nobody else on the machine reads them.
-        mkdirSync(dir, { recursive: true, mode: 0o700 });
-        const info = {
-            version: STATE_VERSION,
-            createdAt: new Date().toISOString(),
-            userId: owner.userId,
-            projectId: owner.projectId,
-        };
-        writeFileSync(`${file}.tmp`, `${JSON.stringify(info, null, 2)}\n`);
-        renameSync(`${file}.tmp`, file);
-        return info;
-    }
-    let info;
-    try {
-        info = JSON.parse(readFileSync(file, 'utf8'));
-    }
-    catch (err) {
-        throw new StateError(`${file} is not readable: ${err instanceof Error ? err.message : err}`);
-    }
-    if (!Number.isInteger(info.version) || info.version < 1) {
-        throw new StateError(`${file} names no layout version`);
-    }
-    if (info.version > STATE_VERSION) {
-        throw new StateError(`${dir} is in layout ${info.version}, written by a newer server; this one knows ${STATE_VERSION}`);
-    }
-    if (info.projectId !== owner.projectId) {
-        throw new StateError(`${dir} belongs to project ${info.projectId}, not ${owner.projectId}`);
-    }
-    if (info.userId !== owner.userId) {
-        throw new StateError(`${dir} belongs to user ${info.userId}, not ${owner.userId}`);
-    }
-    return info;
-}
-//# sourceMappingURL=state.js.map

package/dist/token.d.ts

@@ -1,14 +0,0 @@
-export declare const TOKEN_ENV = "ZEN_SERVER_TOKEN";
-export declare const MIN_TOKEN_LENGTH = 32;
-export interface ServerToken {
-    token: string;
-    /** where it came from: the environment, or the file */
-    from: 'env' | string;
-}
-/** Where a project's server for a user keeps its token. Both are single path segments here. */
-export declare function tokenFile(project: string, user: string): string;
-export declare function serverToken(file: string, opts?: {
-    create?: boolean;
-    env?: NodeJS.ProcessEnv;
-}): ServerToken | undefined;
-//# sourceMappingURL=token.d.ts.map

package/dist/token.js

@@ -1,30 +0,0 @@
-// The server's bearer token, when its host gave none: one per project and user, in the zen home
-// (which no sandboxed command can see), readable by its owner only.
-//   tokenFile    - <zen home>/servers/<project>/<user>.token
-//   serverToken  - $ZEN_SERVER_TOKEN, else the file, else a new one written there (`create`)
-import { randomBytes } from 'node:crypto';
-import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
-import { home } from '@zenera/engine';
-import { dirname, join } from 'node:path';
-export const TOKEN_ENV = 'ZEN_SERVER_TOKEN';
-export const MIN_TOKEN_LENGTH = 32;
-/** Where a project's server for a user keeps its token. Both are single path segments here. */
-export function tokenFile(project, user) {
-    const segment = (s) => s.replace(/[^\w.@-]/g, '_');
-    return join(home(), 'servers', segment(project), `${segment(user)}.token`);
-}
-export function serverToken(file, opts = {}) {
-    const given = (opts.env ?? process.env)[TOKEN_ENV]?.trim();
-    if (given)
-        return { token: given, from: 'env' };
-    if (existsSync(file))
-        return { token: readFileSync(file, 'utf8').trim(), from: file };
-    if (!opts.create)
-        return undefined;
-    mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
-    const token = randomBytes(32).toString('base64url');
-    writeFileSync(file, `${token}\n`, { mode: 0o600, flag: 'wx' });
-    chmodSync(file, 0o600);
-    return { token, from: file };
-}
-//# sourceMappingURL=token.js.map