@zenera/gym 0.0.0-stage → 1.1.39

package/dist/server.js

@@ -0,0 +1,626 @@
+import { createHash, randomInt, randomUUID } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import { createServer } from 'node:http';
+import { join } from 'node:path';
+import { SandboxUnavailable } from "./box.js";
+import { BuildFailed } from "./cache.js";
+import { reason } from "./generate.js";
+import { formatLogLine, formatRegenerated, formatRegenerating, formatRegenLimitHit, RegenerationLimiter, RequestHistory, saveRequestDump, } from "./logger.js";
+import { indexPage } from "./page.js";
+import { cutLoop, getSentToken, PaginationLimiter, pagingSeen, } from "./paging.js";
+import { describeIssues, issues } from "./validate.js";
+// ---------------------------------------------------------------------------
+// The server
+//
+// `node:http` and a switch. A framework would earn its place if there were
+// middleware to compose, and there is not: one route table, one validation
+// step, one generator.
+//
+// The rules that are not obvious from the code are the ones about what does not
+// travel. Request headers are filtered before the envelope is written, because
+// that envelope becomes a file inside a container. Request bodies never reach a
+// prompt at all — the build loop uses probes it made up. And the listener binds
+// to loopback unless somebody says otherwise in as many words.
+// ---------------------------------------------------------------------------
+/** Headers that are none of a generator's business. */
+const REDACTED = /^(authorization|cookie|set-cookie|proxy-authorization)$/i;
+const SECRETISH = /key|token|secret|password|credential/i;
+const DEFAULT_MAX_BODY = 1024 * 1024;
+export function build(opts) {
+    const history = opts.history ?? new RequestHistory();
+    const limiter = opts.limiter ?? new RegenerationLimiter(opts.regenLimit ?? 3);
+    const paginationLimiter = opts.paginationLimiter ?? new PaginationLimiter(opts.maxPages);
+    const requestsDir = opts.requestsDir ?? join(opts.box.root, 'requests');
+    const enriched = {
+        ...opts,
+        history,
+        limiter,
+        paginationLimiter,
+        requestsDir,
+    };
+    return createServer((req, res) => {
+        handle(req, res, enriched).catch((err) => {
+            const message = reason(err);
+            send(res, 500, { error: message });
+            opts.onRequest?.(`${req.method ?? 'GET'} ${req.url ?? '/'} 500 [SERVER_INTERNAL_ERROR] ${message}`);
+        });
+    });
+}
+export async function listen(opts, host, port) {
+    const server = build(opts);
+    await new Promise((settle, fail) => {
+        server.once('error', fail);
+        server.listen(port, host, () => {
+            server.off('error', fail);
+            settle();
+        });
+    });
+    return {
+        server,
+        port: server.address().port,
+        close: () => new Promise((settle) => {
+            server.close(() => settle());
+            server.closeIdleConnections();
+        }),
+    };
+}
+// ---------------------------------------------------------------------------
+// One request
+// ---------------------------------------------------------------------------
+async function handle(req, res, opts) {
+    const started = Date.now();
+    const timestamp = new Date(started);
+    const id = randomUUID().slice(0, 8);
+    const url = new URL(req.url ?? '/', 'http://localhost');
+    const method = (req.method ?? 'GET').toLowerCase();
+    const finish = async (status, meta) => {
+        const responseHeaders = Object.fromEntries(Object.entries(res.getHeaders()).map(([k, v]) => [k, String(v)]));
+        const dumpData = {
+            id,
+            timestamp,
+            durationMs: Date.now() - started,
+            method,
+            url: req.url ?? '/',
+            pathname: url.pathname,
+            search: url.search,
+            operationId: meta.operationId,
+            routePath: meta.routePath,
+            pathParams: meta.pathParams,
+            query: meta.query ?? Object.fromEntries(url.searchParams),
+            requestHeaders: safeHeaders(req),
+            requestBody: meta.requestBody,
+            status,
+            responseHeaders,
+            responseBody: meta.responseBody,
+            errorKind: meta.errorKind,
+            errorMessage: meta.errorMessage,
+            hint: meta.hint,
+            stderr: meta.stderr,
+            buildDump: meta.buildDump,
+            cacheStatus: meta.cacheStatus,
+            regenerated: meta.regenerated,
+            regenAttempts: meta.regenAttempts,
+            regenLimitHit: meta.regenLimitHit,
+            regenLimit: opts.limiter?.limit,
+            issues: meta.issues,
+            note: meta.note,
+        };
+        let dumpPath = '';
+        const shouldDump = Boolean(opts.requestsDir &&
+            url.pathname !== '/' &&
+            !url.pathname.startsWith('/__gym/') &&
+            meta.operationId);
+        if (shouldDump) {
+            try {
+                dumpPath = await saveRequestDump(opts.requestsDir, dumpData);
+            }
+            catch {
+                // Ignore filesystem dump write failures so server remains operational
+            }
+        }
+        if (opts.onRequest) {
+            opts.onRequest(formatLogLine(dumpData, dumpPath));
+        }
+    };
+    if (url.pathname.startsWith('/__gym/')) {
+        introspect(url.pathname, res, opts);
+        await finish(200, { note: 'introspection' });
+        return;
+    }
+    const match = opts.router.match(method, url.pathname, url.searchParams);
+    if (!match) {
+        // The document owns `/` only when it declares it. Otherwise the root is
+        // the contents page, which is what a browser pointed here came for.
+        if (url.pathname === '/' && (method === 'get' || method === 'head')) {
+            const end = prepareHtml(res, indexPage(opts.router.operations));
+            await finish(200, { note: 'index' });
+            end();
+            return;
+        }
+        // The path matched and the query did not, which no one guesses unaided
+        // — and it is not a 405 either, since the method is defined here.
+        const expects = opts.router.expects(method, url.pathname);
+        if (expects.length > 0) {
+            const bodyObj = {
+                error: `${req.method} ${url.pathname} is only defined with a query`,
+                expects,
+            };
+            const end = prepareSend(res, 404, bodyObj);
+            await finish(404, {
+                errorKind: 'CLIENT_ERROR: NOT_FOUND',
+                errorMessage: `expects ${expects.join(' ')}`,
+                responseBody: bodyObj,
+                note: `expects ${expects.join(' ')}`,
+            });
+            end();
+            return;
+        }
+        const allowed = opts.router.allowed(url.pathname, url.searchParams);
+        if (allowed.length > 0) {
+            res.setHeader('allow', allowed.map((m) => m.toUpperCase()).join(', '));
+            const bodyObj = { error: `${req.method} is not defined for ${url.pathname}` };
+            const end = prepareSend(res, 405, bodyObj);
+            await finish(405, {
+                errorKind: 'CLIENT_ERROR: METHOD_NOT_ALLOWED',
+                errorMessage: 'no such method',
+                responseBody: bodyObj,
+                note: 'no such method',
+            });
+            end();
+            return;
+        }
+        const bodyObj = {
+            error: `no operation matches ${req.method} ${url.pathname}`,
+        };
+        const end = prepareSend(res, 404, bodyObj);
+        await finish(404, {
+            errorKind: 'CLIENT_ERROR: NOT_FOUND',
+            errorMessage: 'no route',
+            responseBody: bodyObj,
+            note: 'no route',
+        });
+        end();
+        return;
+    }
+    const { operation, pathParams } = match;
+    res.setHeader('x-gym-operation', operation.operationId);
+    /** The container could not run anything: say so, and never blame the generator. */
+    const sandboxDown = async (err, extra = {}) => {
+        const bodyObj = {
+            error: err.message,
+            hint: err.hint,
+            container: err.container,
+            stderr: err.stderr,
+        };
+        res.setHeader('retry-after', '1');
+        const end = prepareSend(res, 503, bodyObj);
+        await finish(503, {
+            operationId: operation.operationId,
+            routePath: operation.path,
+            pathParams,
+            ...extra,
+            errorKind: 'SANDBOX_UNAVAILABLE',
+            errorMessage: err.message,
+            hint: err.hint,
+            stderr: err.stderr,
+            responseBody: bodyObj,
+            note: 'container engine failed',
+        });
+        end();
+    };
+    let body;
+    try {
+        body = await readBody(req, opts.maxBody ?? DEFAULT_MAX_BODY);
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        const status = message.includes('too large') ? 413 : 400;
+        const errorKind = status === 413 ? 'CLIENT_ERROR: PAYLOAD_TOO_LARGE' : 'CLIENT_ERROR: BAD_REQUEST';
+        const bodyObj = { error: message };
+        const end = prepareSend(res, status, bodyObj);
+        await finish(status, {
+            operationId: operation.operationId,
+            routePath: operation.path,
+            pathParams,
+            errorKind,
+            errorMessage: message,
+            responseBody: bodyObj,
+            note: status === 413 ? 'body too large' : 'bad body',
+        });
+        end();
+        return;
+    }
+    const problems = check(operation, opts.checks, pathParams, url.searchParams, body);
+    if (problems.length > 0) {
+        const desc = describeIssues(problems);
+        const bodyObj = { error: desc, issues: problems };
+        const end = prepareSend(res, 400, bodyObj);
+        await finish(400, {
+            operationId: operation.operationId,
+            routePath: operation.path,
+            pathParams,
+            requestBody: body,
+            errorKind: 'CLIENT_ERROR: VALIDATION_FAILED',
+            errorMessage: desc,
+            issues: problems,
+            responseBody: bodyObj,
+            note: desc,
+        });
+        end();
+        return;
+    }
+    if (!operation.success.schema) {
+        res.statusCode = operation.success.status;
+        await finish(operation.success.status, {
+            operationId: operation.operationId,
+            routePath: operation.path,
+            pathParams,
+            requestBody: body,
+            note: 'no body declared',
+        });
+        res.end();
+        return;
+    }
+    let generator;
+    try {
+        generator = await opts.cache.ensure(operation);
+    }
+    catch (err) {
+        if (err instanceof SandboxUnavailable) {
+            await sandboxDown(err, { requestBody: body });
+            return;
+        }
+        const status = err instanceof BuildFailed ? 501 : 500;
+        const detail = reason(err);
+        const bodyObj = {
+            error: `no generator for ${operation.operationId}`,
+            detail,
+            diagnostics: err instanceof BuildFailed ? err.diagnostics : undefined,
+            report: err instanceof BuildFailed ? err.dump : undefined,
+        };
+        const end = prepareSend(res, status, bodyObj);
+        await finish(status, {
+            operationId: operation.operationId,
+            routePath: operation.path,
+            pathParams,
+            requestBody: body,
+            errorKind: 'GENERATOR_BUILD_FAILED',
+            errorMessage: `no generator: ${detail}`,
+            buildDump: err instanceof BuildFailed ? err.dump : undefined,
+            responseBody: bodyObj,
+            note: `no generator: ${detail}`,
+        });
+        end();
+        return;
+    }
+    res.setHeader('x-gym-cache', generator.cached ? 'hit' : 'miss');
+    let cacheStatus = generator.cached ? 'hit' : 'miss';
+    const input = {
+        operationId: operation.operationId,
+        method: operation.method,
+        path: operation.path,
+        pathParams,
+        query: Object.fromEntries(url.searchParams),
+        headers: safeHeaders(req),
+        body,
+        seed: seedFor(opts.seed, operation, pathParams, url.searchParams),
+    };
+    let outcome;
+    try {
+        outcome = await opts.box.run(operation.key, input);
+    }
+    catch (err) {
+        if (err instanceof SandboxUnavailable) {
+            await sandboxDown(err, { requestBody: body, cacheStatus });
+            return;
+        }
+        throw err;
+    }
+    const responseValidator = opts.checks.for(operation).response;
+    let schemaMismatch = false;
+    if (outcome.ok && responseValidator && !responseValidator(outcome.value)) {
+        schemaMismatch = true;
+        const desc = describeIssues(issues('', responseValidator.errors));
+        outcome = {
+            ok: false,
+            fault: `output does not match response schema — ${desc}`,
+            stderr: outcome.stderr,
+            durationMs: outcome.durationMs,
+            value: outcome.value,
+        };
+    }
+    if (!outcome.ok) {
+        const urlKey = url.pathname;
+        const errorKind = outcome.fault?.includes('longer than')
+            ? 'GENERATOR_TIMEOUT'
+            : schemaMismatch
+                ? 'GENERATOR_SCHEMA_MISMATCH'
+                : 'GENERATOR_FAULT';
+        // Check regeneration limit for this URL
+        if (opts.limiter.isLimitHit(urlKey)) {
+            opts.onRequest?.(formatRegenLimitHit(urlKey, opts.limiter.limit, operation.operationId));
+            const bodyObj = {
+                error: `the generator for ${operation.operationId} ${outcome.fault}`,
+                stderr: outcome.stderr || undefined,
+                regenerationLimitHit: true,
+                regenerationLimit: opts.limiter.limit,
+            };
+            const end = prepareSend(res, 502, bodyObj);
+            await finish(502, {
+                operationId: operation.operationId,
+                routePath: operation.path,
+                pathParams,
+                requestBody: body,
+                errorKind,
+                errorMessage: `${outcome.fault} (regeneration limit reached)`,
+                stderr: outcome.stderr,
+                responseBody: bodyObj,
+                cacheStatus,
+                regenLimitHit: true,
+                note: 'generator faulted (limit hit)',
+            });
+            end();
+            return;
+        }
+        const attempt = opts.limiter.increment(urlKey);
+        opts.onRequest?.(formatRegenerating(urlKey, attempt, opts.limiter.limit, operation.operationId, outcome.fault, outcome.stderr));
+        const regenStarted = Date.now();
+        let currentSource = generator.source;
+        if (!currentSource?.trim()) {
+            currentSource = (await opts.cache.getSource?.(operation.key)) ?? '';
+        }
+        if (!currentSource?.trim()) {
+            try {
+                currentSource = await readFile(opts.box.sourceOf(operation.key), 'utf8');
+            }
+            catch {
+                currentSource = '# generator';
+            }
+        }
+        const examples = opts.history.getSuccessful(operation.key, 2);
+        try {
+            await opts.cache.regenerate(operation, {
+                currentSource,
+                failingInput: input,
+                fault: outcome.fault ?? 'generator faulted',
+                stderr: outcome.stderr,
+                examples,
+            });
+            cacheStatus = 'regenerated';
+            res.setHeader('x-gym-cache', 'regenerated');
+            opts.onRequest?.(formatRegenerated(urlKey, attempt, Date.now() - regenStarted, operation.operationId));
+            // Re-run the request with the newly regenerated generator
+            outcome = await opts.box.run(operation.key, input);
+            if (outcome.ok && responseValidator && !responseValidator(outcome.value)) {
+                outcome = {
+                    ok: false,
+                    fault: 'output does not match response schema after regeneration',
+                    stderr: outcome.stderr,
+                    durationMs: outcome.durationMs,
+                };
+            }
+        }
+        catch (regenErr) {
+            if (regenErr instanceof SandboxUnavailable) {
+                await sandboxDown(regenErr, { requestBody: body, cacheStatus });
+                return;
+            }
+            const detail = reason(regenErr);
+            const bodyObj = {
+                error: `regeneration failed for ${operation.operationId}: ${detail}`,
+                stderr: outcome.stderr || undefined,
+                regenerated: false,
+            };
+            const end = prepareSend(res, 502, bodyObj);
+            await finish(502, {
+                operationId: operation.operationId,
+                routePath: operation.path,
+                pathParams,
+                requestBody: body,
+                errorKind: 'REGENERATION_FAILED',
+                errorMessage: `regeneration failed: ${detail}`,
+                stderr: outcome.stderr,
+                responseBody: bodyObj,
+                cacheStatus,
+                regenAttempts: attempt,
+                note: 'regeneration failed',
+            });
+            end();
+            return;
+        }
+        if (!outcome.ok) {
+            const bodyObj = {
+                error: `the regenerated generator for ${operation.operationId} ${outcome.fault}`,
+                stderr: outcome.stderr || undefined,
+                regenerated: true,
+            };
+            const end = prepareSend(res, 502, bodyObj);
+            await finish(502, {
+                operationId: operation.operationId,
+                routePath: operation.path,
+                pathParams,
+                requestBody: body,
+                errorKind,
+                errorMessage: `regenerated generator ${outcome.fault}`,
+                stderr: outcome.stderr,
+                responseBody: bodyObj,
+                cacheStatus,
+                regenerated: true,
+                regenAttempts: attempt,
+                note: 'regenerated generator faulted',
+            });
+            end();
+            return;
+        }
+    }
+    const note = cacheStatus;
+    const streamKey = `${operation.key}:${req.socket.remoteAddress || 'local'}`;
+    const cutResult = cut(operation, outcome.value, url.searchParams, body, opts.paginationLimiter, streamKey);
+    opts.history.recordSuccess(operation.key, input, outcome.value);
+    const end = prepareSend(res, operation.success.status, outcome.value);
+    await finish(operation.success.status, {
+        operationId: operation.operationId,
+        routePath: operation.path,
+        pathParams,
+        requestBody: body,
+        responseBody: outcome.value,
+        cacheStatus,
+        regenerated: cacheStatus === 'regenerated',
+        note: cutResult?.cut ? `${note} · ${cutResult.reason}` : note,
+    });
+    end();
+}
+/**
+ * The generator on disk was written before the pagination rule existed, and a
+ * cache is not rebuilt just because the rule changed. A body offering back the
+ * token it was handed is therefore still possible, and it is the one bug here
+ * that costs the client rather than the mock: it hangs.
+ *
+ * In addition, pagination is tracked and capped to prevent infinite loops (random
+ * 1-10 max pages by default) and repetitive cycling tokens.
+ */
+function cut(operation, value, query, body, limiter, streamKey = 'default') {
+    const paging = operation.paging ?? pagingSeen(operation.params, operation.success.schema, query, body);
+    if (!paging) {
+        return undefined;
+    }
+    const sent = getSentToken(query, body, paging);
+    if (limiter) {
+        return limiter.process(operation.key, streamKey, sent, value, paging);
+    }
+    if (sent !== undefined && cutLoop(value, paging, sent)) {
+        return { cut: true, reason: 'cut a looping page token' };
+    }
+    return undefined;
+}
+function check(operation, checks, pathParams, query, body) {
+    const compiled = checks.for(operation);
+    const out = [];
+    // Coercion rewrites what it is given, so the copies are validated and the
+    // originals are what reach the generator.
+    if (!compiled.path({ ...pathParams })) {
+        out.push(...issues('path', compiled.path.errors));
+    }
+    if (!compiled.query(Object.fromEntries(query))) {
+        out.push(...issues('query', compiled.query.errors));
+    }
+    if (compiled.body) {
+        if (body === undefined) {
+            if (compiled.bodyRequired) {
+                out.push({ where: 'body', message: 'is required' });
+            }
+        }
+        else if (!compiled.body(body)) {
+            out.push(...issues('body', compiled.body.errors));
+        }
+    }
+    return out;
+}
+// ---------------------------------------------------------------------------
+// Wire details
+// ---------------------------------------------------------------------------
+async function readBody(req, max) {
+    const claimed = Number(req.headers['content-length'] ?? '');
+    if (Number.isFinite(claimed) && claimed > max) {
+        throw new Error(`the body is too large (${claimed} bytes, limit ${max})`);
+    }
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of req) {
+        size += chunk.length;
+        if (size > max) {
+            req.destroy();
+            throw new Error(`the body is too large (limit ${max} bytes)`);
+        }
+        chunks.push(chunk);
+    }
+    if (size === 0) {
+        return undefined;
+    }
+    const text = Buffer.concat(chunks).toString('utf8');
+    const type = String(req.headers['content-type'] ?? '')
+        .split(';')[0]
+        .trim();
+    if (type && type !== 'application/json' && !type.endsWith('+json')) {
+        return text;
+    }
+    try {
+        return JSON.parse(text);
+    }
+    catch (err) {
+        throw new Error(`the body is not JSON: ${err instanceof Error ? err.message : String(err)}`);
+    }
+}
+/**
+ * Headers reach the generator so it can vary on things like `accept-language`.
+ * Credentials do not: the envelope is written to a file inside a container, and
+ * a mock is exactly what people point a real bearer token at by accident.
+ */
+function safeHeaders(req) {
+    const out = {};
+    for (const [name, value] of Object.entries(req.headers)) {
+        if (value === undefined || REDACTED.test(name) || SECRETISH.test(name)) {
+            continue;
+        }
+        out[name] = Array.isArray(value) ? value.join(', ') : value;
+    }
+    return out;
+}
+/**
+ * Without a base seed every call is a fresh roll, which is what a demo wants.
+ * With one, the seed is a function of the request, so polling the same URL
+ * returns the same body and a test can assert on it.
+ */
+function seedFor(base, operation, pathParams, query) {
+    if (base === undefined) {
+        return randomInt(1, 2 ** 31 - 1);
+    }
+    const shape = JSON.stringify([
+        base,
+        operation.key,
+        Object.entries(pathParams).sort(),
+        [...query.entries()].sort(),
+    ]);
+    return createHash('sha256').update(shape).digest().readUInt32BE(0) % 2 ** 31;
+}
+function introspect(pathname, res, opts) {
+    if (pathname === '/__gym/routes') {
+        send(res, 200, opts.router.operations.map((o) => ({
+            method: o.method.toUpperCase(),
+            path: o.path,
+            operationId: o.operationId,
+            status: o.success.status,
+            body: Boolean(o.success.schema),
+            paging: o.paging,
+            key: o.key,
+            source: o.source,
+        })));
+        return;
+    }
+    if (pathname === '/__gym/health') {
+        send(res, 200, { ok: true, operations: opts.router.operations.length });
+        return;
+    }
+    send(res, 404, { error: `no such endpoint: ${pathname}` });
+}
+function prepareSend(res, status, value) {
+    const text = `${JSON.stringify(value, null, 2)}\n`;
+    res.statusCode = status;
+    res.setHeader('content-type', 'application/json; charset=utf-8');
+    res.setHeader('content-length', Buffer.byteLength(text));
+    return () => res.end(text);
+}
+function send(res, status, value) {
+    prepareSend(res, status, value)();
+}
+function prepareHtml(res, text) {
+    res.statusCode = 200;
+    res.setHeader('content-type', 'text/html; charset=utf-8');
+    res.setHeader('content-length', Buffer.byteLength(text));
+    return () => res.end(text);
+}
+/** Node drops the body itself when the request was a HEAD. */
+function html(res, text) {
+    prepareHtml(res, text)();
+}
+//# sourceMappingURL=server.js.map

package/dist/setup.d.ts

@@ -0,0 +1,53 @@
+import { KeyStore } from '@zenera/cli/lib';
+import { type Model } from '@zenera/neo';
+import { Box } from './box.ts';
+import { Cache, type CacheOptions } from './cache.ts';
+import { Router } from './router.ts';
+import { SpecError } from './spec.ts';
+import { Checks } from './validate.ts';
+/** The variable a shell or a CI job names it with — a zen ref. */
+export declare const MODEL_ENV = "ZENERA_GYM_MODEL";
+export type ModelSource = 'flag' | 'env' | 'store' | 'default';
+export declare const SOURCE_LABELS: Record<ModelSource, string>;
+export interface ModelChoice {
+    ref: string;
+    from: ModelSource;
+}
+export interface GymFile {
+    version: 1;
+    model?: string;
+}
+export declare const readSettings: () => Promise<GymFile>;
+export declare function writeSettings(file: GymFile): void;
+/** What `open` would run, and where it came from, so a command can say so too. */
+export declare function chooseModel(flag: string | undefined, keys: KeyStore): Promise<ModelChoice | undefined>;
+export interface SetupOptions {
+    specs: readonly string[];
+    cwd: string;
+    /** where generators and the container's workspace live */
+    cache?: string;
+    model?: string;
+    image?: string;
+    attempts?: number;
+    /** how many generators may be written at once */
+    concurrency?: number;
+    rebuild?: boolean;
+    ephemeral?: boolean;
+    timeout?: number;
+    onImageBuild?: (tag: string) => void;
+    events?: Pick<CacheOptions, 'onStart' | 'onAttempt' | 'onReady' | 'onFail'>;
+}
+export interface Setup {
+    router: Router;
+    checks: Checks;
+    cache: Cache;
+    box: Box;
+    model: Model;
+    choice: ModelChoice;
+    image: string;
+    root: string;
+    close(): Promise<void>;
+}
+export declare function open(opts: SetupOptions): Promise<Setup>;
+export { SpecError };
+//# sourceMappingURL=setup.d.ts.map