@mistriwork/engine 0.0.0-stage → 1.0.0-rc.10

package/dist/server/http/pageRoutes.js

@@ -0,0 +1,291 @@
+import { pageAbouts } from '../../shared/pageModel.js';
+import { resolveOf, validatePage } from '../../shared/page.js';
+import { planPageSections } from '../../shared/pagePlans.js';
+import { filtersPath, filtersSection, writeLayout } from '../layouts.js';
+import { readRecords, readCounts } from '../resourceReadOperations.js';
+import { AppAccessDenied } from '../columnGrants.js';
+import { inBatches } from '../impact.js';
+import { referencesTo, findTable, labelColumn, primaryKey } from '../../shared/schema.js';
+import { httpRead, send } from './reply.js';
+export async function pageRoutes(post, hooks) {
+    const { res, path, body, adapter, schema, registry, connection, configPages, access, principal, rowPolicy, appNow } = post;
+    const rules = hooks.rules;
+    /**
+     * The saved list filters for one list (`list`: an App list view's key,
+     * not a table — two views over one table keep their own): a list of
+     * {name, query}, written whole — the client owns ordering and dedupe, and reads the current list
+     * before it writes — or null to remove the entry. The same store
+     * discipline as saved pages, for the same reason: a filter that lives in
+     * one browser dies with the browser. In a signed-in App the write lands in
+     * the verified account's own section (layouts.ts); the body cannot name
+     * another, and a request with no account that may keep one is refused.
+     */
+    if (path === '/api/filters/save') {
+        const section = filtersSection(connection, { signIn: Boolean(rules?.signsIn(post)), account: principal, rowPolicy });
+        if (section === null) {
+            send(res, 403, { error: 'Saved views are kept per account here, and this request has none that may keep them.' });
+            return true;
+        }
+        const list = String(body.list ?? '');
+        if (!list) {
+            send(res, 400, { error: 'Saved filters belong to a list.' });
+            return true;
+        }
+        const filters = body.filters;
+        if (filters !== null && !Array.isArray(filters)) {
+            send(res, 400, { error: 'Filters must be a list, or null to remove them.' });
+            return true;
+        }
+        /* Shape-checked per entry, refused not coerced: the store round-trips
+           whatever it is handed, so junk accepted here is junk served to every
+           browser from now on. */
+        if (Array.isArray(filters)) {
+            for (const f of filters) {
+                const shaped = f &&
+                    typeof f === 'object' &&
+                    typeof f.name === 'string' &&
+                    typeof f.query === 'string';
+                if (!shaped) {
+                    send(res, 400, { error: 'Each saved filter must be { name, query }, both strings.' });
+                    return true;
+                }
+            }
+        }
+        try {
+            // An array is as opaque to the store as a layout object; it only
+            // round-trips what the client wrote.
+            await writeLayout(section, list, filters ?? null, filtersPath());
+        }
+        catch (err) {
+            /* Unlike layouts, filters keep no localStorage mirror — a failed write
+               means the filter is saved nowhere, and the message must say so. */
+            send(res, 500, rules
+                ? { error: 'The filter could not be saved. Try again shortly.', code: 'definition_save_failed' }
+                : { error: `The filter was not saved — the filters file could not be written: ${err.message}` });
+            return true;
+        }
+        send(res, 200, { ok: true });
+        return true;
+    }
+    /**
+     * What each section of a page needs to run, for one root record.
+     *
+     * One call for the whole page rather than one per section, and it returns
+     * *plans* rather than results: the client runs each view itself, so a count
+     * over a large child table does not hold up the record's own name. Waiting
+     * for all of them would make the slowest section the speed of the page.
+     *
+     * The views are built here because building them is path resolution against
+     * a schema, and a second copy of those rules in the browser is the drift
+     * this codebase has paid for in every feature that tried it.
+     *
+     * A section that cannot be built comes back with its reason, in place. One
+     * bad path must not cost the other five sections — a page is most useful on
+     * exactly the day something is misconfigured.
+     */
+    if (path === '/api/pages/plan') {
+        const key = (body.key ?? {});
+        const name = registry.list().find((c) => c.id === connection)?.name;
+        /* A page arrives either as a name or as itself.
+    
+           Pages come from three places: the config file, the schema, and the
+           builder — and the third lives in the browser's own storage, which the
+           server has never seen. Planning by id alone meant a page someone had
+           just built could be listed, opened and titled, and then rendered as
+           "there is no page called that", because the only part that needed the
+           schema was the part that had never heard of it.
+    
+           Handing the definition over is safe for the same reason a filter is: it
+           is structured, it is validated against this connection before anything
+           runs, and every value in it ends up bound rather than written into
+           SQL. */
+        const asked = body.page;
+        const found = typeof asked === 'object' && asked !== null
+            ? asked
+            : String(asked ?? '').startsWith('suggested:')
+                ? hooks.suggested(post, String(asked).slice('suggested:'.length))
+                : configPages.find((p) => p.id === String(asked ?? '') && (!p.connection || p.connection === connection || p.connection === name));
+        if (!found) {
+            send(res, 404, { error: `There is no page called "${String(asked ?? '')}".` });
+            return true;
+        }
+        if (access && !access.readable.has(found.base))
+            throw new AppAccessDenied();
+        if (access?.fields && rules && !rules.readsPage(schema, found))
+            throw new AppAccessDenied();
+        if (typeof found.base !== 'string' || !Array.isArray(found.sections)) {
+            send(res, 400, { error: 'That is not a page: it needs a "base" table and a list of sections.' });
+            return true;
+        }
+        const page = validatePage(schema, found);
+        if (page.error) {
+            send(res, 400, { error: page.error });
+            return true;
+        }
+        const base = findTable(schema, page.base);
+        const missing = primaryKey(base).filter((c) => key[c] === undefined);
+        if (missing.length) {
+            send(res, 400, {
+                error: `This page is about one ${base.name}, and the key is missing ${missing.join(', ')}.`,
+            });
+            return true;
+        }
+        if (rowPolicy && rules)
+            await rules.requireRecord(adapter, schema, page.base, key);
+        /* Sections about a record the root points at need that record's key,
+           which only the root row knows. One read, of just the reference
+           columns, shared by every such section — and a reference that is null
+           resolves to `null`, which the section renders as its empty state
+           rather than as a filter that can never match. */
+        const ofs = pageAbouts(page);
+        const ofKeys = {};
+        if (ofs.length) {
+            const resolvedOfs = ofs.map((of) => ({ of, ...resolveOf(schema, page.base, of) })).filter((r) => r.fk);
+            const wanted = [...new Set(resolvedOfs.flatMap((r) => r.fk.from.columns))];
+            if (wanted.length) {
+                const row = (await httpRead(() => readRecords(adapter, schema, {
+                    table: page.base,
+                    filter: { groups: [primaryKey(base).map((c) => ({ column: c, op: '=', value: key[c] }))] },
+                    columns: wanted,
+                    limit: 1,
+                    offset: 0,
+                }))).result.rows[0];
+                for (const r of resolvedOfs) {
+                    const values = r.fk.from.columns.map((c) => row?.[c]);
+                    ofKeys[r.of] = values.every((v) => v !== null && v !== undefined)
+                        ? Object.fromEntries(r.fk.to.columns.map((c, i) => [c, values[i]]))
+                        : null;
+                }
+            }
+        }
+        /* A matched list shares the record's own values, so they are read here,
+           on the server, from the record's row — never taken from the request.
+           Read through the same scoped adapter, so a value the reader may not
+           read is refused like any other read of it. */
+        const likeColumns = [
+            ...new Set(page.sections.flatMap((s) => (s.kind === 'list' && s.similar ? [...s.similar.same, ...Object.keys(s.similar.near ?? {})] : []))),
+        ];
+        let rootValues;
+        if (likeColumns.length) {
+            rootValues = ((await httpRead(() => readRecords(adapter, schema, {
+                table: page.base,
+                filter: { groups: [primaryKey(base).map((c) => ({ column: c, op: '=', value: key[c] }))] },
+                columns: likeColumns,
+                limit: 1,
+                offset: 0,
+            }))).result.rows[0] ?? {});
+        }
+        // Capture once after root lookup: every section uses the same App date.
+        const plans = planPageSections(schema, page, key, ofKeys, appNow(), rootValues);
+        send(res, 200, { page, plans });
+        return true;
+    }
+    /**
+     * The endpoint the whole tool exists for: given one row, how many rows in
+     * every other table point at it, and what would you click to see them.
+     *
+     * Counting is done server-side and in parallel because a row in a
+     * well-normalised schema can be referenced from a dozen places, and a dozen
+     * sequential round trips is the difference between instant and sluggish.
+     */
+    if (path === '/api/references') {
+        const tableId = String(body.table ?? '');
+        const key = (body.key ?? {});
+        const table = findTable(schema, tableId);
+        if (!table) {
+            send(res, 404, { error: `Unknown table "${tableId}"` });
+            return true;
+        }
+        const incoming = referencesTo(schema, tableId);
+        /* A declared reference may point at a column that is not this row's key
+           (a code, a slug): its value is the row's, so the row is read for it,
+           in the one read a row policy makes to admit the record anyway. */
+        const unkeyed = [...new Set(incoming.flatMap((fk) => fk.to.columns))].filter((column) => !Object.hasOwn(key, column));
+        let row = key;
+        if (rowPolicy && rules)
+            row = { ...(await rules.readRecord(adapter, schema, tableId, key, unkeyed)), ...key };
+        else if (unkeyed.length && Object.keys(key).length) {
+            const { result } = await readRecords(adapter, schema, {
+                table: table.id,
+                columns: unkeyed,
+                limit: 1,
+                offset: 0,
+                filter: { groups: [Object.entries(key).map(([column, value]) => ({ column, op: '=', value }))] },
+            });
+            row = { ...result.rows[0], ...key };
+        }
+        /* Every reference counted in one statement where the reader can say it
+           in one (readCounts), each bounded: past DEFAULT_COUNT_LIMIT the answer
+           is the bound and `capped`, and the panel says "10,000+". These were a
+           filtered, unbounded COUNT(*) apiece, four at a time — on a hub table
+           with a dozen incoming keys, a dozen scans against the connection
+           serving the page they are for, most of them over unindexed columns. */
+        const described = incoming
+            .map((fk) => {
+            /* Every column of the key has to match, which is what makes this
+             correct for composite keys and not just single-column ones. */
+            const conditions = fk.to.columns.map((toCol, i) => ({
+                column: fk.from.columns[i] ?? fk.from.columns[0],
+                op: '=',
+                value: row[toCol],
+            }));
+            const usable = conditions.every((c) => c.value !== undefined && c.value !== null);
+            const filter = { groups: [conditions] };
+            return usable
+                ? {
+                    fk,
+                    filter,
+                    base: {
+                        table: fk.from.table,
+                        columns: fk.from.columns,
+                        // The matching key columns on this row's side, so the client can
+                        // pair them by position rather than guessing from key order.
+                        toColumns: fk.to.columns,
+                        constraint: fk.name,
+                        filter,
+                        label: labelColumn(findTable(schema, fk.from.table)),
+                    },
+                }
+                : null;
+        })
+            .filter((item) => item !== null);
+        /* A source that reports no totals (an API) is not asked for one: the
+           reference is listed without a count, and the panel's own read of its
+           rows says what is there. Asking would only ever be refused. */
+        const countable = described.filter((item) => findTable(schema, item.fk.from.table)?.countable !== false);
+        const failed = (err) => {
+            const said = rules?.said(err);
+            return { count: -1, error: said ? said.message : err.message, ...(said ? { code: said.code } : {}) };
+        };
+        const counts = new Map();
+        try {
+            const answered = countable.length
+                ? await readCounts(adapter, schema, countable.map((item) => ({ table: item.fk.from.table, filter: item.filter })))
+                : [];
+            countable.forEach((item, i) => counts.set(item.fk.name, { count: answered[i].count, ...(answered[i].capped ? { capped: true } : {}) }));
+        }
+        catch {
+            /* One unreadable table (permissions, a dropped view) should not blank
+               the whole panel: asked again one at a time, each failure stays in
+               its own row. A small window, as before, so the record itself can
+               still be answered on the same connection. */
+            await inBatches(countable, 4, async (item) => {
+                try {
+                    const [one] = await readCounts(adapter, schema, [{ table: item.fk.from.table, filter: item.filter }]);
+                    counts.set(item.fk.name, { count: one.count, ...(one.capped ? { capped: true } : {}) });
+                }
+                catch (err) {
+                    counts.set(item.fk.name, failed(err));
+                }
+            });
+        }
+        const results = described.map((item) => ({ ...item.base, ...counts.get(item.fk.name) }));
+        send(res, 200, {
+            table: tableId,
+            primaryKey: primaryKey(table),
+            references: results,
+        });
+        return true;
+    }
+    return false;
+}

package/dist/server/http/probes.d.ts

@@ -0,0 +1,22 @@
+/**
+ * `/healthz` and `/readyz`, for an orchestrator.
+ *
+ * `/healthz` is liveness: the process answers HTTP. `/readyz` is readiness:
+ * it should be sent traffic now — false while it starts, while a row policy
+ * restarts after a save, and from the moment it begins shutting down, so a
+ * load balancer stops sending before the listener closes. Both are answered
+ * before any gate or App, carry nothing but their word, are never cached and
+ * are not traced (a heartbeat every few seconds is not a trace).
+ *
+ * Both answer `{ deploy, build }` beside the status (`mistri deploy` verifies a
+ * roll by them): `deploy` is MISTRI_DEPLOY_ID, which the platform's Worker sets
+ * in the container's environment on each deploy, and `build` is the id of the
+ * build being served. Both are random or a hash, so safe to expose.
+ *
+ * The operator reads with more in them (`/api/operations/readiness`) are
+ * signed-in and granted; these are not, and so say nothing else.
+ */
+import type { IncomingMessage, ServerResponse } from 'node:http';
+/** Where `/readyz` reads the served build's id from; the App framework registers it when it serves a build. */
+export declare function setBuildIdSource(source: () => string | null): void;
+export declare function probe(req: IncomingMessage, res: ServerResponse, path: string, ready?: () => boolean): boolean;

package/dist/server/http/probes.js

@@ -0,0 +1,27 @@
+let buildSource = () => null;
+/** Where `/readyz` reads the served build's id from; the App framework registers it when it serves a build. */
+export function setBuildIdSource(source) {
+    buildSource = source;
+}
+export function probe(req, res, path, ready) {
+    if (path !== '/healthz' && path !== '/readyz')
+        return false;
+    if (req.method !== 'GET' && req.method !== 'HEAD') {
+        res.writeHead(405, { allow: 'GET, HEAD', 'cache-control': 'no-store' }).end();
+        return true;
+    }
+    let ok = true;
+    if (path === '/readyz') {
+        try {
+            ok = ready?.() ?? true;
+        }
+        catch {
+            ok = false;
+        }
+    }
+    res.writeHead(ok ? 200 : 503, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' });
+    res.end(req.method === 'HEAD'
+        ? undefined
+        : JSON.stringify({ status: ok ? 'ok' : 'unavailable', deploy: process.env.MISTRI_DEPLOY_ID || null, build: buildSource() }));
+    return true;
+}

package/dist/server/http/rateLimit.d.ts

@@ -0,0 +1,131 @@
+/**
+ * A fixed-window limiter per client, for the one surface a signed-out person
+ * may write to: a public form. In memory by default, so a restart forgets
+ * and a second process does not share; when the config names a cache
+ * (`"kind": "cache"`), the same windows are counted there instead, so every
+ * instance of a deployment spends one budget (`SharedWindowLimiter`). The
+ * limits are deployment-owned defaults and not configurable. The address is
+ * the one `forwardedClient` resolves: a forwarded header only when the socket
+ * peer is a trusted proxy — on this machine, or named by `--trust-proxy` —
+ * else the peer itself, so a proxy on another host that nobody named still
+ * collapses its clients to one address, which the docs say.
+ *
+ * Bounded, because the clients are whoever the network sends. The review
+ * that asked for this measured the first version: past its cap it scanned
+ * every tracked address on each new one and dropped only the fully expired,
+ * which under the one-hour window was none — 40,000 addresses held the event
+ * loop for 21 seconds, and every address in one IPv6 /64 had its own budget.
+ * Now: a hard cap on tracked clients, a Map kept in least-recently-seen
+ * order (a touch re-inserts, the front is the stalest) so expiry is a sweep
+ * from the front that stops at the first live entry — O(1) amortised, no
+ * scan and no timer — and an IPv6 client is its /64.
+ */
+import type { IncomingMessage } from 'node:http';
+import { type CacheStore } from '../../adapters/cache/cacheStore.js';
+import { type ProxyRange } from './trust.js';
+interface RateWindow {
+    limit: number;
+    ms: number;
+}
+/** `scope` says whose budget ran out: this client's, or the one every client shares. */
+type RateDecision = {
+    ok: true;
+    remaining: number;
+} | {
+    ok: false;
+    retryAfterSeconds: number;
+    scope: 'client' | 'all';
+};
+/**
+ * Which budgets a take spends: the client's and the shared ceiling together (`'both'`, the default), or one of
+ * them. A door that proves its caller (a ticket, a human check) spends the client's first and the ceiling only
+ * once the proof passes, so a flood that cannot prove anything spends its own addresses' budgets, never the one
+ * every visitor shares.
+ */
+export type RatePart = 'both' | 'client' | 'all';
+/** A limiter that decides now (in memory) or once the shared cache has counted. */
+interface WindowLimiter {
+    take(address: string, part?: RatePart): RateDecision | Promise<RateDecision>;
+    /** Forget this process's own counts; tests and a policy reload use it. A shared cache's windows expire on their own. */
+    reset(): void;
+}
+interface LimiterOptions {
+    /** The most clients tracked at once. */
+    maxClients?: number;
+    /**
+     * A new client, when every slot holds a budget still in force: `refuse` it
+     * until the stalest slot expires, or `evict` the stalest and admit it.
+     */
+    whenFull?: 'refuse' | 'evict';
+    /** Windows over every client together: what rotating addresses cannot exceed. */
+    total?: readonly RateWindow[];
+}
+/**
+ * The key a client's budget is kept under. IPv4 as it is; an IPv4-mapped
+ * IPv6 address (`::ffff:192.0.2.1`, how a dual-stack socket reports an IPv4
+ * peer) as that IPv4 address; any other IPv6 address as its /64, the
+ * smallest network one subscriber is routinely given — one host can
+ * otherwise draw a fresh budget from each of its 2^64 addresses. Anything
+ * that is not an address (`unknown`) is its own key.
+ */
+export declare function limitKey(address: string): string;
+export declare class FixedWindowLimiter implements WindowLimiter {
+    private readonly windows;
+    private readonly now;
+    private readonly hits;
+    private readonly longest;
+    private readonly maxClients;
+    private readonly whenFull;
+    private readonly totalWindows;
+    private total;
+    constructor(windows: readonly RateWindow[], now?: () => number, options?: LimiterOptions);
+    /** Count one request for the client, or say how long it waits. */
+    take(address: string, part?: RatePart): RateDecision;
+    private takeShared;
+    /** How many clients are tracked; tests read it. */
+    get size(): number;
+    /** Forget every client and the shared count; tests and a policy reload use it. */
+    reset(): void;
+    private expire;
+}
+/**
+ * The same windows, counted in the deployment's cache so every instance
+ * spends one budget. Each window is a key per client (and one for everyone)
+ * counted by the cache's atomic windowed count, which starts the window on
+ * the first request; a refused request is counted too, which only keeps an
+ * exceeded window exceeded until it ends on its own. There is no cap on
+ * tracked clients to keep: every key expires with its window.
+ *
+ * When the cache cannot count, the process's own limiter decides — the
+ * budget degrades to one instance's, never to none.
+ */
+export declare class SharedWindowLimiter implements WindowLimiter {
+    private readonly name;
+    private readonly windows;
+    private readonly cache;
+    private readonly fallback;
+    private readonly totalWindows?;
+    constructor(name: string, windows: readonly RateWindow[], cache: CacheStore, fallback: FixedWindowLimiter, totalWindows?: readonly RateWindow[] | undefined);
+    take(address: string, part?: RatePart): Promise<RateDecision>;
+    reset(): void;
+    private count;
+}
+/** The client a limit counts against; a forwarded header only from a trusted peer (loopback, or `--trust-proxy`). */
+export declare function clientAddress(req: Pick<IncomingMessage, 'socket' | 'headers'>, trusted?: readonly ProxyRange[]): string;
+export declare const PUBLIC_FORM_SUBMIT_WINDOWS: readonly RateWindow[];
+export declare const PUBLIC_FORM_SUBMIT_TOTAL: readonly RateWindow[];
+export declare const PUBLIC_FORM_READ_WINDOWS: readonly RateWindow[];
+export declare const INVITATION_CLAIM_WINDOWS: readonly RateWindow[];
+export declare const INVITATION_CLAIM_TOTAL: readonly RateWindow[];
+export declare function publicFormLimiter(kind: 'submit' | 'read' | 'start' | 'status' | 'link' | 'code' | 'check'): FixedWindowLimiter | SharedWindowLimiter;
+/** The invitation claims' own budget (tenantInvitationHttp.ts), never the forms'. */
+export declare function invitationClaimLimiter(): FixedWindowLimiter | SharedWindowLimiter;
+/**
+ * Count the public forms' windows in the deployment's cache, under the
+ * deployment's namespace (`tw:<app>:form:`), or — with no cache, or the
+ * process's own — in this process, as before.
+ */
+export declare function configurePublicFormLimits(cache: CacheStore | undefined, namespace: string): void;
+/** Forgets the forms' budgets and the claims'; tests and a policy reload use it. */
+export declare function resetPublicFormLimits(): void;
+export {};