@mistriwork/engine 0.0.0-stage → 1.0.0-rc.10

package/dist/server/http/door.d.ts

@@ -0,0 +1,57 @@
+/**
+ * The door a dispatcher holds every request to before any route reads it,
+ * in one order: the Host, the proxy's identity, the product's session, then
+ * a write's Origin and body (trust.ts `doorRefusal`). The order is the
+ * defence, so it is kept here once rather than spelled in each dispatcher:
+ *
+ *   - the Host first, on every request, read or write, page or endpoint. The
+ *     name tablewalk was reached by is the one thing a rebinding attack
+ *     cannot fake, so it is checked before anything is served — including
+ *     index.html, so the attempt fails at the door rather than at the first
+ *     fetch;
+ *   - the proxy's identity (`--auth-proxy-header`), unless the product says
+ *     the request names no one whatever its headers carry;
+ *   - the product's session (`SessionDoor`): an App's sign-in, its public
+ *     forms and pages, a host's bearer on another origin. The database
+ *     browser has none;
+ *   - a write's Origin and JSON body last, because the session part decides
+ *     whether an Origin is vouched for (embed/crossOrigin.ts `embedDoor`).
+ *
+ * Returns who is asking and what the session established, or `undefined`
+ * when the door already answered — in which case the caller is done.
+ */
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import { type Bound } from './trust.js';
+import type { Identity } from './context.js';
+/** Which step refused: a dispatcher answers each in its own shape (a command's, the API's, a page's). */
+type DoorCheck = 'host' | 'identity' | 'origin';
+interface DoorAsk {
+    readonly req: IncomingMessage;
+    readonly res: ServerResponse;
+    readonly bound: Bound;
+    /** Answer a refusal, in the dispatcher's own shape. */
+    readonly refuse: (check: DoorCheck, status: 401 | 403, error: string) => void;
+}
+/** What a request arriving at the session part is, once the Host has passed. */
+interface Arrival {
+    /** The session part answered it already: a preflight from a host on another origin. */
+    readonly answered?: boolean;
+    /** It names no one, whatever its headers say: a public storefront's request (http/tenantHosts.ts). */
+    readonly anonymous?: boolean;
+}
+/** A product's part of the door, between the proxy's identity and a write's Origin. */
+interface SessionDoor<S> {
+    /** After the Host, before the proxy identity is read. */
+    arrive?(): Arrival | Promise<Arrival>;
+    /**
+     * The session over the proxy's identity, which it completes in place (the
+     * account, its roles and name): what it established, or `undefined` when it
+     * answered the request.
+     */
+    admit(identity: Identity): Promise<S | undefined>;
+}
+export declare function door<S>(ask: DoorAsk, session: SessionDoor<S>): Promise<{
+    identity: Identity;
+    session: S;
+} | undefined>;
+export {};

package/dist/server/http/door.js

@@ -0,0 +1,39 @@
+import { doorRefusal, identityFor, writes } from './trust.js';
+import { admitAs } from '../../adapters/poolAdmission.js';
+export async function door(ask, session) {
+    const { req, bound, refuse } = ask;
+    /* A write's Origin and body are the door's second asking, below: the
+       session part decides first whether its Origin is vouched for. */
+    const wrongHost = doorRefusal(req, bound, { writing: false })?.error;
+    if (wrongHost) {
+        refuse('host', 403, wrongHost);
+        return undefined;
+    }
+    const arrival = (await session.arrive?.()) ?? {};
+    if (arrival.answered)
+        return undefined;
+    const identity = arrival.anonymous ? {} : identityFor(req, bound);
+    if (identity.refusal) {
+        refuse('identity', 401, identity.refusal);
+        return undefined;
+    }
+    const admitted = await session.admit(identity);
+    if (admitted === undefined)
+        return undefined;
+    /* Signed in, its pool checkouts wait their turn as the account's, not its
+       address's (adapters/poolAdmission.ts): one person's tabs on two networks
+       are one share of the line, and an office behind one address is many. */
+    admitAs(identity.id ?? identity.who);
+    /* Every request that can change something is checked before it is read, so
+       a refusal costs nothing and happens before any work. Admitted on a bearer
+       (embedDoor), the Origin was checked against the one the bearer was issued
+       to; the Host and the JSON body still are. */
+    if (writes(req)) {
+        const refusal = doorRefusal(req, bound, { writing: true })?.error;
+        if (refusal) {
+            refuse('origin', 403, refusal);
+            return undefined;
+        }
+    }
+    return { identity, session: admitted };
+}

package/dist/server/http/exportRoute.d.ts

@@ -0,0 +1,15 @@
+import { parseQuery } from '../../shared/query.js';
+import type { PostContext } from './context.js';
+/** Who `me` is in a question, where its words name `me` (parseQuery's `me`). */
+type Person = NonNullable<Parameters<typeof parseQuery>[1]>['me'];
+/** What a product decides of an export. */
+export interface ExportHooks {
+    /** Whether this request's person may export a table (`''`: any table); `/api/run` bounds its reads by the same answer. */
+    may(post: PostContext): (tableId: string) => boolean;
+    /** Who `me` is in the question `text`, read only where its words name `me`. */
+    person?(post: PostContext, text: string): Promise<Person | undefined>;
+    /** A row as a CSV says it: an amount kept in minor units as the figure it is, a value as its component's word. */
+    amounts?(post: PostContext, tableId: string): ((row: Record<string, unknown>) => Record<string, unknown>) | undefined;
+}
+export declare function exportRoute(post: PostContext, hooks: ExportHooks): Promise<boolean>;
+export {};

package/dist/server/http/exportRoute.js

@@ -0,0 +1,320 @@
+import { createBrotliCompress, createGzip, constants as zlib } from 'node:zlib';
+import { MAX_LIMIT, clampLimit } from '../../shared/reads.js';
+import { parseQuery } from '../../shared/query.js';
+import { textToView } from '../../shared/viewtext.js';
+import { findTable, primaryKey, resolveOrder } from '../../shared/schema.js';
+import { backrefsTo, resolveAggregate, tryResolvePath } from '../../shared/viewPaths.js';
+import { CRLF, csvField, csvRecord, jsonRecord } from '../../shared/csv.js';
+import { readRecords, readView } from '../resourceReadOperations.js';
+import { acceptedEncoding, httpRead, send } from './reply.js';
+import { rollupReads } from '../../shared/rollups.js';
+const EXPORT_PATH = '/api/export';
+/** Rows read per statement: large enough that a 100,000-row file is a hundred
+ * reads, small enough that one batch is never a memory event. */
+const BATCH = 1000;
+const FORMATS = {
+    csv: { type: 'text/csv', extension: 'csv' },
+    json: { type: 'application/json', extension: 'json' },
+    ndjson: { type: 'application/x-ndjson', extension: 'ndjson' },
+};
+export async function exportRoute(post, hooks) {
+    const { res, path, body, adapter, schema, appNow } = post;
+    if (path !== EXPORT_PATH)
+        return false;
+    /* An App that grants by role says who exports which table (appExportable); the file is refused where the Export menu is not offered. */
+    const may = hooks.may(post);
+    if (!may('')) {
+        send(res, 403, { error: 'Export is not available to this person.' });
+        return true;
+    }
+    // Never reached in one (httpRowPolicyAllows), and refused here as well: a policy reply cannot stream.
+    if (post.rowPolicy) {
+        send(res, 403, {
+            error: 'This operation is unavailable in a row-policy application.',
+        });
+        return true;
+    }
+    const format = body.format ?? 'csv';
+    if (typeof format !== 'string' || !Object.hasOwn(FORMATS, format)) {
+        send(res, 400, { error: 'format must be "csv", "json" or "ndjson".' });
+        return true;
+    }
+    if (body.columns !== undefined && (!Array.isArray(body.columns) || body.columns.some((column) => typeof column !== 'string'))) {
+        send(res, 400, { error: 'columns must be a list of column names.' });
+        return true;
+    }
+    const text = String(body.q ?? '');
+    const me = await hooks.person?.(post, text);
+    const person = me === undefined ? {} : { me };
+    const parsed = parseQuery(text, { schema, now: appNow(), ...person });
+    /* A question only a view can answer (a path through a reference) exports
+       as a view, paged by offset: its rows have no single key to resume from. */
+    if ((!parsed.query || parsed.errors.length) && adapter.runView) {
+        const asView = textToView(schema, text, {
+            id: 'export',
+            name: '',
+            now: appNow(),
+            ...person,
+        });
+        if (asView.view && !asView.errors.length) {
+            const view = asView.view;
+            /* Every table the view reaches, not only its base: a column through a reference, an aggregate or a filter on one reads another table's rows into the file. */
+            for (const reached of viewTables(schema, view))
+                if (!may(reached))
+                    return refuseTable(res, reached === UNKNOWN_TABLE ? view.base : reached);
+            const total = clampLimit(view.limit ?? MAX_LIMIT);
+            const read = (offset) => httpRead(() => readView(adapter, schema, view, { limit: Math.min(BATCH, total - offset), offset }, 'none')).then(({ result }) => result);
+            await stream(res, format, view.base, total, read, requested(body) ?? undefined, hooks.amounts?.(post, view.base));
+            return true;
+        }
+    }
+    if (!parsed.query) {
+        send(res, 200, { errors: parsed.errors });
+        return true;
+    }
+    if (parsed.errors.length) {
+        send(res, 200, { errors: parsed.errors });
+        return true;
+    }
+    const query = parsed.query;
+    const table = findTable(schema, query.table);
+    if (!table) {
+        send(res, 404, { error: `Unknown table "${query.table}"` });
+        return true;
+    }
+    if (!may(table.id))
+        return refuseTable(res, table.id);
+    const wanted = requested(body) ?? (query.columns?.length ? query.columns : undefined);
+    for (const reached of queryTables(schema, table, query, wanted))
+        if (!may(reached))
+            return refuseTable(res, reached === UNKNOWN_TABLE ? table.id : reached);
+    const total = clampLimit(query.limit ?? MAX_LIMIT);
+    const order = resolveOrder(table, query.orderBy).order;
+    const keyset = keysetOrder(table, order, schema.dialect);
+    /* The key is read even when the file does not show it: it is where the
+       next batch starts. It never reaches the file (`wanted` is the file's). */
+    const reading = wanted && keyset ? [...new Set([...wanted, ...keyset.map((o) => o.column)])] : wanted;
+    let after;
+    const read = async (offset) => {
+        const { result } = await httpRead(() => readRecords(adapter, schema, {
+            ...query,
+            ...(reading ? { columns: reading } : {}),
+            orderBy: order,
+            filter: keyset && after ? beyond(query.filter, keyset, after) : query.filter,
+            limit: Math.min(BATCH, total - offset),
+            offset: keyset ? 0 : offset,
+        }, 'none'));
+        after = result.rows.at(-1);
+        return result;
+    };
+    await stream(res, format, table.id, total, read, wanted, hooks.amounts?.(post, table.id));
+    return true;
+}
+function refuseTable(res, tableId) {
+    send(res, 403, {
+        error: `Export of "${tableId}" is not available to this person.`,
+    });
+    return true;
+}
+/** Stands for a table a view reaches that cannot be named (a many-valued condition it cannot resolve): never exportable. */
+const UNKNOWN_TABLE = '\u0000unknown';
+/** Every table a view reads from: its base, each table along a reference path in its columns, filter and order, and the rows its aggregates and `any` conditions read. */
+function viewTables(schema, view) {
+    const tables = new Set([view.base]);
+    const path = pathReach(schema, view.base, tables);
+    for (const column of view.columns)
+        path(column.path);
+    for (const order of view.orderBy ?? [])
+        if (!order.alias)
+            path(order.path);
+    for (const group of view.filter?.groups ?? []) {
+        for (const condition of group) {
+            if (condition.any) {
+                const child = backrefsTo(schema, view.base).find((b) => b.name === condition.any.via);
+                tables.add(child ? child.child : UNKNOWN_TABLE);
+            }
+            else
+                path(condition.path);
+            if (condition.other)
+                path(condition.other);
+        }
+    }
+    // The ones pinned to the view already (viewLabels.ts withViewComputed), as a read pins them.
+    for (const rollup of Object.values(view.computed ?? {}))
+        for (const { table } of rollupReads(rollup))
+            tables.add(table);
+    for (const expressed of Object.values(view.expressions ?? {}))
+        for (const read of expressed.reads)
+            path(expressed.on ? `${expressed.on}.${read}` : read);
+    for (const aggregate of view.aggregates ?? []) {
+        try {
+            const resolved = resolveAggregate(schema, view.base, aggregate);
+            tables.add(resolved.child.id);
+            tables.add(resolved.parent.id);
+        }
+        catch {
+            tables.add(UNKNOWN_TABLE);
+        }
+    }
+    return tables;
+}
+/** Every table a single-table query's file reads: its own, and those its computed columns and rollups read. */
+function queryTables(schema, table, query, wanted) {
+    const tables = new Set([table.id]);
+    const path = pathReach(schema, table.id, tables);
+    // A stored column is the table's own; an unknown one the read refuses by name.
+    const derived = (name) => Object.hasOwn(table.rollups ?? {}, name) || Object.hasOwn(table.computed ?? {}, name);
+    const names = [
+        ...(wanted ?? table.columns.map((each) => each.name)),
+        ...(query.orderBy ?? []).map((order) => order.column),
+        ...(query.filter?.groups ?? []).flatMap((group) => group.flatMap((condition) => [condition.column, ...(condition.other ? [condition.other] : [])])),
+    ];
+    for (const name of new Set(names))
+        if (derived(name))
+            path(name);
+    return tables;
+}
+/* Adds to `tables` every table a path from `base` reads. */
+function pathReach(schema, base, tables) {
+    const seen = new Set();
+    const path = (text) => {
+        const { resolved } = tryResolvePath(schema, base, text);
+        if (!resolved) {
+            tables.add(UNKNOWN_TABLE);
+            return;
+        }
+        for (const hop of resolved.hops)
+            tables.add(hop.to.table);
+        tables.add(resolved.table.id);
+        /* A rollup is a figure of another table's rows, and a computed column is
+           worked out of the columns it reads: either carries those tables into
+           the file as surely as a column through a reference does. */
+        if (resolved.rollup)
+            for (const { table } of rollupReads(resolved.rollup))
+                tables.add(table);
+        if (resolved.computed && !seen.has(text)) {
+            seen.add(text);
+            const on = text.includes('.') ? text.slice(0, text.lastIndexOf('.') + 1) : '';
+            for (const read of resolved.computed.reads)
+                path(`${on}${read}`);
+        }
+    };
+    return path;
+}
+function requested(body) {
+    return Array.isArray(body.columns) && body.columns.length ? body.columns : undefined;
+}
+/* Types whose values come back exactly as stored on Postgres. */
+const EXACT_TYPE = /^(tiny|small|medium|big)?int(eger)?\d*\b|^(big|small)?serial|char|text|uuid/i;
+/**
+ * The order a keyset can resume, or undefined. Only the key, and only
+ * columns whose value comes back exactly as it is stored: an integer or text
+ * on Postgres — a timestamp comes back to the millisecond and a
+ * float rounded, and resuming "after" a rounded value repeats or skips rows —
+ * and anything on SQLite, which hands back what it holds.
+ */
+function keysetOrder(table, order, dialect) {
+    const key = primaryKey(table);
+    if (!key.length || order.length !== key.length || order.some((o, i) => o.column !== key[i]))
+        return undefined;
+    if (dialect === 'http')
+        return undefined;
+    const exact = order.every((o) => {
+        const column = table.columns.find((c) => c.name === o.column);
+        if (!column || column.nullable)
+            return false;
+        return dialect === 'sqlite' || EXACT_TYPE.test(column.type);
+    });
+    return exact ? order : undefined;
+}
+/**
+ * The rows after `last` in `order`, within the question's own filter: for a
+ * key (a, b) ascending, `a > x OR (a = x AND b > y)`, crossed with each of the
+ * filter's OR-groups so every group keeps all of its own conditions. Values
+ * are bound like any other condition's.
+ */
+function beyond(filter, order, last) {
+    const steps = order.map((o, i) => [
+        ...order.slice(0, i).map((earlier) => ({
+            column: earlier.column,
+            op: '=',
+            value: last[earlier.column],
+        })),
+        {
+            column: o.column,
+            op: o.direction === 'desc' ? '<' : '>',
+            value: last[o.column],
+        },
+    ]);
+    const groups = filter?.groups?.length ? filter.groups : [[]];
+    return {
+        groups: groups.flatMap((group) => steps.map((step) => [...group, ...step])),
+    };
+}
+async function stream(res, format, table, total, read, wanted, amounts) {
+    /* The first batch is read before a byte is written: a refusal, a column
+       that does not exist, a database that is down all still answer as the
+       ordinary error they are, through the handler's own mapping. */
+    let batch = await read(0);
+    const columns = wanted ?? batch.columns;
+    const spec = FORMATS[format];
+    const encoding = acceptedEncoding(res);
+    res.writeHead(200, {
+        'content-type': `${spec.type}; charset=utf-8`,
+        'content-disposition': `attachment; filename="${table.replace(/[^\w.-]+/g, '_')}.${spec.extension}"`,
+        'cache-control': 'private, no-store',
+        'x-content-type-options': 'nosniff',
+        vary: 'Accept-Encoding',
+        ...(encoding ? { 'content-encoding': encoding } : {}),
+    });
+    const out = encoding === 'br'
+        ? createBrotliCompress({
+            params: {
+                [zlib.BROTLI_PARAM_QUALITY]: 4,
+                [zlib.BROTLI_PARAM_MODE]: zlib.BROTLI_MODE_TEXT,
+            },
+        })
+        : encoding === 'gzip'
+            ? createGzip({ level: 6 })
+            : res;
+    if (out !== res)
+        out.pipe(res);
+    const write = (chunk) => new Promise((resolve, reject) => {
+        if (res.destroyed)
+            return reject(new Error('The export was abandoned.'));
+        if (out.write(chunk))
+            resolve();
+        else
+            out.once('drain', resolve);
+    });
+    try {
+        /* A spreadsheet opens a UTF-8 CSV without a byte-order mark in the local
+           codepage; the browser's own export writes one too (export.js). */
+        if (format === 'csv')
+            await write(`${columns.map(csvField).join(',')}${CRLF}`);
+        let written = 0;
+        for (;;) {
+            const lines = batch.rows.map((row, i) => format === 'csv'
+                ? `${csvRecord(columns, amounts ? amounts(row) : row)}${CRLF}`
+                : format === 'ndjson'
+                    ? `${JSON.stringify(jsonRecord(columns, row))}\n`
+                    : // A JSON array as export.js's toJson writes one: two-space indent, one record per element.
+                        `${written + i === 0 ? '[\n' : ',\n'}${JSON.stringify(jsonRecord(columns, row), null, 2).replace(/^/gm, '  ')}`);
+            if (lines.length)
+                await write(lines.join(''));
+            written += batch.rows.length;
+            if (batch.rows.length < BATCH || written >= total)
+                break;
+            batch = await read(written);
+        }
+        if (format === 'json')
+            await write(written ? '\n]\n' : '[]\n');
+        out.end();
+    }
+    catch {
+        /* Past the first byte a status cannot change: the connection is cut, so
+           the browser reports a failed download rather than saving half a file
+           that looks whole. */
+        res.destroy();
+    }
+}

package/dist/server/http/firstPaint.d.ts

@@ -0,0 +1,108 @@
+import type { IncomingMessage } from 'node:http';
+import { type FirstPaintItemKind } from '../../shared/firstPaint.js';
+/** For tests on a busy machine: how long a page waits for its answers. */
+export declare function setFirstPaintBudget(ms: number): void;
+/** One route's reply, as the dispatcher answered a request the server asked itself. */
+export interface LoopAnswer {
+    status: number;
+    body: unknown;
+    /** `x-mistri-account`: who the reply said it was for. */
+    account?: string;
+    /** Whether it may be carried: no cookie set, no header the client reads from a reply. */
+    carriable: boolean;
+}
+/** Ask the dispatcher one request, as this page's reader (handler.ts). */
+export type Loop = (method: 'GET' | 'POST', target: string, body?: Record<string, unknown>, signal?: AbortSignal) => Promise<LoopAnswer | undefined>;
+/**
+ * The request a fetch from this page would make: the page's own headers —
+ * its cookie, its proxy identity, its host — for the page's own origin, as
+ * `fetch` sends them (same-origin, JSON), and the body, if any. Enough of
+ * IncomingMessage for the gate, the auth store and the body reader.
+ */
+export declare function loopbackRequest(page: IncomingMessage, method: 'GET' | 'POST', target: string, body?: Record<string, unknown>): IncomingMessage;
+declare const BATCH_MOST: {
+    readonly '/api/breakdowns': 48;
+    readonly '/api/counts': 64;
+};
+/** What the page's data block holds for one address. */
+interface FirstPaintPage {
+    json?: string;
+}
+/** One request a room makes, exactly as the client sends it. */
+export interface RoomRead {
+    method: 'GET' | 'POST';
+    target: string;
+    body?: Record<string, unknown>;
+}
+/** Reads the client carries in a batch, asked here in one; each item is answered under its own key. */
+export interface RoomBatch {
+    path: keyof typeof BATCH_MOST;
+    kind: FirstPaintItemKind;
+    /** What the batch carries beside its items (`countLimit`). */
+    body?: Record<string, unknown>;
+    /** Each item: the key it is answered under, what the batch sends for it, and its kind where not the batch's (a list's rows among breakdowns). */
+    items: Array<{
+        key: unknown;
+        sent: unknown;
+        kind?: FirstPaintItemKind;
+    }>;
+    /** A batch result as the client takes it for one item, or undefined for none. */
+    unwrap?: (result: unknown) => unknown;
+}
+/** What a room asks first: its reads, and its batches. */
+export interface RoomPlan {
+    reads: RoomRead[];
+    batches: RoomBatch[];
+}
+/** Ask one request as the page's reader, and carry its answer where it may be carried; the answer, or none. */
+type PaintAsk = (method: 'GET' | 'POST', target: string, body?: Record<string, unknown>) => Promise<unknown>;
+/**
+ * What a product's first page asks past who-am-I and the runtime: an App's
+ * public pages, its App and its first room (the App framework's
+ * firstPaintRooms.ts). Without one — the database browser — a page carries
+ * who-am-I and the runtime, and its rooms ask for the rest.
+ */
+export interface PaintRooms {
+    /** The reads of a page that is all its own reads (an App's public form, its landing), asked before who-am-I; none for any other. */
+    alone(path: string): readonly RoomRead[] | undefined;
+    /**
+     * Once who-am-I and the runtime have answered: what the shell asks next,
+     * through `ask`, and then the App the page is for and its room's plan, or
+     * none where the page draws no room of an App.
+     */
+    room(page: {
+        path: string;
+        search: string;
+        /** The page's first segment, decoded: the App its address names, where it names one. */
+        requested: string | null;
+        me: Record<string, unknown>;
+        runtime: Record<string, unknown>;
+        ask: PaintAsk;
+    }): Promise<{
+        app: string;
+        plan: RoomPlan;
+    } | undefined>;
+}
+/**
+ * The answers for the page at `path` (its inside form, shared/mount.ts)
+ * with `search`: what boot.js asks, in its order, and then what the first
+ * room asks (`rooms`, an App's: firstPaintRooms.ts). Each round's requests
+ * are asked together.
+ */
+export declare function firstPaint({ path, search, loop: asked, most, budget, rooms, }: {
+    path: string;
+    search: string;
+    loop: Loop;
+    most?: number;
+    budget?: number;
+    /** What the product's page asks past who-am-I and the runtime (PaintRooms), or nothing more. */
+    rooms?: PaintRooms;
+}): Promise<FirstPaintPage>;
+/**
+ * The data block, as the page carries it: JSON with every `<`, `>` and `&`
+ * written as its escape, and the two line separators JavaScript once took
+ * for line ends, so no value — a row's text is anybody's — can close the
+ * script element, open a comment inside it, or be read as markup.
+ */
+export declare function firstPaintBlock(id: string, json: string): string;
+export {};