@rebasepro/server-postgres 0.24.0 → 0.24.1-canary.gcb918fc

package/dist/services/sql-script.d.ts

@@ -1,5 +1,125 @@
-import type { Pool, PoolClient } from "pg";
-import type { SqlScriptResult } from "@rebasepro/types";
+import { type Connection, type Pool, type PoolClient, type Submittable } from "pg";
+import type { SqlCursor, SqlScriptResult } from "@rebasepro/types";
+/**
+ * Running a SQL script a person wrote — the Studio console — and saying what
+ * came back, column by column.
+ *
+ * The console used to receive bare rows and guess, from the query's text, which
+ * table a cell belonged to and which column held its key. The guess wrote
+ * `UPDATE authors SET name = … WHERE id = <the post's id>` for a cell of
+ * `SELECT p.id, a.name FROM posts p JOIN authors a …`, and `UPDATE users SET
+ * name = …` for `lower(email) AS name`. The database knows the answer: every
+ * column of a result's row description carries the table and column it was
+ * read from, or zero when it was computed. This reads it off the wire.
+ *
+ * Values come back as the text the database wrote, never parsed. Parsed, a
+ * `text[]` read as a JavaScript array and went back as JSON — a malformed array
+ * literal; a `bytea` as a serialized Buffer; a `timestamp` as a Date shifted by
+ * the server's timezone; a `numeric` as a float. Text is what the database
+ * reads back exactly.
+ */
+/** The parts of a `RowDescription` field this reads. */
+interface WireField {
+    name: string;
+    /** The OID of the table the column was read from; 0 when it was computed. */
+    tableID: number;
+    /** Its attribute number in that table; 0 when it was computed. */
+    columnID: number;
+    dataTypeID: number;
+}
+/** One statement of the script, as it came back. */
+export interface StatementOutcome {
+    fields: WireField[];
+    rows: Record<string, string | null>[];
+    /** How many rows the database sent for it, kept or not. */
+    received: number;
+    /** Rows past the {@link SqlResultLimits} were counted and not kept. */
+    truncated: boolean;
+    /** The command tag, e.g. `SELECT 3`, `INSERT 0 1`, `CREATE TABLE`. */
+    tag?: string;
+}
+/** How much of one statement's result a run keeps. */
+export interface SqlResultLimits {
+    /** Rows kept; the rest are counted and let go. */
+    maxRows: number;
+    /** Characters of values kept, summed over the kept rows. */
+    maxTextLength: number;
+}
+/**
+ * What the Studio console keeps of a result: its first 10,000 rows, or its
+ * first 32 Mi characters of values, whichever ends first.
+ *
+ * Every row used to be kept. `TABLE events` on a million rows held 418 MB of
+ * row objects and built a 475 MB frame, and a server with 1 GiB died of it —
+ * every other user's connection with it. Ten times the console's own
+ * `LIMIT 1000`, so a run that asked for a page always gets it whole; the text
+ * budget is for the rows a count does not bound — a document column, a
+ * `jsonb` blob — and keeps the frame, the string it is built from and the
+ * socket's copy of it around 100 MB at worst.
+ *
+ * Postgres still sends every row (a simple query cannot be stopped short of
+ * cancelling it, which would abort the statements after it); they are counted,
+ * so the result says how many there were.
+ */
+export declare const SQL_CONSOLE_RESULT_LIMITS: SqlResultLimits;
+/**
+ * The script, sent as one simple query and read message by message.
+ *
+ * A `Submittable` rather than `client.query(text)`: node-postgres parses every
+ * value through its type parsers and drops each field's origin, and for a
+ * multi-statement script returns an array the driver's callers took for a
+ * result with no rows. The message handlers below are the ones node-postgres
+ * calls on the active query — the same protocol `pg-cursor` implements.
+ */
+export declare class ScriptRun implements Submittable {
+    private readonly text;
+    private readonly limits;
+    /**
+     * Set by node-postgres when the pool has a `query_timeout`: a wrapper that
+     * clears its read timer. Called on every outcome, so the timer never
+     * outlives the run.
+     */
+    callback?: (error: Error | null) => void;
+    /**
+     * Every statement of the script, as it came back. Only the last holds its
+     * rows: the console shows the last, and a statement's rows are let go when
+     * the next one begins — `SELECT * FROM big; SELECT 1` held every row of
+     * `big` to the end, to show one.
+     */
+    readonly statements: StatementOutcome[];
+    readonly done: Promise<StatementOutcome[]>;
+    /** The statement whose row description arrived and whose command has not completed. */
+    private open?;
+    /** What {@link open} has kept of its rows. */
+    private budget;
+    private settled;
+    private resolveRun;
+    private rejectRun;
+    constructor(text: string, limits?: SqlResultLimits);
+    submit(connection: Connection): void;
+    handleRowDescription(message: {
+        fields: WireField[];
+    }): void;
+    handleDataRow(message: {
+        fields: (string | null)[];
+    }): void;
+    handleCommandComplete(message: {
+        text: string;
+    }): void;
+    handleEmptyQuery(): void;
+    handlePortalSuspended(): void;
+    /**
+     * `COPY … FROM STDIN` waits for data the console has no way to send. Fail
+     * the copy rather than leave the session waiting for it.
+     */
+    handleCopyInResponse(connection: Connection): void;
+    handleCopyData(): void;
+    handleError(error: Error): void;
+    handleReadyForQuery(): void;
+    /** Let go of the rows of the statement before the one beginning: only the last is shown. */
+    private releaseRows;
+    private settle;
+}
 /**
  * `INSERT 0 1` → `INSERT`, 1. `CREATE TABLE` → `CREATE TABLE`, no count.
  *
@@ -44,6 +164,17 @@ export interface OwnSessionOptions {
      * it ran as the connection owner.
      */
     assumeRole?: (connection: PoolClient) => Promise<void>;
+    /**
+     * Cancels the run when aborted: the statement in progress is stopped with
+     * `pg_cancel_backend`, and the run fails with the database's "canceling
+     * statement due to user request". A signal already aborted runs nothing.
+     */
+    signal?: AbortSignal;
+}
+/** A run whose signal was aborted before it began. */
+export declare class SqlRunCancelledError extends Error {
+    readonly code = "SQL_RUN_CANCELLED";
+    constructor();
 }
 /**
  * Run `work` — SQL a person wrote — on a connection of its own from `pool`,
@@ -68,4 +199,77 @@ export declare function runSqlScriptOnPool(pool: Pool, sql: string, options?: Ow
  * pool (PGlite in process): every value as text, and no column with a source —
  * nothing to say where a value came from, so nothing the console will edit.
  */
-export declare function sqlScriptResultFromRows(rows: Record<string, unknown>[]): SqlScriptResult;
+export declare function sqlScriptResultFromRows(rows: Record<string, unknown>[], limits?: SqlResultLimits): SqlScriptResult;
+/**
+ * How many bytes of rows one answer of the admin `executeSql` door — the
+ * WebSocket `EXECUTE_SQL` without `mode: "script"` — may carry: 128 MiB.
+ *
+ * That door had no limit at all. A million-row `SELECT` held every row parsed
+ * (432 MB of heap on the console's door before it was capped), then built the
+ * frame from them, and a server with 1 GiB died of it — with every other
+ * user's connection. Unlike the console's script door it does not keep a
+ * prefix: a program reading rows through it would take a prefix for the whole
+ * answer, so past the budget the answer is refused, {@link SqlResultTooLargeError}.
+ *
+ * Generous on purpose. The catalogue reads the admin UI makes through this door
+ * are far below it — the SQL console's schema browser reads one
+ * `information_schema.columns` row per column, 7 MB for the 100,000 columns of
+ * 5,000 tables — and anything near it is a result no browser tab should hold.
+ */
+export declare const SQL_RESULT_BYTE_BUDGET: number;
+/**
+ * A result larger than the bytes its caller allowed, refused rather than
+ * returned in part. Its message is for the person who wrote the SQL.
+ */
+export declare class SqlResultTooLargeError extends Error {
+    readonly limitBytes: number;
+    readonly code = "RESULT_TOO_LARGE";
+    constructor(limitBytes: number);
+}
+/**
+ * Refuse rows that came back whole and are past `maxBytes`.
+ *
+ * Where nothing could watch the read, it is measured once done — and the
+ * answer still refused rather than sent.
+ */
+export declare function refuseOversizedRows<T extends Record<string, unknown>>(rows: T[], maxBytes: number): T[];
+/**
+ * Run `work` — one query on `connection` — and stop reading once its rows pass
+ * `maxBytes`.
+ *
+ * node-postgres keeps every row of a query until the query completes, and a
+ * simple query cannot be stopped short of its end, so the only way to stop
+ * holding rows is to stop receiving them: the socket is closed. Postgres sees
+ * its client gone and abandons the statement; the session — and anything it
+ * held — goes with it, and `onOwnSession` destroys the connection rather than
+ * return it to the pool. What the caller gets is {@link SqlResultTooLargeError},
+ * never the rows read so far.
+ *
+ * Counted from the wire, message by message, as they arrive: what is held when
+ * it stops is the budget plus one network read.
+ */
+export declare function readWithinBudget<T extends Record<string, unknown>>(connection: PoolClient, maxBytes: number, work: () => Promise<T[]>): Promise<T[]>;
+/** Rows one read of a {@link SqlCursor} fetches. */
+export declare const SQL_CURSOR_BATCH_ROWS = 1000;
+/**
+ * Open a cursor over ONE query a person wrote — the SQL console's "Download
+ * full result as CSV".
+ *
+ * On a connection of its own, in the role asked for, inside
+ * `BEGIN TRANSACTION READ ONLY`: the database refuses a write, a
+ * data-modifying `WITH` and a function that writes. The query is declared over
+ * the extended protocol, which refuses a second statement — `SELECT 1; DROP
+ * TABLE t` cannot ride along. Rows are then fetched {@link SQL_CURSOR_BATCH_ROWS}
+ * at a time, each `FETCH` under the session's `statement_timeout`; nothing
+ * fetches ahead of the reader, so memory holds one batch. The first batch is
+ * read here, so a query that fails on its first rows fails before anything has
+ * been sent.
+ *
+ * The caller closes the cursor — always. A reader that stops reading without
+ * closing is ended by `idle_in_transaction_session_timeout` after
+ * {@link SQL_CURSOR_IDLE_TIMEOUT_MS}.
+ */
+export declare function openSqlCursorOnPool(pool: Pool, sql: string, options?: OwnSessionOptions & {
+    batchRows?: number;
+}): Promise<SqlCursor>;
+export {};

package/dist/services/vector-iterative-scan.d.ts

@@ -0,0 +1,40 @@
+import type { DrizzleClient } from "../interfaces.js";
+/**
+ * pgvector's iterative index scans, for a vector read that also filters.
+ *
+ * An HNSW index answers a nearest-neighbour query with the `hnsw.ef_search`
+ * (40 by default) candidates it found, and every other condition of the query
+ * — a `where`, a search, a soft-delete stamp, a row-level-security policy — is
+ * applied to those candidates afterwards. Where few of them pass, the page
+ * comes back short, or empty, though plenty of rows match: measured on 20,000
+ * rows with half of them matching a `status` filter far from the query vector,
+ * `LIMIT 10` served 0 rows. Offset pages over the index were empty or short the
+ * same way, while deeper pages, where the planner switched to an exact scan,
+ * served rows, so a walk skipped some and could repeat others.
+ *
+ * From pgvector 0.8.0, `hnsw.iterative_scan` makes the index keep scanning
+ * until the query has its rows (bounded by `hnsw.max_scan_tuples`, 20,000 by
+ * default). `strict_order` and not `relaxed_order`: the planner reads the
+ * index's output as already sorted by distance, so out-of-order rows would be
+ * cut at the wrong place by `LIMIT` and the same row could land on two pages.
+ * Measured on the case above: 0/10 → 10/10, and the first 200 rows by offset
+ * pages of 10 with no duplicates (`relaxed_order`: 18).
+ *
+ * Set with `set_config(…, true)` — local to the transaction the read runs in,
+ * so it never outlives it on a pooled connection. A read not in a transaction
+ * is given one.
+ *
+ * pgvector before 0.8.0 has no such setting, and with its library loaded it
+ * refuses an unknown `hnsw.` parameter, so the setting is applied only where
+ * the extension's version has it; there a filtered vector read keeps the
+ * behaviour above. IVFFlat indexes are not touched: their iterative scan only
+ * comes in relaxed order.
+ */
+/** Whether a pgvector version (`extversion`, e.g. `"0.8.1"`) has iterative index scans. */
+export declare function hasIterativeIndexScans(version: string | undefined | null): boolean;
+/** Whether `db`'s pgvector has iterative index scans. Asked once per `key`. */
+export declare function iterativeScansAvailable(db: DrizzleClient, key: object): Promise<boolean>;
+/** Whether `db` is a transaction, in which a `set_config(…, true)` stays. */
+export declare function inTransaction(db: DrizzleClient): boolean;
+/** Turn on strict-order iterative HNSW scans for the rest of `db`'s transaction. */
+export declare function enableIterativeScan(db: DrizzleClient): Promise<void>;

package/dist/services/write-depth.d.ts

@@ -14,3 +14,9 @@ export interface WriteFrame {
 }
 /** Run one write as a frame, refusing it when the chain is already too deep. */
 export declare function inWriteFrame<T>(path: string, operation: "save" | "delete", fn: (frame: WriteFrame) => Promise<T>): Promise<T>;
+/**
+ * How deep in hook-started writes the code running now is: 0 outside any
+ * write, 1 inside a write a caller started, 2 or more inside a write a hook of
+ * another write started (`context.data`).
+ */
+export declare function currentWriteDepth(): number;

package/dist/services/write-transaction-scope.d.ts

@@ -28,6 +28,8 @@ export declare class WriteTransactionScope implements AmbientTransaction {
     /** The first tracked statement that failed, whether or not its caller caught it. */
     private failure?;
     private readonly settledHooks;
+    /** Whether application code (a hook) has run on this transaction. See {@link noteApplicationCode}. */
+    private applicationCodeRan;
     private settledContext?;
     /** The transaction handle, for statements in this package that are already written against drizzle. */
     tx?: DrizzleClient;
@@ -74,6 +76,13 @@ export declare class WriteTransactionScope implements AmbientTransaction {
      * was safe.)
      */
     assertCommittable(): Promise<void>;
+    /**
+     * Application code — a collection, global or property hook, a
+     * `beforeQuery` — is about to run on this transaction, so it may catch a
+     * failed statement and leave the transaction aborted: the commit then has
+     * to ask ({@link assertCommittable}).
+     */
+    noteApplicationCode(): void;
     /**
      * What a hook deferred with {@link afterSettled} is handed as `context`:
      * the same caller, on a driver where every call is a transaction of its

package/dist/socket-opt-ins.d.ts

@@ -0,0 +1,64 @@
+import { WebSocket } from "ws";
+/**
+ * Behaviours a client opts into when it opens the socket.
+ *
+ * The client names them on the socket URL, `?rebase_caps=heartbeat,channel_seq,cancel_sql`,
+ * and the server answers with a `hello` frame listing the ones it will honour:
+ *
+ * ```json
+ * { "type": "hello", "capabilities": ["heartbeat", "channel_seq", "cancel_sql"], "heartbeatMs": 30000, "clientId": "client_…" }
+ * ```
+ *
+ * Both halves are needed, so both directions of a version mismatch keep
+ * working. An older client names none and gets no `hello`, no heartbeat and no
+ * acknowledgement frames — nothing it would not recognise. An older server
+ * ignores the query parameter and sends no `hello`, and a newer client that
+ * hears none behaves as it always did.
+ *
+ * One list for both kinds: what the server sends (`heartbeat`, `channel_seq`)
+ * and the frames it takes beyond the base protocol (`cancel_sql`). Named before
+ * sign-in, so a socket that never signs in is served as one that does.
+ *
+ * - `heartbeat`: a `{ "type": "heartbeat" }` frame every `heartbeatMs`. A
+ *   browser cannot send a WebSocket ping or see one, so a connection that went
+ *   half-open — a NAT that forgot it, a laptop that slept — looks connected to
+ *   it forever. With these, two missed beats tell it to redial.
+ * - `channel_seq`: on a retained channel, the sender of a broadcast gets
+ *   `{ "type": "broadcast_ack", "channel", "seq" }` where every other member
+ *   gets the message. Its own messages then leave no hole in the sequence it
+ *   watches, so a hole means a message it really missed — see
+ *   `RealtimeChannel` in the SDK.
+ * - `cancel_sql`: the server takes `CANCEL_SQL { requestId }`, which stops a
+ *   console run (`EXECUTE_SQL`, `mode: "script"`) of the same socket. A client
+ *   sends it only to a server whose `hello` listed it.
+ *
+ * @module
+ */
+/** The query parameter a client names its opt-ins in. */
+export declare const SOCKET_OPT_IN_PARAM = "rebase_caps";
+/** Every behaviour this server can be asked for. */
+export declare const SOCKET_OPT_INS: readonly ["heartbeat", "channel_seq", "cancel_sql"];
+export type SocketOptIn = typeof SOCKET_OPT_INS[number];
+/** How often a socket that asked for heartbeats is sent one, by default. */
+export declare const DEFAULT_HEARTBEAT_INTERVAL_MS = 30000;
+/**
+ * What the client that opened this connection asked for, from its request
+ * URL: `undefined` when it named nothing (a client that predates this, which
+ * is sent no `hello`), otherwise the names this server honours — possibly
+ * none.
+ */
+export declare function requestedOptIns(url: string | undefined): Set<SocketOptIn> | undefined;
+/**
+ * The `hello` frame for a client that asked for `optIns`. With `channel_seq`
+ * it also names the socket's own id, the `senderId` its broadcasts carry in a
+ * channel's history, so a client filling a hole from history can tell its own
+ * messages (which it is never sent live) from the ones it missed.
+ */
+export declare function helloFrame(optIns: ReadonlySet<SocketOptIn>, heartbeatMs: number, clientId: string): string;
+/**
+ * Send a heartbeat every `intervalMs` to each socket `sockets()` lists. One
+ * timer for the whole server, unref'd so it never holds the process open.
+ *
+ * @returns Stops it.
+ */
+export declare function startHeartbeats(sockets: () => Iterable<WebSocket>, intervalMs: number): () => void;