@mason-data/cli 0.0.0-stage → 0.6.2

package/dist/src/cdc/pg-wire.js

@@ -0,0 +1,482 @@
+/**
+ * A POSTGRES CLIENT, DEPENDENCY-FREE, for the change reader: exactly what Mason needs from the
+ * wire protocol and nothing else — startup (TLS per libpq's sslmode), password / md5 / SCRAM-SHA-256 auth,
+ * the simple query protocol, and the streaming replication sub-protocol (CopyBoth, XLogData, keepalives,
+ * standby status updates). Every value travels as TEXT: the reader turns text into ClickHouse values itself,
+ * so no binary type decoding lives here.
+ *
+ * Why not `pg`: the scripts here run with no node_modules on the test server, the replication sub-protocol is a
+ * separate package on top of it, and the part Mason uses is a few hundred lines this file can hold whole.
+ *
+ * Protocol reference (Postgres's own documentation): https://www.postgresql.org/docs/current/protocol-message-formats.html and
+ * https://www.postgresql.org/docs/current/protocol-replication.html.
+ */
+import * as crypto from "node:crypto";
+import * as net from "node:net";
+import * as tls from "node:tls";
+/** A server ErrorResponse, with its SQLSTATE: callers branch on `code` (42P01, 42710, 55006 …). */
+export class PgError extends Error {
+    code;
+    detail;
+    constructor(fields) {
+        super(`${fields.S ?? "ERROR"}: ${fields.M ?? "unknown error"}${fields.D ? ` (${fields.D})` : ""}`);
+        this.code = fields.C ?? "";
+        this.detail = fields.D;
+    }
+}
+/** Microseconds between the Unix epoch and Postgres's (2000-01-01). */
+const PG_EPOCH_OFFSET_US = 946684800000000n;
+export class PgConnection {
+    socket;
+    buf = Buffer.alloc(0);
+    /** chunks received after `buf` while the message they complete is still short of `need` bytes (onData) */
+    chunks = [];
+    chunked = 0;
+    waiting = null;
+    queue = [];
+    /** the bytes of the messages in `queue`: what backpressure bounds (onData) */
+    queued = 0;
+    closedWith = null;
+    /** Server parameters (server_version, TimeZone, …) as reported at startup. */
+    parameters = {};
+    opts;
+    constructor(opts) { this.opts = opts; }
+    static async connect(opts) {
+        const c = new PgConnection(opts);
+        await c.open();
+        return c;
+    }
+    // ------------------------------------------------------------------ transport
+    /**
+     * THE CONNECT TIMEOUT BOUNDS THE WHOLE HANDSHAKE: it once covered the TCP connect alone, and the SSL
+     * answer and the startup exchange waited for ever. A socket something accepts and never answers — a port
+     * proxy in front of a container that is restarting, a half-open connection — hung the reader's reconnect loop
+     * past the test's 90 s with no "the source is away" line to say so (the ClickHouse stack's change-reader fill test,
+     * twice in CI). One deadline from the first SYN to ReadyForQuery; past it the socket is destroyed and the error reads as
+     * the source being away (reader.ts `sourceAway`), so the reconnect loop tries again.
+     */
+    async open() {
+        const o = this.opts;
+        const deadline = o.connectTimeoutMs ?? 15_000;
+        let pending = null;
+        let timer;
+        const timedOut = new Promise((_, reject) => {
+            timer = setTimeout(() => { pending?.destroy(); reject(new Error(`connecting to ${o.host}:${o.port} timed out`)); }, deadline);
+        });
+        try {
+            await Promise.race([this.handshake((s) => { pending = s; }), timedOut]);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    /** TCP connect, the SSL negotiation and the startup exchange; `hold` is told the live socket, for a timeout to destroy. */
+    async handshake(hold) {
+        const o = this.opts;
+        const mode = o.sslmode ?? "prefer";
+        const raw = await new Promise((resolve, reject) => {
+            const s = net.connect({ host: o.host, port: o.port });
+            hold(s);
+            s.once("connect", () => resolve(s));
+            s.once("error", (e) => reject(new Error(`connecting to ${o.host}:${o.port}: ${socketErrorText(e)}`)));
+        });
+        let sock = raw;
+        if (mode !== "disable" && mode !== "allow") {
+            // SSLRequest: length 8, code 80877103; the server answers one byte, S or N
+            raw.write(Buffer.from([0, 0, 0, 8, 0x04, 0xd2, 0x16, 0x2f]));
+            const answer = await new Promise((resolve, reject) => {
+                raw.once("data", (d) => resolve(String.fromCharCode(d[0])));
+                raw.once("error", reject);
+            });
+            if (answer === "S") {
+                const verify = mode === "verify-ca" || mode === "verify-full";
+                sock = await new Promise((resolve, reject) => {
+                    const t = tls.connect({
+                        socket: raw, rejectUnauthorized: verify,
+                        ...(mode === "verify-full" ? { servername: o.host } : { checkServerIdentity: () => undefined }),
+                    });
+                    hold(t);
+                    t.once("secureConnect", () => resolve(t));
+                    t.once("error", reject);
+                });
+            }
+            else if (mode !== "prefer") {
+                raw.destroy();
+                throw new Error(`sslmode=${mode}, but ${o.host} does not accept TLS`);
+            }
+        }
+        this.socket = sock;
+        sock.on("data", (d) => this.onData(d));
+        sock.on("error", (e) => this.fail(e));
+        sock.on("close", () => this.fail(new Error("connection closed by the server")));
+        await this.startup();
+    }
+    fail(e) {
+        if (this.closedWith)
+            return;
+        this.closedWith = e;
+        const w = this.waiting;
+        this.waiting = null;
+        if (w)
+            w(e);
+    }
+    onData(d) {
+        // A LARGE MESSAGE ARRIVES IN MANY CHUNKS, and joining each chunk to what came before copies it over and over:
+        // quadratic in the message, all of it off the V8 heap, where no heap limit makes the collector hurry. A row with
+        // a multi-megabyte payload took the reader past its container's 512 MiB (a cgroup OOM kill, the test server's first
+        // copy). So chunks wait in a list until the message they belong to is complete, and join once.
+        this.chunks.push(d);
+        this.chunked += d.length;
+        const have = this.buf.length + this.chunked;
+        const need = this.buf.length >= 5 ? 1 + this.buf.readInt32BE(1) : 5;
+        if (have < need)
+            return;
+        this.buf = Buffer.concat([this.buf, ...this.chunks], have);
+        this.chunks = [];
+        this.chunked = 0;
+        while (this.buf.length >= 5) {
+            const len = this.buf.readInt32BE(1);
+            if (this.buf.length < 1 + len)
+                break;
+            const msg = { type: String.fromCharCode(this.buf[0]), body: this.buf.subarray(5, 1 + len) };
+            this.buf = this.buf.subarray(1 + len);
+            if (msg.type === "N")
+                continue; // NoticeResponse: informational
+            if (msg.type === "S") {
+                this.parameter(msg.body);
+                continue;
+            }
+            const w = this.waiting;
+            if (w) {
+                this.waiting = null;
+                w(msg);
+            }
+            else {
+                this.queue.push(msg);
+                this.queued += msg.body.length;
+            }
+        }
+        // Backpressure: a consumer that is not reading (the reader flushing into ClickHouse) must not let the
+        // stream pile up in memory — counted in messages AND bytes, since one row may carry megabytes. The queue
+        // resumes the socket when it drains.
+        if (this.queue.length > 1000 || this.queued > QUEUE_PAUSE_BYTES)
+            this.socket.pause();
+    }
+    parameter(body) {
+        const [k, v] = cstrings(body, 2);
+        if (k !== undefined)
+            this.parameters[k] = v ?? "";
+    }
+    next() {
+        const m = this.queue.shift();
+        if (m) {
+            this.queued -= m.body.length;
+            if (this.queue.length < 100 && this.queued < QUEUE_PAUSE_BYTES / 4 && this.socket.isPaused())
+                this.socket.resume();
+            return Promise.resolve(m);
+        }
+        if (this.closedWith)
+            return Promise.reject(this.closedWith);
+        return new Promise((resolve, reject) => {
+            this.waiting = (x) => x instanceof Error ? reject(x) : resolve(x);
+        });
+    }
+    send(type, body) {
+        if (this.closedWith)
+            return; // a status update after the server went away has nowhere to go
+        const head = Buffer.alloc(5);
+        head.write(type, 0, "latin1");
+        head.writeInt32BE(body.length + 4, 1);
+        this.socket.write(Buffer.concat([head, body]));
+    }
+    // ------------------------------------------------------------------ startup and auth
+    async startup() {
+        const o = this.opts;
+        const params = [
+            ["user", o.user], ["database", o.database], ["client_encoding", "UTF8"],
+            // Text values in the forms the reader parses: ISO dates in UTC, floats round-trippable
+            ["DateStyle", "ISO, YMD"], ["TimeZone", "UTC"], ["extra_float_digits", "3"], ["IntervalStyle", "iso_8601"],
+            ["application_name", o.applicationName ?? "mason"],
+        ];
+        if (o.replication)
+            params.push(["replication", "database"]);
+        const body = Buffer.concat([
+            int32(196608), // protocol 3.0
+            ...params.map(([k, v]) => Buffer.concat([cstr(k), cstr(v)])),
+            Buffer.from([0]),
+        ]);
+        const len = Buffer.alloc(4);
+        len.writeInt32BE(body.length + 4);
+        this.socket.write(Buffer.concat([len, body]));
+        for (;;) {
+            const m = await this.next();
+            if (m.type === "E")
+                throw new PgError(errorFields(m.body));
+            if (m.type === "R") {
+                await this.auth(m.body);
+                continue;
+            }
+            if (m.type === "K")
+                continue; // BackendKeyData
+            if (m.type === "Z")
+                return;
+            throw new Error(`unexpected message ${m.type} during startup`);
+        }
+    }
+    scram = null;
+    async auth(body) {
+        const code = body.readInt32BE(0);
+        const pw = this.opts.password;
+        switch (code) {
+            case 0: return; // AuthenticationOk
+            case 3:
+                this.send("p", cstr(pw));
+                return;
+            case 5: {
+                const salt = body.subarray(4, 8);
+                const inner = md5hex(Buffer.concat([Buffer.from(pw), Buffer.from(this.opts.user)]));
+                this.send("p", cstr("md5" + md5hex(Buffer.concat([Buffer.from(inner), salt]))));
+                return;
+            }
+            case 10: {
+                const mechanisms = cstrings(body.subarray(4));
+                if (!mechanisms.includes("SCRAM-SHA-256"))
+                    throw new Error(`the server offers ${mechanisms.join(", ")}; only SCRAM-SHA-256 is supported`);
+                const nonce = crypto.randomBytes(18).toString("base64");
+                const clientFirstBare = `n=,r=${nonce}`;
+                this.scram = { nonce, clientFirstBare };
+                const first = Buffer.from(`n,,${clientFirstBare}`);
+                this.send("p", Buffer.concat([cstr("SCRAM-SHA-256"), int32(first.length), first]));
+                return;
+            }
+            case 11: { // SASLContinue
+                const s = this.scram;
+                if (!s)
+                    throw new Error("SASLContinue without a SASL exchange");
+                const serverFirst = body.subarray(4).toString();
+                const attrs = Object.fromEntries(serverFirst.split(",").map((kv) => [kv[0], kv.slice(2)]));
+                const r = attrs.r ?? "", salt = Buffer.from(attrs.s ?? "", "base64"), iterations = Number(attrs.i);
+                if (!r.startsWith(s.nonce))
+                    throw new Error("SCRAM: the server's nonce does not extend ours");
+                const salted = crypto.pbkdf2Sync(pw.normalize("NFKC"), salt, iterations, 32, "sha256");
+                const clientKey = hmac(salted, "Client Key");
+                const storedKey = crypto.createHash("sha256").update(clientKey).digest();
+                const clientFinalNoProof = `c=biws,r=${r}`;
+                const authMessage = `${s.clientFirstBare},${serverFirst},${clientFinalNoProof}`;
+                const signature = hmac(storedKey, authMessage);
+                const proof = Buffer.alloc(clientKey.length);
+                for (let i = 0; i < proof.length; i++)
+                    proof[i] = clientKey[i] ^ signature[i];
+                s.serverSignature = hmac(hmac(salted, "Server Key"), authMessage);
+                this.send("p", Buffer.from(`${clientFinalNoProof},p=${proof.toString("base64")}`));
+                return;
+            }
+            case 12: { // SASLFinal: the server proves it knew the password too
+                const v = /v=([^,]+)/.exec(body.subarray(4).toString())?.[1];
+                if (!this.scram?.serverSignature || v !== this.scram.serverSignature.toString("base64")) {
+                    throw new Error("SCRAM: the server's signature does not verify");
+                }
+                return;
+            }
+            default: throw new Error(`authentication method ${code} is not supported (use password, md5 or scram-sha-256)`);
+        }
+    }
+    // ------------------------------------------------------------------ queries
+    /** One statement (or several `;`-separated) through the simple query protocol; the LAST result set. */
+    async query(sql) {
+        this.send("Q", cstr(sql));
+        let result = { fields: [], rows: [], command: "" };
+        let error = null;
+        for (;;) {
+            const m = await this.next();
+            switch (m.type) {
+                case "T":
+                    result = { fields: rowDescription(m.body), rows: [], command: "" };
+                    break;
+                case "D":
+                    result.rows.push(dataRow(m.body));
+                    break;
+                case "C":
+                    result.command = cstrings(m.body)[0] ?? "";
+                    break;
+                case "E":
+                    error = new PgError(errorFields(m.body));
+                    break;
+                case "I": break; // EmptyQueryResponse
+                case "Z":
+                    if (error)
+                        throw error;
+                    return result;
+                default: break;
+            }
+        }
+    }
+    /**
+     * One statement's rows AS THEY ARRIVE, never held together: a page of a copy may carry hundreds of megabytes of
+     * payloads (68 MB in 64 heap blocks of a table of wide JSON payloads), which `query` would hold whole. The socket
+     * pauses while the consumer is busy (onData's backpressure), so memory is bounded by the queue, not the result.
+     * A consumer that stops early must drain it: the connection is unusable until the server's ReadyForQuery.
+     */
+    async *rows(sql) {
+        this.send("Q", cstr(sql));
+        let error = null;
+        for (;;) {
+            const m = await this.next();
+            switch (m.type) {
+                case "D":
+                    if (error === null)
+                        yield dataRow(m.body);
+                    break;
+                case "E":
+                    error = new PgError(errorFields(m.body));
+                    break;
+                case "Z":
+                    if (error)
+                        throw error;
+                    return;
+                default: break;
+            }
+        }
+    }
+    // ------------------------------------------------------------------ replication
+    /** START_REPLICATION, then an async stream of XLogData and keepalives until the connection ends. */
+    async *replicate(command) {
+        this.send("Q", cstr(command));
+        for (;;) {
+            const m = await this.next();
+            if (m.type === "E")
+                throw new PgError(errorFields(m.body));
+            if (m.type === "W")
+                break; // CopyBothResponse
+        }
+        for (;;) {
+            const m = await this.next();
+            if (m.type === "E")
+                throw new PgError(errorFields(m.body));
+            if (m.type === "c" || m.type === "C" || m.type === "Z")
+                return; // the server ended the stream
+            if (m.type !== "d")
+                continue;
+            const b = m.body;
+            if (b[0] === 0x77) { // 'w' XLogData
+                yield { kind: "data", start: b.readBigUInt64BE(1), walEnd: b.readBigUInt64BE(9), data: b.subarray(25) };
+            }
+            else if (b[0] === 0x6b) { // 'k' primary keepalive
+                yield { kind: "keepalive", walEnd: b.readBigUInt64BE(1), replyRequested: b[17] === 1 };
+            }
+        }
+    }
+    /**
+     * A standby status update: everything up to `flushed` is durable on our side, so the server may discard
+     * the WAL behind it. The ONLY place the reader tells Postgres a change is done.
+     */
+    sendStatus(flushed, replyRequested = false) {
+        const b = Buffer.alloc(34);
+        b[0] = 0x72; // 'r'
+        b.writeBigUInt64BE(flushed, 1); // written
+        b.writeBigUInt64BE(flushed, 9); // flushed
+        b.writeBigUInt64BE(flushed, 17); // applied
+        b.writeBigUInt64BE(BigInt(Date.now()) * 1000n - PG_EPOCH_OFFSET_US, 25);
+        b[33] = replyRequested ? 1 : 0;
+        this.send("d", b);
+    }
+    close() {
+        if (this.closedWith)
+            return;
+        if (this.socket.writable)
+            this.send("X", Buffer.alloc(0)); // Terminate: the server ends the backend cleanly
+        // a read waiting for the server's next message (a replication stream between changes) ends with it, never waits on
+        // a socket nothing reads any more (stopping a DuckDB stream once hung on its idle START_REPLICATION)
+        this.fail(new Error("connection closed"));
+        this.socket.end();
+    }
+}
+// ------------------------------------------------------------------ helpers
+/** `16/B374D848` -> its 64-bit position; throws on anything else. */
+export function parseLsn(text) {
+    const m = /^([0-9A-Fa-f]{1,8})\/([0-9A-Fa-f]{1,8})$/.exec(text.trim());
+    if (!m)
+        throw new Error(`not a WAL position: ${JSON.stringify(text)}`);
+    return (BigInt(`0x${m[1]}`) << 32n) | BigInt(`0x${m[2]}`);
+}
+export function formatLsn(lsn) {
+    return `${(lsn >> 32n).toString(16).toUpperCase()}/${(lsn & 0xffffffffn).toString(16).toUpperCase()}`;
+}
+/** A SQL string literal (standard_conforming_strings on, the default since 9.1). */
+export function pgLiteral(value) {
+    if (value.includes("\0"))
+        throw new Error("a NUL byte cannot appear in a Postgres string");
+    return `'${value.replace(/'/g, "''")}'`;
+}
+/** A quoted identifier. */
+export function pgIdent(name) {
+    return `"${name.replace(/"/g, '""')}"`;
+}
+function int32(n) { const b = Buffer.alloc(4); b.writeInt32BE(n); return b; }
+function cstr(s) { return Buffer.concat([Buffer.from(s, "utf8"), Buffer.from([0])]); }
+function md5hex(b) { return crypto.createHash("md5").update(b).digest("hex"); }
+function hmac(key, data) { return crypto.createHmac("sha256", key).update(data).digest(); }
+function cstrings(b, limit = Infinity) {
+    const out = [];
+    let i = 0;
+    while (i < b.length && out.length < limit) {
+        const end = b.indexOf(0, i);
+        if (end < 0)
+            break;
+        if (end === i && out.length > 0 && limit === Infinity)
+            break; // the list terminator
+        out.push(b.toString("utf8", i, end));
+        i = end + 1;
+    }
+    return out;
+}
+function errorFields(b) {
+    const f = {};
+    let i = 0;
+    while (i < b.length && b[i] !== 0) {
+        const code = String.fromCharCode(b[i]);
+        const end = b.indexOf(0, i + 1);
+        f[code] = b.toString("utf8", i + 1, end);
+        i = end + 1;
+    }
+    return f;
+}
+function rowDescription(b) {
+    const n = b.readInt16BE(0);
+    const names = [];
+    let i = 2;
+    for (let k = 0; k < n; k++) {
+        const end = b.indexOf(0, i);
+        names.push(b.toString("utf8", i, end));
+        i = end + 1 + 18; // table oid, column no, type oid, size, modifier, format
+    }
+    return names;
+}
+/** onData pauses the socket above this many queued bytes, and next() resumes it below a quarter of it. */
+const QUEUE_PAUSE_BYTES = 16 * 1024 * 1024;
+function dataRow(b) {
+    const n = b.readInt16BE(0);
+    const out = [];
+    let i = 2;
+    for (let k = 0; k < n; k++) {
+        const len = b.readInt32BE(i);
+        i += 4;
+        if (len < 0) {
+            out.push(null);
+            continue;
+        }
+        out.push(b.toString("utf8", i, i + len));
+        i += len;
+    }
+    return out;
+}
+/**
+ * A socket's error as words: its message, or each address's where Node tried several (`localhost` is ::1 and 127.0.0.1:
+ * an AggregateError whose own message is EMPTY — `connecting to localhost:5432: ` said nothing, and `mason start` could
+ * not tell "nothing is listening" from a wrong URL), or its code.
+ */
+export function socketErrorText(e) {
+    if (e instanceof AggregateError && e.errors.length > 0)
+        return e.errors.map((x) => x instanceof Error ? socketErrorText(x) : String(x)).join("; ");
+    return e.message || e.code || e.name;
+}

package/dist/src/cdc/pgoutput.js

@@ -0,0 +1,107 @@
+/**
+ * THE `pgoutput` DECODER (protocol version 1): Postgres's built-in logical decoding output, the messages
+ * inside the replication stream's XLogData. Pure — a Buffer in, a message out — so every message shape is
+ * pinned by tests/cdc-pgoutput.test.ts against bytes built by hand from the protocol reference (Postgres's own documentation):
+ * https://www.postgresql.org/docs/current/protocol-logicalrep-message-formats.html
+ *
+ * Version 1 is enough: no streamed in-progress transactions (the server hands a transaction over only at
+ * its commit, which is what makes "acknowledge only complete transactions" simple), no two-phase commit.
+ */
+class Cursor {
+    b;
+    i = 0;
+    constructor(b) { this.b = b; }
+    byte() { return String.fromCharCode(this.b[this.i++]); }
+    int8() { return this.b.readInt8(this.i++); }
+    int16() { const v = this.b.readInt16BE(this.i); this.i += 2; return v; }
+    int32() { const v = this.b.readInt32BE(this.i); this.i += 4; return v; }
+    uint32() { const v = this.b.readUInt32BE(this.i); this.i += 4; return v; }
+    int64() { const v = this.b.readBigUInt64BE(this.i); this.i += 8; return v; }
+    cstr() {
+        const end = this.b.indexOf(0, this.i);
+        if (end < 0)
+            throw new Error("pgoutput: unterminated string");
+        const s = this.b.toString("utf8", this.i, end);
+        this.i = end + 1;
+        return s;
+    }
+    bytes(n) { const v = this.b.subarray(this.i, this.i + n); this.i += n; return v; }
+}
+function tuple(c) {
+    const n = c.int16();
+    const out = [];
+    for (let k = 0; k < n; k++) {
+        const kind = c.byte();
+        if (kind === "n")
+            out.push({ kind: "null" });
+        else if (kind === "u")
+            out.push({ kind: "unchanged" });
+        else if (kind === "t") {
+            const len = c.int32();
+            out.push({ kind: "text", text: c.bytes(len).toString("utf8") });
+        }
+        else if (kind === "b")
+            throw new Error("pgoutput: a binary column value arrived, but the reader asks for text");
+        else
+            throw new Error(`pgoutput: unknown tuple column kind ${JSON.stringify(kind)}`);
+    }
+    return out;
+}
+export function decode(data) {
+    const c = new Cursor(data);
+    const tag = c.byte();
+    switch (tag) {
+        case "B": return { type: "begin", finalLsn: c.int64(), commitTime: c.int64(), xid: c.int32() };
+        case "C": {
+            c.int8();
+            return { type: "commit", commitLsn: c.int64(), endLsn: c.int64(), commitTime: c.int64() };
+        }
+        case "R": {
+            const relid = c.uint32(), namespace = c.cstr(), name = c.cstr(), replicaIdentity = c.byte();
+            const n = c.int16();
+            const columns = [];
+            for (let k = 0; k < n; k++) {
+                const flags = c.int8();
+                columns.push({ isKey: (flags & 1) === 1, name: c.cstr(), typeOid: c.uint32(), typmod: c.int32() });
+            }
+            return { type: "relation", relid, namespace, name, replicaIdentity, columns };
+        }
+        case "I": {
+            const relid = c.uint32();
+            expect(c.byte(), "N", "insert");
+            return { type: "insert", relid, tuple: tuple(c) };
+        }
+        case "U": {
+            const relid = c.uint32();
+            let marker = c.byte();
+            let oldKey = null;
+            if (marker === "K" || marker === "O") {
+                oldKey = tuple(c);
+                marker = c.byte();
+            }
+            expect(marker, "N", "update");
+            return { type: "update", relid, oldKey, tuple: tuple(c) };
+        }
+        case "D": {
+            const relid = c.uint32();
+            const marker = c.byte();
+            if (marker !== "K" && marker !== "O")
+                throw new Error(`pgoutput: delete carries ${marker}, expected K or O`);
+            return { type: "delete", relid, key: tuple(c) };
+        }
+        case "T": {
+            const n = c.int32();
+            c.int8(); // options: CASCADE / RESTART IDENTITY
+            const relids = [];
+            for (let k = 0; k < n; k++)
+                relids.push(c.uint32());
+            return { type: "truncate", relids };
+        }
+        // Origin, Type and logical decoding messages carry nothing the reader stores
+        default: return { type: "other", tag };
+    }
+}
+function expect(got, want, what) {
+    if (got !== want)
+        throw new Error(`pgoutput: ${what} carries ${JSON.stringify(got)} where ${want} was expected`);
+}

package/dist/src/cdc/poll.js

@@ -0,0 +1,67 @@
+/**
+ * THE POLL'S SQL (so that development can read a local database without WAL): the
+ * xmin poll `mason start` keeps a copy current with by default (src/duck/sync.ts), with plain SELECTs — no
+ * `wal_level=logical`, no replication slot, no publication, no REPLICATION attribute, nothing created or changed on
+ * the source. Each poll reads, in one snapshot, every row whose `xmin` (the transaction that wrote this version of it)
+ * is at or past the WATERMARK — the oldest transaction still running at the last poll (`pg_snapshot_xmin`):
+ * everything that committed before it was read by an earlier poll (or the copy); a transaction that was running then
+ * has an id at or past it, and so has every subtransaction of it (a savepoint's rows carry the SUBtransaction's id,
+ * which `pg_visible_in_snapshot` would misjudge, so the bound is a range, never a visibility test). A copy taken in
+ * several source transactions is held to its first snapshot by the complement (`xminBeforeSql`).
+ *
+ * WHAT IT COSTS THE SOURCE: a poll reads every polled table whole (no index covers `xmin`), so the interval stretches
+ * to four times what the last poll took (`nextPollDelayMs`).
+ */
+/** The snapshot's `xmin` — the oldest transaction running when it was taken — from its text (`xmin:xmax:xip,…`). */
+export function snapshotXmin(snapshot) {
+    const m = /^(\d+):(\d+):/.exec(snapshot);
+    if (!m)
+        throw new Error(`not a pg_snapshot: ${JSON.stringify(snapshot)}`);
+    return BigInt(m[1]);
+}
+/** The poll's snapshot, its xmin and xmax (64-bit transaction ids, Postgres 13+), read as the transaction's first statement. */
+export const SNAPSHOT_SQL = "SELECT pg_current_snapshot()::text, pg_snapshot_xmax(pg_current_snapshot())::text";
+/**
+ * A row's `xmin` as a 64-bit transaction id. `xmin` is the 32-bit id; its epoch is the current one, or the one
+ * before when its low bits are past the current id's (it was assigned before the counter last wrapped). A row older
+ * than one wrap (2^31 transactions) is frozen long since and reads as old either way: it is never at the watermark.
+ */
+export function xmin8Sql(currentXid8) {
+    const epoch = currentXid8 >> 32n, low = currentXid8 & 0xffffffffn;
+    return `((${epoch}::bigint - (xmin::text::bigint > ${low})::int) << 32 | xmin::text::bigint)`;
+}
+/**
+ * A ROW WRITTEN BEFORE THE WATERMARK: the complement of `xmin8Sql(…) >= watermark` (the poll's "changed since"), for a
+ * copy taken in several source transactions that is held to its FIRST snapshot (src/duck/sync.ts `copy`): every
+ * transaction that commits after a snapshot has an id at or past its xmin (it was running then, or began later), and a
+ * tuple's xmin never changes, so a read in any later snapshot that keeps only these rows keeps none written after the
+ * first — and the catch-up poll from that watermark reads every row this leaves out.
+ *
+ * Written with `age()`, never `xmin::text::bigint` (measured on PG 18 over 3 M rows: the text casts took a
+ * scan from 320 to 1,640 ms of the source's CPU, `age` to 437): `age(x)` is the 32-bit distance from the transaction's
+ * stable latest id L (the next id, read once per transaction) to `x`, so for every row within 2^31 transactions of L,
+ * `age(xmin) > age(watermark)` is exactly "older than the watermark" — while L is within 2^31 of the watermark, which
+ * `xminBeforeHolds` checks. A row whose `age` is negative has a raw xmin more than 2^31 transactions old (frozen: a
+ * freeze keeps the raw id since 9.4), which `xmin8Sql` may place at or past a watermark: it is kept here AND read again by
+ * the catch-up (written by key, so once), never neither. `age` of a special id (frozen, bootstrap) is INT_MAX: kept.
+ * Normal ids never have low bits below 3 (FirstNormalTransactionId), so a watermark's below 3 reads as 3: the same rows.
+ */
+export function xminBeforeSql(watermark) {
+    const low = watermark & 0xffffffffn;
+    return `(age(xmin) > (SELECT age('${low < 3n ? 3n : low}'::xid)) OR age(xmin) < 0)`;
+}
+/** `xminBeforeSql` is exact while the transaction's latest id is within 2^31 of the watermark: held to a margin of half
+ *  that (a billion transactions), `current` being the reading transaction's snapshot xmax (`SNAPSHOT_SQL`) */
+export function xminBeforeHolds(current, watermark) {
+    return current >= watermark && current - watermark < 2n ** 30n;
+}
+/**
+ * The wait before the next poll: MASON_POLL_MS, or four times what the last poll took when that is longer, so a
+ * source whose tables take a while to read spends at most a fifth of its time on polls. Never sooner than either
+ * (the test server: a cap of a minute polled a 22 M-row production table every 74 s for 6.5 hours under
+ * MASON_POLL_MS=3600000, and a poll of 20 s every 60 s is a third of the source's time): the database's load decides,
+ * and a slow poll's freshness is said by its own timing line.
+ */
+export function nextPollDelayMs(pollMs, lastPollMs) {
+    return Math.max(pollMs, 4 * lastPollMs);
+}