@rebasepro/server-postgres 0.9.1-canary.f2f61da → 0.9.1-canary.faee831

package/src/security/policy-drift.test.ts

@@ -1,7 +1,7 @@
 import { describe, expect, it } from "@jest/globals";
 import type { CollectionConfig } from "@rebasepro/types";
 
-import { checkPolicyDrift, parseExpectedPolicies, formatPolicyDrift, hasDrift, type PolicyRef, type Queryable } from "./policy-drift";
+import { checkPolicyDrift, parseExpectedPolicies, formatPolicyDrift, hasDrift, dropOrphanedPolicies, isGeneratedPolicyName, type PolicyRef, type Queryable } from "./policy-drift";
 import { generatePostgresPoliciesDdl } from "../schema/generate-postgres-ddl-logic";
 
 function collection(slug: string): CollectionConfig {
@@ -154,3 +154,108 @@ describe("checkPolicyDrift", () => {
         expect(drift.missing.length).toBeGreaterThan(0);
     });
 });
+
+describe("isGeneratedPolicyName", () => {
+    it("recognises the generator's own shape", () => {
+        expect(isGeneratedPolicyName("documents_insert_a1b2c3d", "documents")).toBe(true);
+        // One rule spanning several operations appends the operation index.
+        expect(isGeneratedPolicyName("documents_update_a1b2c3d_1", "documents")).toBe(true);
+    });
+
+    it("does not claim names a human could have written", () => {
+        expect(isGeneratedPolicyName("owner_access", "documents")).toBe(false);
+        expect(isGeneratedPolicyName("documents_default_admin_read", "documents")).toBe(false);
+        // Right shape, wrong table — belongs to something else.
+        expect(isGeneratedPolicyName("teams_insert_a1b2c3d", "documents")).toBe(false);
+        // A digest is 7 lowercase hex characters, nothing else.
+        expect(isGeneratedPolicyName("documents_insert_notahex", "documents")).toBe(false);
+        expect(isGeneratedPolicyName("documents_grant_a1b2c3d", "documents")).toBe(false);
+    });
+});
+
+describe("dropOrphanedPolicies", () => {
+    /** Records the DDL issued so the test can assert on what was dropped. */
+    function recordingDb(rows: Record<string, unknown>[]) {
+        const executed: string[] = [];
+        const db: Queryable = {
+            query: async (text: string) => {
+                if (!/^SELECT/i.test(text)) executed.push(text);
+                return { rows: rows as never[] };
+            }
+        };
+        return { db, executed };
+    }
+
+    it("drops the policy a rule edit superseded", async () => {
+        // The reported failure: tightening a rule renames its policy, and the
+        // permissive original stays live and keeps ORing itself back in.
+        const cols = [collection("documents")];
+        const expected = parseExpectedPolicies(generatePostgresPoliciesDdl(cols));
+        const stale = {
+            schemaname: "public", tablename: "documents", policyname: "documents_insert_dead1ee",
+            roles: ["public"], cmd: "INSERT", qual: null, with_check: "true"
+        };
+        const live = [...expected.map((p) => liveRow(p)), stale];
+
+        const drift = await checkPolicyDrift(dbWith(live), cols);
+        const { db, executed } = recordingDb(live);
+        const { dropped, kept } = await dropOrphanedPolicies(db, drift, cols);
+
+        expect(dropped).toHaveLength(1);
+        expect(dropped[0].name).toBe("documents_insert_dead1ee");
+        expect(kept).toHaveLength(0);
+        expect(executed).toEqual([
+            'DROP POLICY IF EXISTS "documents_insert_dead1ee" ON "public"."documents"'
+        ]);
+    });
+
+    it("leaves a hand-written policy alone and reports it instead", async () => {
+        const cols = [collection("documents")];
+        const expected = parseExpectedPolicies(generatePostgresPoliciesDdl(cols));
+        const live = [
+            ...expected.map((p) => liveRow(p)),
+            { schemaname: "public", tablename: "documents", policyname: "ops_break_glass", roles: ["public"], cmd: "ALL", qual: "true", with_check: null }
+        ];
+
+        const drift = await checkPolicyDrift(dbWith(live), cols);
+        const { db, executed } = recordingDb(live);
+        const { dropped, kept } = await dropOrphanedPolicies(db, drift, cols);
+
+        expect(dropped).toHaveLength(0);
+        expect(kept.map((p) => p.name)).toEqual(["ops_break_glass"]);
+        expect(executed).toEqual([]);
+    });
+
+    it("never touches a table the collections do not describe", async () => {
+        // Another application sharing the schema owns this table; a
+        // generator-shaped name there is coincidence, not our leftover.
+        const cols = [collection("documents")];
+        const expected = parseExpectedPolicies(generatePostgresPoliciesDdl(cols));
+        const live = [
+            ...expected.map((p) => liveRow(p)),
+            { schemaname: "public", tablename: "legacy", policyname: "legacy_select_a1b2c3d", roles: ["public"], cmd: "SELECT", qual: "true", with_check: null }
+        ];
+
+        const drift = await checkPolicyDrift(dbWith(live), cols);
+        const { db, executed } = recordingDb(live);
+        const { dropped, kept } = await dropOrphanedPolicies(db, drift, cols);
+
+        expect(dropped).toHaveLength(0);
+        expect(kept.map((p) => p.name)).toEqual(["legacy_select_a1b2c3d"]);
+        expect(executed).toEqual([]);
+    });
+
+    it("does nothing when the database already matches", async () => {
+        const cols = [collection("documents")];
+        const expected = parseExpectedPolicies(generatePostgresPoliciesDdl(cols));
+        const live = expected.map((p) => liveRow(p));
+
+        const drift = await checkPolicyDrift(dbWith(live), cols);
+        const { db, executed } = recordingDb(live);
+        const { dropped, kept } = await dropOrphanedPolicies(db, drift, cols);
+
+        expect(dropped).toHaveLength(0);
+        expect(kept).toHaveLength(0);
+        expect(executed).toEqual([]);
+    });
+});

package/src/security/policy-drift.ts

@@ -192,6 +192,62 @@ export async function checkPolicyDrift(
     return drift;
 }
 
+/**
+ * Does this name look like one the generator produced for this table?
+ *
+ * Unnamed rules compile to `<table>_<op>_<sha1[0:7]>` (plus `_<idx>` when one
+ * rule spans several operations), and the hash covers the rule's semantics — so
+ * *editing* a rule renames its policy. The policy under the old name is left
+ * behind by `db push`, which only DROPs the names it is about to CREATE, and
+ * Postgres ORs PERMISSIVE policies together: a superseded `USING (true)` keeps
+ * granting everything no matter how tight its replacement is.
+ *
+ * Matching the shape is what makes dropping them safe. A hand-written policy
+ * would have to collide with a 7-hex digest to be mistaken for generated one;
+ * a policy named anything else is left alone and merely reported, because a
+ * custom name is indistinguishable from one someone wrote in SQL on purpose.
+ */
+export function isGeneratedPolicyName(name: string, table: string): boolean {
+    return new RegExp(`^${table.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}_(select|insert|update|delete|all)_[0-9a-f]{7}(_\\d+)?$`)
+        .test(name);
+}
+
+export interface OrphanCleanup {
+    /** Superseded generated policies that were dropped. */
+    dropped: PolicyRef[];
+    /** Orphans left in place because their names are not generator-shaped. */
+    kept: PolicyRef[];
+}
+
+/**
+ * Drop the policies an earlier push superseded but never removed.
+ *
+ * Only touches tables the collections describe — a table with no expected
+ * policy is not ours to reconcile, and scanning by schema alone would sweep up
+ * policies belonging to something else sharing the database.
+ */
+export async function dropOrphanedPolicies(
+    client: Queryable,
+    drift: PolicyDrift,
+    collections: CollectionConfig[]
+): Promise<OrphanCleanup> {
+    const expected = parseExpectedPolicies(generatePostgresPoliciesDdl(collections));
+    const managed = new Set(expected.map((p) => `${p.schema}.${p.table}`));
+
+    const cleanup: OrphanCleanup = { dropped: [], kept: [] };
+    for (const p of drift.orphaned) {
+        if (!managed.has(`${p.schema}.${p.table}`) || !isGeneratedPolicyName(p.name, p.table)) {
+            cleanup.kept.push(p);
+            continue;
+        }
+        // Identifiers are quoted, and the name came from pg_policies rather than
+        // from user input, so it is already a valid identifier.
+        await client.query(`DROP POLICY IF EXISTS "${p.name}" ON "${p.schema}"."${p.table}"`);
+        cleanup.dropped.push(p);
+    }
+    return cleanup;
+}
+
 export const hasDrift = (d: PolicyDrift): boolean =>
     d.missing.length > 0 || d.orphaned.length > 0 || d.diverged.length > 0;
 

package/src/security/rls-enforcement.ts

@@ -59,7 +59,7 @@ export interface ConnectionPosture {
 }
 
 export interface AuthContext {
-    userId: string;
+    uid: string;
     /** Raw roles as carried on the user (strings or `{ id }` objects). */
     roles: unknown[];
 }
@@ -208,15 +208,15 @@ export async function ensureAppRole(run: RawSqlRunner, schemas: string[]): Promi
  * semantics: it is why `auth.uid() IS NOT NULL` is true for anonymous requests.
  */
 export async function applyAuthContext(tx: SqlTx, auth: AuthContext, userRole?: string): Promise<void> {
-    const userId = typeof auth.userId === "string" && auth.userId.trim() !== "" ? auth.userId : ANONYMOUS_USER_ID;
+    const uid = typeof auth.uid === "string" && auth.uid.trim() !== "" ? auth.uid : ANONYMOUS_USER_ID;
     const normalizedRoles = auth.roles.map((r: unknown) =>
         typeof r === "string" ? r : (r as Record<string, unknown>)?.id ?? String(r)
     );
     await tx.execute(drizzleSql`
         SELECT
-            set_config('app.user_id', ${userId}, true),
+            set_config('app.user_id', ${uid}, true),
             set_config('app.user_roles', ${normalizedRoles.join(",")}, true),
-            set_config('app.jwt', ${JSON.stringify({ sub: userId, roles: auth.roles })}, true)
+            set_config('app.jwt', ${JSON.stringify({ sub: uid, roles: auth.roles })}, true)
     `);
     if (userRole) {
         await tx.execute(drizzleSql.raw(`SET LOCAL ROLE ${quoteIdent(userRole)}`));

package/src/services/channel-history.ts

@@ -0,0 +1,343 @@
+/**
+ * Ordered, replayable per-channel message history.
+ *
+ * Broadcast on its own is fire-and-forget to whoever is connected at the
+ * instant it is sent: fine for presence and for "someone saved" notifications,
+ * not enough for op-based collaborative editing, where a client that blinks
+ * out for two seconds has to resync a whole document rather than catch up on
+ * the four operations it missed. This adds the missing half — every retained
+ * broadcast gets a per-channel sequence number, and a client can ask for
+ * everything after the last one it saw.
+ *
+ * Three decisions worth stating, because each rules out a simpler-looking one:
+ *
+ *  - **Retention is server-side and opt-in.** A channel is created by whoever
+ *    names it, so a client-supplied history depth would let any visitor commit
+ *    the backend to unbounded storage. And presence channels — the common case
+ *    — must not pay for this: with no rules configured nothing is written, no
+ *    table is created, and `broadcast` runs exactly the code it ran before.
+ *
+ *  - **Sequence numbers come from the database, not from a counter in this
+ *    process.** They have to survive a restart and be shared across instances;
+ *    an in-memory counter would restart at 1 after a deploy and hand a
+ *    reconnecting client a replay from the wrong era, silently.
+ *
+ *  - **The cursor row outlives the messages it numbered.** Pruning is what
+ *    makes retention affordable, but pruning the cursor along with the messages
+ *    would restart the sequence and make `sinceSeq` mean something different
+ *    before and after — the worst kind of bug, because replay would still
+ *    return rows and they would look plausible. Cursors are tiny and are kept
+ *    forever; see {@link prune}, which touches only `channel_messages`.
+ */
+
+import { sql } from "drizzle-orm";
+import { NodePgDatabase } from "drizzle-orm/node-postgres";
+import type { ChannelHistoryEntry, ChannelRetentionRule } from "@rebasepro/types";
+import { logger } from "@rebasepro/server";
+
+/** How many messages a replay returns when the caller does not say. */
+const DEFAULT_REPLAY_LIMIT = 200;
+
+/**
+ * Hard ceiling on one replay, whatever the caller asks for.
+ *
+ * A reconnecting client names its own `limit`, so this is the only thing
+ * standing between a stale `sinceSeq` and a single frame carrying a channel's
+ * entire retained history. A client that is further behind than this is told so
+ * via `latestSeq` and can decide to resync wholesale instead of paging.
+ */
+const MAX_REPLAY_LIMIT = 1000;
+
+/** Minimum gap between two prunes of the same channel. */
+const PRUNE_THROTTLE_MS = 30_000;
+
+/**
+ * Parse a retention TTL into milliseconds.
+ *
+ * Accepts a raw millisecond count or a short duration string (`"30s"`, `"15m"`,
+ * `"24h"`, `"7d"`). Returns undefined for anything unparseable, which the
+ * caller treats as "no TTL" — a misspelt duration must not silently become an
+ * aggressive one.
+ */
+export function parseTtlMs(ttl: number | string | undefined): number | undefined {
+    if (ttl === undefined || ttl === null) return undefined;
+    if (typeof ttl === "number") return Number.isFinite(ttl) && ttl > 0 ? ttl : undefined;
+
+    const match = /^\s*(\d+(?:\.\d+)?)\s*(ms|s|m|h|d)\s*$/i.exec(ttl);
+    if (!match) {
+        logger.warn(`⚠️ [ChannelHistory] Ignoring unparseable retention ttl "${ttl}" — expected e.g. "30s", "15m", "24h", "7d".`);
+        return undefined;
+    }
+    const value = parseFloat(match[1]);
+    const unit = match[2].toLowerCase();
+    const multiplier = unit === "ms" ? 1
+        : unit === "s" ? 1_000
+            : unit === "m" ? 60_000
+                : unit === "h" ? 3_600_000
+                    : 86_400_000;
+    const ms = value * multiplier;
+    return ms > 0 ? ms : undefined;
+}
+
+/**
+ * Whether `channel` is covered by `rule`.
+ *
+ * Exact match, or a trailing `*` acting as a prefix. Not a general glob: this
+ * decides what reaches disk, and a pattern language whose reach is not obvious
+ * at a glance is the wrong tool for that job.
+ */
+export function channelMatchesRule(channel: string, rule: ChannelRetentionRule): boolean {
+    const pattern = rule.match;
+    if (pattern === "*") return true;
+    if (pattern.endsWith("*")) return channel.startsWith(pattern.slice(0, -1));
+    return channel === pattern;
+}
+
+/** A rule with its TTL already resolved to milliseconds. */
+export interface ResolvedRetention {
+    limit?: number;
+    ttlMs?: number;
+}
+
+/**
+ * Persistence and replay for retained channels.
+ *
+ * Inert unless constructed with at least one rule: {@link enabled} is false,
+ * {@link ensureTables} does nothing, and {@link retentionFor} answers undefined
+ * for every channel, so the realtime service never reaches the SQL below.
+ */
+export class ChannelHistoryStore {
+    private rules: ChannelRetentionRule[];
+    /** Resolved rule per channel name, so the match runs once per channel. */
+    private resolved = new Map<string, ResolvedRetention | null>();
+    /** Channel → timestamp of its last prune, for {@link PRUNE_THROTTLE_MS}. */
+    private lastPruned = new Map<string, number>();
+    private tablesReady = false;
+
+    constructor(private db: NodePgDatabase<Record<string, unknown>>, rules: ChannelRetentionRule[] = []) {
+        this.rules = rules.filter(rule => {
+            if (!rule?.match) {
+                logger.warn("⚠️ [ChannelHistory] Ignoring a retention rule with no `match`.");
+                return false;
+            }
+            const hasBound = rule.limit !== undefined || rule.ttl !== undefined;
+            if (!hasBound) {
+                // Unbounded retention is almost never intended and cannot be
+                // walked back once the table has grown, so it is refused rather
+                // than honoured.
+                logger.warn(`⚠️ [ChannelHistory] Retention rule "${rule.match}" sets neither \`limit\` nor \`ttl\` — ignoring it, as it would retain forever.`);
+                return false;
+            }
+            return true;
+        });
+    }
+
+    /** Whether any channel retains anything at all. */
+    get enabled(): boolean {
+        return this.rules.length > 0;
+    }
+
+    /**
+     * The retention that applies to `channel`, or undefined when none does.
+     *
+     * First matching rule wins, so callers order them most-specific first.
+     */
+    retentionFor(channel: string): ResolvedRetention | undefined {
+        if (!this.enabled || !channel) return undefined;
+
+        const cached = this.resolved.get(channel);
+        if (cached !== undefined) return cached ?? undefined;
+
+        const rule = this.rules.find(r => channelMatchesRule(channel, r));
+        const resolved: ResolvedRetention | null = rule
+            ? { limit: rule.limit, ttlMs: parseTtlMs(rule.ttl) }
+            : null;
+
+        // Bounded by the number of distinct channel names seen, which is the
+        // same thing the in-memory channel and presence maps are bounded by.
+        this.resolved.set(channel, resolved);
+        return resolved ?? undefined;
+    }
+
+    /**
+     * Create the history tables. Idempotent, and a no-op when no rule is set —
+     * a deployment that never retains anything gets no schema for it.
+     */
+    async ensureTables(): Promise<void> {
+        if (!this.enabled || this.tablesReady) return;
+
+        await this.db.execute(sql`CREATE SCHEMA IF NOT EXISTS rebase`);
+
+        // The primary key is exactly the replay query's access path
+        // (`channel = $1 AND seq > $2 ORDER BY seq`), so it needs no further
+        // index of its own.
+        await this.db.execute(sql`
+            CREATE TABLE IF NOT EXISTS rebase.channel_messages (
+                channel TEXT NOT NULL,
+                seq BIGINT NOT NULL,
+                event TEXT NOT NULL,
+                payload JSONB,
+                sender_id TEXT,
+                created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+                PRIMARY KEY (channel, seq)
+            )
+        `);
+
+        // Only for the TTL arm of pruning; the limit arm rides the primary key.
+        await this.db.execute(sql`
+            CREATE INDEX IF NOT EXISTS idx_channel_messages_created
+            ON rebase.channel_messages (created_at)
+        `);
+
+        // Never pruned — see the note at the top of this file. One row per
+        // channel that has ever retained a message.
+        await this.db.execute(sql`
+            CREATE TABLE IF NOT EXISTS rebase.channel_cursors (
+                channel TEXT PRIMARY KEY,
+                last_seq BIGINT NOT NULL
+            )
+        `);
+
+        this.tablesReady = true;
+        logger.info(`✅ [ChannelHistory] Retained channels ready (${this.rules.length} rule(s)).`);
+    }
+
+    /**
+     * Append a broadcast and return the sequence number it was given.
+     *
+     * The sequence is allocated by the same statement that stores the message,
+     * so a crash between the two is not a possibility. `ON CONFLICT DO UPDATE`
+     * takes a row lock on the channel's cursor, which is what makes concurrent
+     * broadcasts to one channel line up in a single order — and what keeps
+     * different channels from contending with each other at all.
+     */
+    async append(
+        channel: string,
+        event: string,
+        payload: unknown,
+        senderId?: string
+    ): Promise<{ seq: number; at: string }> {
+        const result = await this.db.execute(sql`
+            WITH next AS (
+                INSERT INTO rebase.channel_cursors (channel, last_seq)
+                VALUES (${channel}, 1)
+                ON CONFLICT (channel)
+                DO UPDATE SET last_seq = rebase.channel_cursors.last_seq + 1
+                RETURNING last_seq
+            )
+            INSERT INTO rebase.channel_messages (channel, seq, event, payload, sender_id)
+            SELECT ${channel}, next.last_seq, ${event}, ${JSON.stringify(payload ?? null)}::jsonb, ${senderId ?? null}
+            FROM next
+            RETURNING seq, created_at
+        `);
+
+        const row = result.rows[0] as { seq: string | number; created_at: Date | string } | undefined;
+        if (!row) throw new Error(`Failed to append to channel history for "${channel}"`);
+
+        return {
+            // BIGINT comes back as a string from node-postgres; the wire type is
+            // a number, and a channel would need 2^53 messages to notice.
+            seq: Number(row.seq),
+            at: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at)
+        };
+    }
+
+    /**
+     * Everything retained for `channel` after `sinceSeq`, oldest first.
+     *
+     * `latestSeq` is reported whether or not the messages were capped, so a
+     * client that is further behind than one page can tell.
+     */
+    async replay(
+        channel: string,
+        sinceSeq = 0,
+        limit = DEFAULT_REPLAY_LIMIT
+    ): Promise<{ messages: ChannelHistoryEntry[]; latestSeq: number }> {
+        const capped = Math.max(1, Math.min(Math.floor(limit) || DEFAULT_REPLAY_LIMIT, MAX_REPLAY_LIMIT));
+        const after = Number.isFinite(sinceSeq) && sinceSeq > 0 ? Math.floor(sinceSeq) : 0;
+
+        const result = await this.db.execute(sql`
+            SELECT seq, event, payload, sender_id, created_at
+            FROM rebase.channel_messages
+            WHERE channel = ${channel} AND seq > ${after}
+            ORDER BY seq ASC
+            LIMIT ${capped}
+        `);
+
+        const messages = (result.rows as Array<{
+            seq: string | number;
+            event: string;
+            payload: unknown;
+            sender_id: string | null;
+            created_at: Date | string;
+        }>).map(row => ({
+            seq: Number(row.seq),
+            event: row.event,
+            payload: row.payload,
+            senderId: row.sender_id ?? undefined,
+            at: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at)
+        }));
+
+        // Read from the cursor rather than from the messages: the cursor is the
+        // authority on how far the channel has got, and still says so after
+        // pruning has removed the messages it counted.
+        const cursor = await this.db.execute(sql`
+            SELECT last_seq FROM rebase.channel_cursors WHERE channel = ${channel}
+        `);
+        const cursorRow = cursor.rows[0] as { last_seq: string | number } | undefined;
+        const latestSeq = cursorRow ? Number(cursorRow.last_seq) : 0;
+
+        return { messages, latestSeq };
+    }
+
+    /**
+     * Enforce a channel's retention bounds.
+     *
+     * Throttled per channel, so a burst of operations prunes once rather than
+     * once per message — the cost then tracks elapsed time instead of write
+     * volume, which is what makes retention affordable on a hot channel.
+     */
+    async prune(channel: string, retention: ResolvedRetention): Promise<number> {
+        const now = Date.now();
+        const last = this.lastPruned.get(channel) ?? 0;
+        if (now - last < PRUNE_THROTTLE_MS) return 0;
+        this.lastPruned.set(channel, now);
+
+        let deleted = 0;
+
+        if (retention.ttlMs !== undefined) {
+            const result = await this.db.execute(sql`
+                DELETE FROM rebase.channel_messages
+                WHERE channel = ${channel}
+                  AND created_at < NOW() - MAKE_INTERVAL(secs => ${retention.ttlMs / 1000})
+            `);
+            deleted += result.rowCount ?? 0;
+        }
+
+        if (retention.limit !== undefined && retention.limit > 0) {
+            // OFFSET past the newest `limit` rows to find the highest seq that
+            // is no longer wanted, then delete everything at or below it. Fewer
+            // rows than the limit leaves the subquery empty, and the comparison
+            // with NULL deletes nothing.
+            const result = await this.db.execute(sql`
+                DELETE FROM rebase.channel_messages
+                WHERE channel = ${channel}
+                  AND seq <= (
+                      SELECT seq FROM rebase.channel_messages
+                      WHERE channel = ${channel}
+                      ORDER BY seq DESC
+                      OFFSET ${Math.floor(retention.limit)} LIMIT 1
+                  )
+            `);
+            deleted += result.rowCount ?? 0;
+        }
+
+        return deleted;
+    }
+
+    /** Forget throttle and match caches. Called on shutdown. */
+    clear(): void {
+        this.resolved.clear();
+        this.lastPruned.clear();
+    }
+}