@protobase/server 0.51.2 → 0.51.3

package/dist/package.js

@@ -1,4 +1,4 @@
 //#region package.json
-var version = "0.51.2";
+var version = "0.51.3";
 //#endregion
 export { version };

package/dist/realtime/broadcast-frame.js

@@ -46,25 +46,33 @@ const decodeBroadcastPush = (frame) => {
 const fitsFrame = (text) => encoder.encode(text).length <= 255;
 /**
 * A broadcast as the binary frame realtime-js reads: kind, topic, user event and metadata lengths, encoding, fields,
-* payload. A payload is JSON, or the bytes of a binary broadcast.
+* metadata as JSON, payload. A payload is JSON, or the bytes of a binary broadcast.
 */
-const encodeBroadcast = (topic, event, payload) => {
+const encodeBroadcast = (topic, event, payload, meta) => {
 	const topicBytes = encoder.encode(topic);
 	const eventBytes = encoder.encode(event);
+	const metaBytes = encoder.encode(meta === void 0 ? "" : JSON.stringify(meta));
 	const binary = payload instanceof ArrayBuffer;
 	const body = binary ? new Uint8Array(payload) : encoder.encode(JSON.stringify(payload ?? {}));
-	if (topicBytes.length > 255 || eventBytes.length > 255) throw new Error("A broadcast topic and event are at most 255 bytes each");
-	const frame = new Uint8Array(5 + topicBytes.length + eventBytes.length + body.length);
+	if (topicBytes.length > 255 || eventBytes.length > 255 || metaBytes.length > 255) throw new Error("A broadcast topic, event and metadata are at most 255 bytes each");
+	const frame = new Uint8Array(5 + topicBytes.length + eventBytes.length + metaBytes.length + body.length);
 	frame.set([
 		userBroadcast,
 		topicBytes.length,
 		eventBytes.length,
-		0,
+		metaBytes.length,
 		binary ? binaryEncoding : jsonEncoding
 	]);
-	frame.set(topicBytes, 5);
-	frame.set(eventBytes, 5 + topicBytes.length);
-	frame.set(body, 5 + topicBytes.length + eventBytes.length);
+	let offset = 5;
+	for (const part of [
+		topicBytes,
+		eventBytes,
+		metaBytes,
+		body
+	]) {
+		frame.set(part, offset);
+		offset += part.length;
+	}
 	return frame.buffer;
 };
 //#endregion

package/dist/realtime/broadcast-hub.js

@@ -1,5 +1,5 @@
-import { realtimeLimits } from "./realtime-limits.js";
 import { fixedWindows } from "./fixed-window.js";
+import { realtimeLimits } from "./realtime-limits.js";
 //#region src/realtime/broadcast-hub.ts
 /**
 * The channels of a process by topic, which hear each other's broadcasts in-process: a cloud app runs as one process,
@@ -40,7 +40,7 @@ const broadcastHub = ({ now = () => Date.now() } = {}) => {
 			const takes = /* @__PURE__ */ new Map();
 			for (const { topic } of publications) takes.set(topic, (takes.get(topic) ?? 0) + 1);
 			if (!rate.all(takes)) return false;
-			for (const publication of publications) for (const member of topicsOf(publication.private).get(publication.topic) ?? []) member.receive(publication.event, publication.payload);
+			for (const publication of publications) for (const member of topicsOf(publication.private).get(publication.topic) ?? []) member.receive(publication.event, publication.payload, publication.meta);
 			return true;
 		}
 	};

package/dist/realtime/broadcast-request.js

@@ -1,6 +1,6 @@
 import { HttpProblem } from "../problem.js";
-import { fitsFrame } from "./broadcast-frame.js";
 import { realtimeLimits } from "./realtime-limits.js";
+import { fitsFrame } from "./broadcast-frame.js";
 //#region src/realtime/broadcast-request.ts
 /** A refused broadcast request; `message` too, which realtime-js's `httpSend` reads as the error. */
 const broadcastProblem = (status, slug, title, detail) => new HttpProblem(status, slug, title, detail, { message: detail });

package/dist/realtime/database-broadcasts.js

@@ -0,0 +1,83 @@
+import { writeTransaction } from "../transactions.js";
+import { realtimeLimits } from "./realtime-limits.js";
+import { fitsFrame } from "./broadcast-frame.js";
+import { sql } from "kysely";
+import { Buffer } from "node:buffer";
+//#region src/realtime/database-broadcasts.ts
+/** Messages taken in one transaction at most. */
+const batchSize = 1e3;
+/**
+* The message as the channels joined on its topic hear it, with its id as `meta`, as Supabase sends it; `undefined` for
+* one Supabase would skip, or whose topic or event does not fit a frame or whose payload is over a frame.
+*/
+const publicationOfRow = (row) => {
+	if (row.topic === null || row.event === null || row.payload === null || row.private === null) return void 0;
+	const topic = `realtime:${row.topic}`;
+	if (!fitsFrame(topic) || !fitsFrame(row.event)) return void 0;
+	if (Buffer.byteLength(JSON.stringify(row.payload)) > realtimeLimits.maxFrameBytes) return void 0;
+	return {
+		topic,
+		event: row.event,
+		payload: row.payload,
+		private: row.private,
+		meta: { id: row.id }
+	};
+};
+const takeMessages = (db, statementTimeoutMs) => writeTransaction(db, statementTimeoutMs, async (trx) => {
+	const { rows } = await sql`with taken as (
+        delete from realtime.messages where id in (
+          select id from realtime.messages where extension = 'broadcast' order by inserted_at limit ${batchSize} for update skip locked
+        ) returning id, topic, event, payload, private, inserted_at
+      )
+      select id::text as id, topic, event, payload, private from taken order by inserted_at`.execute(trx);
+	return rows;
+});
+const tableExists = async (db) => (await sql`select to_regclass('realtime.messages') is not null as found`.execute(db)).rows[0].found;
+/**
+* Supabase's broadcast from the database: `realtime.send()` and `realtime.broadcast_changes()` insert a message into
+* `realtime.messages` in the caller's transaction, so only a committed one is seen, and while a socket of this process
+* is open the table is read every `intervalMs`, its messages taken out of it and sent to the channels joined on their
+* topics, the private or the public ones, under the topic's broadcast rate. A message nobody hears is dropped, as on
+* Supabase: the first read when a socket opens takes those sent while none was. A database without
+* `realtime.messages` sends nothing, and is checked again at each read.
+*/
+const databaseBroadcasts = ({ db, statementTimeoutMs, broadcasts, intervalMs = 1e3, report }) => {
+	let followers = 0;
+	let running = false;
+	const read = async () => {
+		if (!await tableExists(db)) return;
+		for (;;) {
+			const rows = await takeMessages(db, statementTimeoutMs);
+			for (const row of rows) {
+				const publication = publicationOfRow(row);
+				if (publication) broadcasts.publish([publication]);
+			}
+			if (rows.length < batchSize) return;
+		}
+	};
+	const tick = async () => {
+		if (followers === 0) {
+			running = false;
+			return;
+		}
+		await read().catch(report);
+		setTimeout(() => void tick(), intervalMs);
+	};
+	return { 
+	/** Reads the table from now on, until the returned function is called; once for each call, however often. */
+follow: () => {
+		followers += 1;
+		if (!running) {
+			running = true;
+			tick();
+		}
+		let followed = true;
+		return () => {
+			if (!followed) return;
+			followed = false;
+			followers -= 1;
+		};
+	} };
+};
+//#endregion
+export { databaseBroadcasts, publicationOfRow };

package/dist/realtime/realtime-app.js

@@ -1,12 +1,15 @@
+import { broadcastHub } from "./broadcast-hub.js";
 import { changeChecker } from "./change-checker.js";
 import { changeHub } from "./change-hub.js";
 import { channelAccess } from "./channel-access.js";
+import { databaseBroadcasts } from "./database-broadcasts.js";
 import { realtimeRoutes } from "./realtime-routes.js";
 //#region src/realtime/realtime-app.ts
 /**
 * `/api/sb/realtime/v1`: the `postgres_changes` of the live resources, each checked under the subscriber's role, claims and
 * the tables' RLS policies before it is sent, and broadcasts and presence between the channels of the process, on a
-* private channel as the policies on `realtime.messages` allow.
+* private channel as the policies on `realtime.messages` allow, and the broadcasts the database sends with
+* `realtime.send()`.
 */
 const realtimeApp = ({ deps, feed, upgradeWebSocket, caller, report }) => {
 	const live = deps.registry.entries.filter((entry) => entry.model.live);
@@ -25,11 +28,20 @@ const realtimeApp = ({ deps, feed, upgradeWebSocket, caller, report }) => {
 		db: deps.db,
 		statementTimeoutMs: deps.statementTimeoutMs
 	});
+	const broadcasts = broadcastHub();
+	const database = databaseBroadcasts({
+		db: deps.db,
+		statementTimeoutMs: deps.statementTimeoutMs,
+		broadcasts,
+		report
+	});
 	return realtimeRoutes({
 		upgradeWebSocket,
 		caller,
 		live,
 		hub,
+		broadcasts,
+		database,
 		access,
 		report
 	});

package/dist/realtime/realtime-routes.js

@@ -1,6 +1,5 @@
 import { HttpProblem } from "../problem.js";
 import { broadcastApi } from "./broadcast-api.js";
-import { broadcastHub } from "./broadcast-hub.js";
 import { bufferedOf } from "./buffered-amount.js";
 import { vsnOf } from "./phoenix-message.js";
 import { presenceHub } from "./presence-hub.js";
@@ -16,11 +15,11 @@ const keyRequest = (apikey, token) => new Request("http://realtime.invalid/", {
 * resources, and broadcasts and presence between the sockets of this process (see `realtimeSocket`). A missing or wrong
 * key is refused before the upgrade, with 401. A socket whose raw socket is not one of npm `ws`, such as Bun's own `ws`
 * shim's, is closed at once with 1011, since its unsent bytes cannot be counted (`realtimeLimits.maxUnsentBytes`).
-* `POST /api/sb/realtime/v1/api/broadcast` sends to the same channels without a socket (see `broadcastApi`).
+* `POST /api/sb/realtime/v1/api/broadcast` sends to the same channels without a socket (see `broadcastApi`), and so
+* does the database (see `databaseBroadcasts`).
 */
-const realtimeRoutes = ({ upgradeWebSocket, caller, live, hub, access, report }) => {
+const realtimeRoutes = ({ upgradeWebSocket, caller, live, hub, broadcasts, database, access, report }) => {
 	const app = new Hono();
-	const broadcasts = broadcastHub();
 	const presences = presenceHub();
 	let lastId = 0;
 	const nextId = () => ++lastId;
@@ -37,12 +36,18 @@ const realtimeRoutes = ({ upgradeWebSocket, caller, live, hub, access, report })
 	}, upgradeWebSocket((c) => {
 		const apikey = c.req.query("apikey");
 		let socket;
+		let unfollow = () => {};
+		const closed = () => {
+			unfollow();
+			socket?.closed();
+		};
 		return {
 			onOpen: (_event, ws) => {
 				if (bufferedOf(ws.raw) === void 0) {
 					report(/* @__PURE__ */ new Error("Not implemented: realtime needs WebSockets of npm ws, which count unsent bytes; on Bun, load npm ws by its path rather than Bun's ws shim, or run on Node"));
 					return ws.close(1011);
 				}
+				unfollow = database.follow();
 				socket = realtimeSocket({
 					vsn: vsnOf(c.req.query("vsn")),
 					send: (frame) => ws.send(frame),
@@ -59,8 +64,8 @@ const realtimeRoutes = ({ upgradeWebSocket, caller, live, hub, access, report })
 				});
 			},
 			onMessage: (event) => socket?.message(event.data),
-			onError: () => socket?.closed(),
-			onClose: () => socket?.closed()
+			onError: closed,
+			onClose: closed
 		};
 	}));
 	return app;

package/dist/realtime/realtime-socket.js

@@ -1,8 +1,8 @@
 import { HttpProblem } from "../problem.js";
+import { fixedWindow, fixedWindows } from "./fixed-window.js";
+import { realtimeLimits } from "./realtime-limits.js";
 import { parseBindings } from "./postgres-changes.js";
 import { decodeBroadcastPush, encodeBroadcast, fitsFrame } from "./broadcast-frame.js";
-import { realtimeLimits } from "./realtime-limits.js";
-import { fixedWindow, fixedWindows } from "./fixed-window.js";
 import { decodeMessage, encodeMessage, reply } from "./phoenix-message.js";
 import { privateGate, publicGate, readRefusalText } from "./channel-gate.js";
 import { randomUUID } from "node:crypto";
@@ -72,14 +72,15 @@ const realtimeSocket = ({ vsn, send, buffered, close, resolve, live, hub, broadc
 		event,
 		payload
 	});
-	const hear = (topic, channel, event, payload) => {
+	const hear = (topic, channel, event, payload, meta) => {
 		if (!channel.gate.hears("broadcast")) return;
-		if (vsn === "2.0.0") return deliver(encodeBroadcast(topic, event, payload));
+		if (vsn === "2.0.0") return deliver(encodeBroadcast(topic, event, payload, meta));
 		if (payload instanceof ArrayBuffer) return;
 		toChannel(topic, channel, "broadcast", {
 			type: "broadcast",
 			event,
-			payload
+			payload,
+			...meta && { meta }
 		});
 	};
 	const leave = (topic) => {
@@ -128,7 +129,7 @@ const realtimeSocket = ({ vsn, send, buffered, close, resolve, live, hub, broadc
 		const member = {
 			self: broadcast.self === true,
 			private: config.private === true,
-			receive: (event, payload) => hear(topic, channel, event, payload)
+			receive: (event, payload, meta) => hear(topic, channel, event, payload, meta)
 		};
 		const key = typeof presence.key === "string" && presence.key !== "" ? presence.key : randomUUID();
 		const seat = presences.join(topic, {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@protobase/server",
-  "version": "0.51.2",
+  "version": "0.51.3",
   "description": "The Protobase REST API as a Hono app, with Better Auth sign-in.",
   "license": "Apache-2.0",
   "repository": {
@@ -40,9 +40,9 @@
     "@better-auth/passkey": "^1.7.12",
     "@better-auth/sso": "^1.7.12",
     "@modelcontextprotocol/server": "^2.3.1",
-    "@protobase/layout": "0.51.2",
-    "@protobase/query": "0.51.2",
-    "@protobase/schema": "0.51.2",
+    "@protobase/layout": "0.51.3",
+    "@protobase/query": "0.51.3",
+    "@protobase/schema": "0.51.3",
     "@react-email/render": "^2.1.0",
     "@scalar/hono-api-reference": "^0.12.10",
     "@sentry/core": "^11.7.0",