@protobase/server 0.51.1 → 0.51.3

package/dist/package.js

@@ -1,4 +1,4 @@
 //#region package.json
-var version = "0.51.1";
+var version = "0.51.3";
 //#endregion
 export { version };

package/dist/realtime/broadcast-api.js

@@ -0,0 +1,66 @@
+import { isDatabaseError } from "../error-handler.js";
+import { realtimeLimits } from "./realtime-limits.js";
+import { batchOf, broadcastProblem, publicationOf } from "./broadcast-request.js";
+import { Hono } from "hono";
+import { bodyLimit } from "hono/body-limit";
+//#region src/realtime/broadcast-api.ts
+const tooFast = () => broadcastProblem(429, "too-many-broadcasts", "Too Many Requests", "Too many messages per second on this topic");
+const typeOf = (c) => c.req.header("content-type")?.split(";")[0]?.trim().toLowerCase();
+const parsedJson = async (c) => {
+	try {
+		return JSON.parse(await c.req.text());
+	} catch (error) {
+		if (error instanceof SyntaxError) throw broadcastProblem(400, "invalid-json", "Bad Request", "The body is not JSON");
+		throw error;
+	}
+};
+/**
+* `POST /api/broadcast` and `POST /api/broadcast/:topic/events/:event`, Supabase Realtime's broadcast API, which
+* supabase-js's `send()` on a channel that has not joined and `httpSend()` call: they send to the channels joined on a
+* topic in this process, as a broadcast sent on the socket, under the same topic rate. A request is admitted by the
+* app's key and the token in its bearer, as the socket is, and its body is at most a socket's frame. A private message
+* reaches private channels only, and only when the caller may send broadcasts on its topic by the policies on
+* `realtime.messages`: the batch drops one it may not, as Supabase does, and the single message is refused with 403.
+*/
+const broadcastApi = ({ caller, broadcasts, access, report }) => {
+	const app = new Hono();
+	const admitted = async (c, next) => {
+		c.set("caller", await caller(c.req.raw));
+		await next();
+	};
+	const limited = bodyLimit({
+		maxSize: realtimeLimits.maxFrameBytes,
+		onError: () => {
+			throw broadcastProblem(413, "too-large", "Payload Too Large", `A broadcast request is at most ${realtimeLimits.maxFrameBytes} bytes`);
+		}
+	});
+	const mayWrite = (who, topic) => access.write(who, topic.replace(/^realtime:/, ""), "broadcast").catch((error) => {
+		if (!isDatabaseError(error)) throw error;
+		report(error);
+		return false;
+	});
+	const allowed = async (who, publications) => {
+		const topics = /* @__PURE__ */ new Map();
+		for (const { topic } of publications.filter((publication) => publication.private)) if (!topics.has(topic)) topics.set(topic, await mayWrite(who, topic));
+		return publications.filter((publication) => !publication.private || topics.get(publication.topic));
+	};
+	app.post("/", admitted, limited, async (c) => {
+		if (typeOf(c) !== "application/json") throw broadcastProblem(415, "json-only", "Unsupported Media Type", "Send the messages as application/json");
+		const publications = await allowed(c.var.caller, batchOf(await parsedJson(c)));
+		if (!broadcasts.publish(publications)) throw tooFast();
+		return c.body(null, 202);
+	});
+	app.post("/:topic/events/:event", admitted, limited, async (c) => {
+		const type = typeOf(c);
+		if (type !== "application/json" && type !== "application/octet-stream") throw broadcastProblem(415, "unsupported-payload", "Unsupported Media Type", "Send the payload as application/json or application/octet-stream");
+		const payload = type === "application/json" ? await parsedJson(c) : await c.req.arrayBuffer();
+		if (type === "application/json" && (typeof payload !== "object" || payload === null || Array.isArray(payload))) throw broadcastProblem(422, "invalid-broadcast", "Unprocessable Content", "Payload must be a JSON object");
+		const publication = publicationOf(c.req.param("topic"), c.req.param("event"), payload, c.req.query("private") === "true");
+		if ((await allowed(c.var.caller, [publication])).length === 0) throw broadcastProblem(403, "unauthorized", "Forbidden", "Unauthorized");
+		if (!broadcasts.publish([publication])) throw tooFast();
+		return c.body(null, 202);
+	});
+	return app;
+};
+//#endregion
+export { broadcastApi };

package/dist/realtime/broadcast-frame.js

@@ -46,25 +46,33 @@ const decodeBroadcastPush = (frame) => {
 const fitsFrame = (text) => encoder.encode(text).length <= 255;
 /**
 * A broadcast as the binary frame realtime-js reads: kind, topic, user event and metadata lengths, encoding, fields,
-* payload. A payload is JSON, or the bytes of a binary broadcast.
+* metadata as JSON, payload. A payload is JSON, or the bytes of a binary broadcast.
 */
-const encodeBroadcast = (topic, event, payload) => {
+const encodeBroadcast = (topic, event, payload, meta) => {
 	const topicBytes = encoder.encode(topic);
 	const eventBytes = encoder.encode(event);
+	const metaBytes = encoder.encode(meta === void 0 ? "" : JSON.stringify(meta));
 	const binary = payload instanceof ArrayBuffer;
 	const body = binary ? new Uint8Array(payload) : encoder.encode(JSON.stringify(payload ?? {}));
-	if (topicBytes.length > 255 || eventBytes.length > 255) throw new Error("A broadcast topic and event are at most 255 bytes each");
-	const frame = new Uint8Array(5 + topicBytes.length + eventBytes.length + body.length);
+	if (topicBytes.length > 255 || eventBytes.length > 255 || metaBytes.length > 255) throw new Error("A broadcast topic, event and metadata are at most 255 bytes each");
+	const frame = new Uint8Array(5 + topicBytes.length + eventBytes.length + metaBytes.length + body.length);
 	frame.set([
 		userBroadcast,
 		topicBytes.length,
 		eventBytes.length,
-		0,
+		metaBytes.length,
 		binary ? binaryEncoding : jsonEncoding
 	]);
-	frame.set(topicBytes, 5);
-	frame.set(eventBytes, 5 + topicBytes.length);
-	frame.set(body, 5 + topicBytes.length + eventBytes.length);
+	let offset = 5;
+	for (const part of [
+		topicBytes,
+		eventBytes,
+		metaBytes,
+		body
+	]) {
+		frame.set(part, offset);
+		offset += part.length;
+	}
 	return frame.buffer;
 };
 //#endregion

package/dist/realtime/broadcast-hub.js

@@ -3,15 +3,19 @@ import { realtimeLimits } from "./realtime-limits.js";
 //#region src/realtime/broadcast-hub.ts
 /**
 * The channels of a process by topic, which hear each other's broadcasts in-process: a cloud app runs as one process,
-* so channels of other processes are not reached. A topic takes `realtimeLimits.topicBroadcastsPerSecond`, whoever sends
+* so channels of other processes are not reached. Private and public channels of a topic do not hear each other, as on
+* Supabase, so a public channel cannot reach past the policies of a private one. A topic takes `realtimeLimits.topicBroadcastsPerSecond`, whoever sends
 * them; its window is kept here, so a topic left empty and joined again does not start a new one.
 */
 const broadcastHub = ({ now = () => Date.now() } = {}) => {
-	const topics = /* @__PURE__ */ new Map();
+	const publicTopics = /* @__PURE__ */ new Map();
+	const privateTopics = /* @__PURE__ */ new Map();
+	const topicsOf = (isPrivate) => isPrivate ? privateTopics : publicTopics;
 	const rate = fixedWindows(realtimeLimits.topicBroadcastsPerSecond, 1e3, now);
 	return {
 		/** Puts the member on the topic until the returned function takes it off. */
 		join: (topic, member) => {
+			const topics = topicsOf(member.private);
 			const members = topics.get(topic) ?? /* @__PURE__ */ new Set();
 			topics.set(topic, members.add(member));
 			return () => {
@@ -25,7 +29,18 @@ const broadcastHub = ({ now = () => Date.now() } = {}) => {
 		*/
 		send: (topic, from, event, payload) => {
 			if (!rate(topic)) return false;
-			for (const member of topics.get(topic) ?? []) if (member !== from || member.self) member.receive(event, payload);
+			for (const member of topicsOf(from.private).get(topic) ?? []) if (member !== from || member.self) member.receive(event, payload);
+			return true;
+		},
+		/**
+		* Sends broadcasts that no channel sends, all of them or, when a topic would take more than its broadcasts for this
+		* second, none: `false` then.
+		*/
+		publish: (publications) => {
+			const takes = /* @__PURE__ */ new Map();
+			for (const { topic } of publications) takes.set(topic, (takes.get(topic) ?? 0) + 1);
+			if (!rate.all(takes)) return false;
+			for (const publication of publications) for (const member of topicsOf(publication.private).get(publication.topic) ?? []) member.receive(publication.event, publication.payload, publication.meta);
 			return true;
 		}
 	};

package/dist/realtime/broadcast-request.js

@@ -0,0 +1,43 @@
+import { HttpProblem } from "../problem.js";
+import { realtimeLimits } from "./realtime-limits.js";
+import { fitsFrame } from "./broadcast-frame.js";
+//#region src/realtime/broadcast-request.ts
+/** A refused broadcast request; `message` too, which realtime-js's `httpSend` reads as the error. */
+const broadcastProblem = (status, slug, title, detail) => new HttpProblem(status, slug, title, detail, { message: detail });
+const invalid = (detail) => broadcastProblem(422, "invalid-broadcast", "Unprocessable Content", detail);
+const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+/**
+* A message as the channels hear it, on the topic they joined: realtime-js names the topic without the `realtime:` its
+* channels join with. A topic and an event are at most 255 bytes, as on the socket.
+*/
+const publicationOf = (topic, event, payload, isPrivate) => {
+	const channelTopic = `realtime:${topic}`;
+	if (!fitsFrame(channelTopic) || !fitsFrame(event)) throw invalid("A broadcast topic and event are at most 255 bytes each");
+	return {
+		topic: channelTopic,
+		event,
+		payload,
+		private: isPrivate
+	};
+};
+/**
+* The body of `POST /api/broadcast`, Supabase's `{ messages: [{ topic, event, payload, private }] }`: each message has a
+* topic, an event and an object payload, and is private when `private` is `true`. A batch holds at most as many messages
+* as a socket sends in a second, refused with 429 as Supabase refuses a batch over its rate.
+*/
+const batchOf = (body) => {
+	const messages = isRecord(body) ? body.messages : void 0;
+	if (!Array.isArray(messages) || messages.length === 0) throw invalid("Send the broadcasts as a list of messages");
+	if (messages.length > realtimeLimits.eventsPerSecond) throw broadcastProblem(429, "too-many-broadcasts", "Too Many Requests", `Too many messages to broadcast, send at most ${realtimeLimits.eventsPerSecond} at once`);
+	return messages.map((message) => {
+		if (!isRecord(message)) throw invalid("Each message is an object");
+		const { topic, event, payload } = message;
+		if (typeof topic !== "string" || topic === "") throw invalid("Each message needs a topic");
+		if (typeof event !== "string" || event === "") throw invalid("Each message needs an event");
+		if (!isRecord(payload)) throw invalid("Each message needs a payload object");
+		if (message.private !== void 0 && message.private !== null && typeof message.private !== "boolean") throw invalid("A message's private is true or false");
+		return publicationOf(topic, event, payload, message.private === true);
+	});
+};
+//#endregion
+export { batchOf, broadcastProblem, publicationOf };

package/dist/realtime/database-broadcasts.js

@@ -0,0 +1,83 @@
+import { writeTransaction } from "../transactions.js";
+import { realtimeLimits } from "./realtime-limits.js";
+import { fitsFrame } from "./broadcast-frame.js";
+import { sql } from "kysely";
+import { Buffer } from "node:buffer";
+//#region src/realtime/database-broadcasts.ts
+/** Messages taken in one transaction at most. */
+const batchSize = 1e3;
+/**
+* The message as the channels joined on its topic hear it, with its id as `meta`, as Supabase sends it; `undefined` for
+* one Supabase would skip, or whose topic or event does not fit a frame or whose payload is over a frame.
+*/
+const publicationOfRow = (row) => {
+	if (row.topic === null || row.event === null || row.payload === null || row.private === null) return void 0;
+	const topic = `realtime:${row.topic}`;
+	if (!fitsFrame(topic) || !fitsFrame(row.event)) return void 0;
+	if (Buffer.byteLength(JSON.stringify(row.payload)) > realtimeLimits.maxFrameBytes) return void 0;
+	return {
+		topic,
+		event: row.event,
+		payload: row.payload,
+		private: row.private,
+		meta: { id: row.id }
+	};
+};
+const takeMessages = (db, statementTimeoutMs) => writeTransaction(db, statementTimeoutMs, async (trx) => {
+	const { rows } = await sql`with taken as (
+        delete from realtime.messages where id in (
+          select id from realtime.messages where extension = 'broadcast' order by inserted_at limit ${batchSize} for update skip locked
+        ) returning id, topic, event, payload, private, inserted_at
+      )
+      select id::text as id, topic, event, payload, private from taken order by inserted_at`.execute(trx);
+	return rows;
+});
+const tableExists = async (db) => (await sql`select to_regclass('realtime.messages') is not null as found`.execute(db)).rows[0].found;
+/**
+* Supabase's broadcast from the database: `realtime.send()` and `realtime.broadcast_changes()` insert a message into
+* `realtime.messages` in the caller's transaction, so only a committed one is seen, and while a socket of this process
+* is open the table is read every `intervalMs`, its messages taken out of it and sent to the channels joined on their
+* topics, the private or the public ones, under the topic's broadcast rate. A message nobody hears is dropped, as on
+* Supabase: the first read when a socket opens takes those sent while none was. A database without
+* `realtime.messages` sends nothing, and is checked again at each read.
+*/
+const databaseBroadcasts = ({ db, statementTimeoutMs, broadcasts, intervalMs = 1e3, report }) => {
+	let followers = 0;
+	let running = false;
+	const read = async () => {
+		if (!await tableExists(db)) return;
+		for (;;) {
+			const rows = await takeMessages(db, statementTimeoutMs);
+			for (const row of rows) {
+				const publication = publicationOfRow(row);
+				if (publication) broadcasts.publish([publication]);
+			}
+			if (rows.length < batchSize) return;
+		}
+	};
+	const tick = async () => {
+		if (followers === 0) {
+			running = false;
+			return;
+		}
+		await read().catch(report);
+		setTimeout(() => void tick(), intervalMs);
+	};
+	return { 
+	/** Reads the table from now on, until the returned function is called; once for each call, however often. */
+follow: () => {
+		followers += 1;
+		if (!running) {
+			running = true;
+			tick();
+		}
+		let followed = true;
+		return () => {
+			if (!followed) return;
+			followed = false;
+			followers -= 1;
+		};
+	} };
+};
+//#endregion
+export { databaseBroadcasts, publicationOfRow };

package/dist/realtime/fixed-window.js

@@ -7,17 +7,18 @@
 const fixedWindows = (max, windowMs, now = () => Date.now()) => {
 	let start = -Infinity;
 	let counts = /* @__PURE__ */ new Map();
-	return (key) => {
+	/** Takes as many as each key names, only when they all fit: whether they did. */
+	const all = (takes) => {
 		const at = now();
 		if (at - start >= windowMs) {
 			start = at;
 			counts = /* @__PURE__ */ new Map();
 		}
-		const count = counts.get(key) ?? 0;
-		if (count >= max) return false;
-		counts.set(key, count + 1);
+		for (const [key, n] of takes) if ((counts.get(key) ?? 0) + n > max) return false;
+		for (const [key, n] of takes) counts.set(key, (counts.get(key) ?? 0) + n);
 		return true;
 	};
+	return Object.assign((key) => all(/* @__PURE__ */ new Map([[key, 1]])), { all });
 };
 /** `fixedWindows` for one key. */
 const fixedWindow = (max, windowMs, now = () => Date.now()) => {

package/dist/realtime/presence-hub.js

@@ -8,10 +8,11 @@ const stateOf = (tracked) => {
 * The presences of a process by topic, in-process like broadcasts: each joined channel tracks at most one, under its
 * key, and channels that listen hear every join and leave on their topic, their own included, as Phoenix Presence
 * sends them. Tracking again replaces the presence: a leave of the old one and a join of the new. Refs count up, unique
-* in the process.
+* in the process. Private and public channels of a topic do not see each other, as with broadcasts.
 */
 const presenceHub = () => {
-	const topics = /* @__PURE__ */ new Map();
+	const publicTopics = /* @__PURE__ */ new Map();
+	const privateTopics = /* @__PURE__ */ new Map();
 	let lastRef = 0;
 	const tell = (members, diff) => {
 		for (const member of members.keys()) if (member.listens) member.receive(diff);
@@ -19,6 +20,7 @@ const presenceHub = () => {
 	return { 
 	/** Puts the member on the topic, with what it may do there until it leaves. */
 join: (topic, member) => {
+		const topics = member.private ? privateTopics : publicTopics;
 		const members = topics.get(topic) ?? /* @__PURE__ */ new Map();
 		topics.set(topic, members.set(member, void 0));
 		const untrack = () => {

package/dist/realtime/realtime-app.js

@@ -1,12 +1,15 @@
+import { broadcastHub } from "./broadcast-hub.js";
 import { changeChecker } from "./change-checker.js";
 import { changeHub } from "./change-hub.js";
 import { channelAccess } from "./channel-access.js";
+import { databaseBroadcasts } from "./database-broadcasts.js";
 import { realtimeRoutes } from "./realtime-routes.js";
 //#region src/realtime/realtime-app.ts
 /**
 * `/api/sb/realtime/v1`: the `postgres_changes` of the live resources, each checked under the subscriber's role, claims and
 * the tables' RLS policies before it is sent, and broadcasts and presence between the channels of the process, on a
-* private channel as the policies on `realtime.messages` allow.
+* private channel as the policies on `realtime.messages` allow, and the broadcasts the database sends with
+* `realtime.send()`.
 */
 const realtimeApp = ({ deps, feed, upgradeWebSocket, caller, report }) => {
 	const live = deps.registry.entries.filter((entry) => entry.model.live);
@@ -25,11 +28,20 @@ const realtimeApp = ({ deps, feed, upgradeWebSocket, caller, report }) => {
 		db: deps.db,
 		statementTimeoutMs: deps.statementTimeoutMs
 	});
+	const broadcasts = broadcastHub();
+	const database = databaseBroadcasts({
+		db: deps.db,
+		statementTimeoutMs: deps.statementTimeoutMs,
+		broadcasts,
+		report
+	});
 	return realtimeRoutes({
 		upgradeWebSocket,
 		caller,
 		live,
 		hub,
+		broadcasts,
+		database,
 		access,
 		report
 	});

package/dist/realtime/realtime-routes.js

@@ -1,5 +1,5 @@
 import { HttpProblem } from "../problem.js";
-import { broadcastHub } from "./broadcast-hub.js";
+import { broadcastApi } from "./broadcast-api.js";
 import { bufferedOf } from "./buffered-amount.js";
 import { vsnOf } from "./phoenix-message.js";
 import { presenceHub } from "./presence-hub.js";
@@ -15,13 +15,20 @@ const keyRequest = (apikey, token) => new Request("http://realtime.invalid/", {
 * resources, and broadcasts and presence between the sockets of this process (see `realtimeSocket`). A missing or wrong
 * key is refused before the upgrade, with 401. A socket whose raw socket is not one of npm `ws`, such as Bun's own `ws`
 * shim's, is closed at once with 1011, since its unsent bytes cannot be counted (`realtimeLimits.maxUnsentBytes`).
+* `POST /api/sb/realtime/v1/api/broadcast` sends to the same channels without a socket (see `broadcastApi`), and so
+* does the database (see `databaseBroadcasts`).
 */
-const realtimeRoutes = ({ upgradeWebSocket, caller, live, hub, access, report }) => {
+const realtimeRoutes = ({ upgradeWebSocket, caller, live, hub, broadcasts, database, access, report }) => {
 	const app = new Hono();
-	const broadcasts = broadcastHub();
 	const presences = presenceHub();
 	let lastId = 0;
 	const nextId = () => ++lastId;
+	app.route("/api/broadcast", broadcastApi({
+		caller,
+		broadcasts,
+		access,
+		report
+	}));
 	app.get("/websocket", async (c, next) => {
 		if (!vsnOf(c.req.query("vsn"))) throw new HttpProblem(400, "invalid-request", "Bad Request", "Use vsn 1.0.0 or 2.0.0");
 		await caller(keyRequest(c.req.query("apikey") ?? ""));
@@ -29,12 +36,18 @@ const realtimeRoutes = ({ upgradeWebSocket, caller, live, hub, access, report })
 	}, upgradeWebSocket((c) => {
 		const apikey = c.req.query("apikey");
 		let socket;
+		let unfollow = () => {};
+		const closed = () => {
+			unfollow();
+			socket?.closed();
+		};
 		return {
 			onOpen: (_event, ws) => {
 				if (bufferedOf(ws.raw) === void 0) {
 					report(/* @__PURE__ */ new Error("Not implemented: realtime needs WebSockets of npm ws, which count unsent bytes; on Bun, load npm ws by its path rather than Bun's ws shim, or run on Node"));
 					return ws.close(1011);
 				}
+				unfollow = database.follow();
 				socket = realtimeSocket({
 					vsn: vsnOf(c.req.query("vsn")),
 					send: (frame) => ws.send(frame),
@@ -51,8 +64,8 @@ const realtimeRoutes = ({ upgradeWebSocket, caller, live, hub, access, report })
 				});
 			},
 			onMessage: (event) => socket?.message(event.data),
-			onError: () => socket?.closed(),
-			onClose: () => socket?.closed()
+			onError: closed,
+			onClose: closed
 		};
 	}));
 	return app;

package/dist/realtime/realtime-socket.js

@@ -1,9 +1,9 @@
 import { HttpProblem } from "../problem.js";
-import { parseBindings } from "./postgres-changes.js";
 import { fixedWindow, fixedWindows } from "./fixed-window.js";
 import { realtimeLimits } from "./realtime-limits.js";
-import { decodeMessage, encodeMessage, reply } from "./phoenix-message.js";
+import { parseBindings } from "./postgres-changes.js";
 import { decodeBroadcastPush, encodeBroadcast, fitsFrame } from "./broadcast-frame.js";
+import { decodeMessage, encodeMessage, reply } from "./phoenix-message.js";
 import { privateGate, publicGate, readRefusalText } from "./channel-gate.js";
 import { randomUUID } from "node:crypto";
 import { Buffer } from "node:buffer";
@@ -72,14 +72,15 @@ const realtimeSocket = ({ vsn, send, buffered, close, resolve, live, hub, broadc
 		event,
 		payload
 	});
-	const hear = (topic, channel, event, payload) => {
+	const hear = (topic, channel, event, payload, meta) => {
 		if (!channel.gate.hears("broadcast")) return;
-		if (vsn === "2.0.0") return deliver(encodeBroadcast(topic, event, payload));
+		if (vsn === "2.0.0") return deliver(encodeBroadcast(topic, event, payload, meta));
 		if (payload instanceof ArrayBuffer) return;
 		toChannel(topic, channel, "broadcast", {
 			type: "broadcast",
 			event,
-			payload
+			payload,
+			...meta && { meta }
 		});
 	};
 	const leave = (topic) => {
@@ -127,11 +128,13 @@ const realtimeSocket = ({ vsn, send, buffered, close, resolve, live, hub, broadc
 		const { topic } = message;
 		const member = {
 			self: broadcast.self === true,
-			receive: (event, payload) => hear(topic, channel, event, payload)
+			private: config.private === true,
+			receive: (event, payload, meta) => hear(topic, channel, event, payload, meta)
 		};
 		const key = typeof presence.key === "string" && presence.key !== "" ? presence.key : randomUUID();
 		const seat = presences.join(topic, {
 			key,
+			private: config.private === true,
 			get listens() {
 				return listens && gate.hears("presence");
 			},

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@protobase/server",
-  "version": "0.51.1",
+  "version": "0.51.3",
   "description": "The Protobase REST API as a Hono app, with Better Auth sign-in.",
   "license": "Apache-2.0",
   "repository": {
@@ -40,9 +40,9 @@
     "@better-auth/passkey": "^1.7.12",
     "@better-auth/sso": "^1.7.12",
     "@modelcontextprotocol/server": "^2.3.1",
-    "@protobase/layout": "0.51.1",
-    "@protobase/query": "0.51.1",
-    "@protobase/schema": "0.51.1",
+    "@protobase/layout": "0.51.3",
+    "@protobase/query": "0.51.3",
+    "@protobase/schema": "0.51.3",
     "@react-email/render": "^2.1.0",
     "@scalar/hono-api-reference": "^0.12.10",
     "@sentry/core": "^11.7.0",