@protobase/server 0.59.1 → 0.59.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/better-auth/store/migrations-table.js +1 -1
- package/dist/channels/binary-frame.js +44 -0
- package/dist/channels/channel-protocol.js +16 -3
- package/dist/channels/channel-refusals.js +1 -1
- package/dist/channels/channel-routes.js +22 -9
- package/dist/channels/channel-sender.d.ts +1 -1
- package/dist/channels/channel-sender.js +24 -12
- package/dist/channels/channel-socket.js +12 -7
- package/dist/channels/database-sends.js +57 -0
- package/dist/channels/sends-prune.js +15 -0
- package/dist/channels/sends-schema-sql.js +30 -0
- package/dist/channels/sends-schema.d.ts +7 -0
- package/dist/channels/sends-schema.js +18 -0
- package/dist/channels/sent.js +17 -1
- package/dist/create-admin.d.ts +5 -2
- package/dist/create-admin.js +33 -4
- package/dist/deps.d.ts +2 -0
- package/dist/index.d.ts +6 -2
- package/dist/index.js +6 -2
- package/dist/internal.d.ts +2 -1
- package/dist/internal.js +2 -1
- package/dist/meta.d.ts +2 -0
- package/dist/meta.js +1 -0
- package/dist/notifications/caller.js +2 -2
- package/dist/notifications/deliver.js +14 -2
- package/dist/notifications/push-job.js +5 -3
- package/dist/notifications/schema.js +7 -4
- package/dist/package.js +1 -1
- package/dist/realtime/database-broadcasts.js +2 -25
- package/dist/realtime/followed-poll.js +34 -0
- package/dist/realtime/realtime-prune.d.ts +5 -3
- package/dist/realtime/realtime-prune.js +8 -3
- package/dist/realtime/relay/realtime-options.d.ts +7 -0
- package/dist/realtime/relay/realtime-options.js +7 -1
- package/dist/status/do-not-disturb.d.ts +9 -0
- package/dist/status/do-not-disturb.js +15 -0
- package/dist/status/options.d.ts +15 -0
- package/dist/status/options.js +25 -0
- package/dist/status/routes.js +38 -0
- package/dist/status/schema-sql.js +19 -0
- package/dist/status/schema.d.ts +12 -0
- package/dist/status/schema.js +33 -0
- package/dist/status/status-input.js +33 -0
- package/dist/status/status-store.js +39 -0
- package/package.json +4 -4
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
//#region src/channels/binary-frame.ts
|
|
2
|
+
const encoder = new TextEncoder();
|
|
3
|
+
const decoder = new TextDecoder();
|
|
4
|
+
const lengthBytes = 4;
|
|
5
|
+
/**
|
|
6
|
+
* A binary frame of `/api/channels`: the byte length of a JSON header, the header, then the bytes it carries. A binary
|
|
7
|
+
* broadcast travels as one, its header the message without its payload.
|
|
8
|
+
*/
|
|
9
|
+
const encodeBinaryFrame = (header, bytes) => {
|
|
10
|
+
const head = encoder.encode(JSON.stringify(header));
|
|
11
|
+
const frame = new Uint8Array(lengthBytes + head.length + bytes.byteLength);
|
|
12
|
+
new DataView(frame.buffer).setUint32(0, head.length);
|
|
13
|
+
frame.set(head, lengthBytes);
|
|
14
|
+
frame.set(new Uint8Array(bytes), lengthBytes + head.length);
|
|
15
|
+
return frame.buffer;
|
|
16
|
+
};
|
|
17
|
+
/** The header and bytes of a binary frame, or `undefined` for one cut short or whose header is not a JSON object. */
|
|
18
|
+
const decodeBinaryFrame = (frame) => {
|
|
19
|
+
if (frame.byteLength < lengthBytes) return;
|
|
20
|
+
const end = lengthBytes + new DataView(frame).getUint32(0);
|
|
21
|
+
if (end > frame.byteLength) return;
|
|
22
|
+
const header = parsed(decoder.decode(new Uint8Array(frame, lengthBytes, end - lengthBytes)));
|
|
23
|
+
if (typeof header !== "object" || header === null || Array.isArray(header)) return;
|
|
24
|
+
return {
|
|
25
|
+
header,
|
|
26
|
+
bytes: frame.slice(end)
|
|
27
|
+
};
|
|
28
|
+
};
|
|
29
|
+
const parsed = (text) => {
|
|
30
|
+
try {
|
|
31
|
+
return JSON.parse(text);
|
|
32
|
+
} catch (error) {
|
|
33
|
+
if (error instanceof SyntaxError) return;
|
|
34
|
+
throw error;
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
/** The bytes of an `ArrayBuffer` or a typed array, as an `ArrayBuffer` of their own; `undefined` for anything else. */
|
|
38
|
+
const bytesOf = (value) => {
|
|
39
|
+
if (value instanceof ArrayBuffer) return value;
|
|
40
|
+
if (!ArrayBuffer.isView(value)) return;
|
|
41
|
+
return new Uint8Array(value.buffer, value.byteOffset, value.byteLength).slice().buffer;
|
|
42
|
+
};
|
|
43
|
+
//#endregion
|
|
44
|
+
export { bytesOf, decodeBinaryFrame, encodeBinaryFrame };
|
|
@@ -1,9 +1,22 @@
|
|
|
1
|
+
import { decodeBinaryFrame } from "./binary-frame.js";
|
|
1
2
|
//#region src/channels/channel-protocol.ts
|
|
2
3
|
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3
4
|
const text = (value) => typeof value === "string";
|
|
4
5
|
/** The message a text frame carries, or `undefined` for one that is not JSON or not a message. */
|
|
5
|
-
const decodeClientMessage = (frame) =>
|
|
6
|
-
|
|
6
|
+
const decodeClientMessage = (frame) => messageOf(parsed(frame));
|
|
7
|
+
/**
|
|
8
|
+
* The send of bytes a binary frame carries (`decodeBinaryFrame`): its header a `send` without a payload, its bytes the
|
|
9
|
+
* payload. `undefined` for any other frame.
|
|
10
|
+
*/
|
|
11
|
+
const decodeBinaryMessage = (frame) => {
|
|
12
|
+
const decoded = decodeBinaryFrame(frame);
|
|
13
|
+
if (!decoded || decoded.header.type !== "send" || "payload" in decoded.header) return;
|
|
14
|
+
return messageOf({
|
|
15
|
+
...decoded.header,
|
|
16
|
+
payload: decoded.bytes
|
|
17
|
+
});
|
|
18
|
+
};
|
|
19
|
+
const messageOf = (message) => {
|
|
7
20
|
if (!isRecord(message)) return;
|
|
8
21
|
if (message.ref !== void 0 && !text(message.ref)) return;
|
|
9
22
|
const { type } = message;
|
|
@@ -30,4 +43,4 @@ const parsed = (frame) => {
|
|
|
30
43
|
}
|
|
31
44
|
};
|
|
32
45
|
//#endregion
|
|
33
|
-
export { decodeClientMessage };
|
|
46
|
+
export { decodeBinaryMessage, decodeClientMessage };
|
|
@@ -10,7 +10,7 @@ const refusal = (code, message) => ({
|
|
|
10
10
|
*/
|
|
11
11
|
const channelRefusals = {
|
|
12
12
|
notFound: refusal("not-found", "There is no such record"),
|
|
13
|
-
invalidMessage: refusal("invalid-message", "Send a JSON text frame of a ping, token, join, leave, track, untrack or send"),
|
|
13
|
+
invalidMessage: refusal("invalid-message", "Send a JSON text frame of a ping, token, join, leave, track, untrack or send, or a binary frame of a send"),
|
|
14
14
|
notJoined: refusal("not-joined", "Join the channel first"),
|
|
15
15
|
alreadyJoined: refusal("already-joined", "This connection has joined the channel already"),
|
|
16
16
|
nameTooLong: refusal("name-too-long", "A channel name, its resource, a slash and its key, is at most 255 bytes"),
|
|
@@ -16,11 +16,30 @@ const withBearer = (request, token) => {
|
|
|
16
16
|
headers.set("authorization", `Bearer ${token}`);
|
|
17
17
|
return new Request(request, { headers });
|
|
18
18
|
};
|
|
19
|
-
const invalidBody = () => new HttpProblem(422, "invalid-broadcast", "Unprocessable Content", "Send { \"event\": \"…\", \"payload\": … } as application/json");
|
|
19
|
+
const invalidBody = () => new HttpProblem(422, "invalid-broadcast", "Unprocessable Content", "Send { \"event\": \"…\", \"payload\": … } as application/json, or bytes as application/octet-stream with ?event=…");
|
|
20
|
+
const broadcastOf = async (request) => {
|
|
21
|
+
if (request.header("content-type")?.split(";")[0].trim().toLowerCase() === "application/octet-stream") {
|
|
22
|
+
const event = request.query("event");
|
|
23
|
+
if (event === void 0) throw invalidBody();
|
|
24
|
+
return {
|
|
25
|
+
event,
|
|
26
|
+
payload: await request.arrayBuffer()
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
const body = await request.json().catch((error) => {
|
|
30
|
+
if (error instanceof SyntaxError) throw invalidBody();
|
|
31
|
+
throw error;
|
|
32
|
+
});
|
|
33
|
+
if (!isRecord(body) || typeof body.event !== "string" || !("payload" in body)) throw invalidBody();
|
|
34
|
+
return {
|
|
35
|
+
event: body.event,
|
|
36
|
+
payload: body.payload
|
|
37
|
+
};
|
|
38
|
+
};
|
|
20
39
|
/**
|
|
21
40
|
* `GET /channels`, a WebSocket of channels named by records (see `channelSocket`), refused before the upgrade like
|
|
22
41
|
* any request its caller cannot make. `POST /channels/:resource/:key/broadcast` sends one broadcast without a socket,
|
|
23
|
-
* as the caller, when they may read the record.
|
|
42
|
+
* as the caller, when they may read the record: JSON, or bytes (`broadcastOf`).
|
|
24
43
|
*/
|
|
25
44
|
const channelRoutes = ({ deps, upgradeWebSocket, authenticate, authenticateUpgrade, onError, hubs, follow, report }) => {
|
|
26
45
|
const app = new Hono();
|
|
@@ -72,16 +91,10 @@ const channelRoutes = ({ deps, upgradeWebSocket, authenticate, authenticateUpgra
|
|
|
72
91
|
throw new HttpProblem(413, "too-large", "Payload Too Large", `A broadcast request is at most ${realtimeLimits.maxFrameBytes} bytes`);
|
|
73
92
|
}
|
|
74
93
|
}), async (c) => {
|
|
75
|
-
const body = await c.req.json().catch((error) => {
|
|
76
|
-
if (error instanceof SyntaxError) throw invalidBody();
|
|
77
|
-
throw error;
|
|
78
|
-
});
|
|
79
|
-
if (!isRecord(body) || typeof body.event !== "string" || !("payload" in body)) throw invalidBody();
|
|
80
94
|
const message = {
|
|
81
95
|
resource: c.req.param("resource"),
|
|
82
96
|
key: c.req.param("key"),
|
|
83
|
-
|
|
84
|
-
payload: body.payload
|
|
97
|
+
...await broadcastOf(c.req)
|
|
85
98
|
};
|
|
86
99
|
const record = broadcastRecord(deps.registry, message);
|
|
87
100
|
const session = c.get("session");
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import "../registry.js";
|
|
2
2
|
import "./channel-name.js";
|
|
3
3
|
//#region src/channels/channel-sender.d.ts
|
|
4
|
-
/** A broadcast to the channel of a record, from outside a connection. */
|
|
4
|
+
/** A broadcast to the channel of a record, from outside a connection: a JSON payload, or bytes in an `ArrayBuffer` or typed array. */
|
|
5
5
|
export type ChannelBroadcast = {
|
|
6
6
|
resource: string;
|
|
7
7
|
key: string;
|
|
@@ -2,31 +2,43 @@ import { HttpProblem, notFound } from "../problem.js";
|
|
|
2
2
|
import { fitsFrame } from "../realtime/broadcast-frame.js";
|
|
3
3
|
import { realtimeLimits } from "../realtime/realtime-limits.js";
|
|
4
4
|
import { channelRecord } from "./channel-name.js";
|
|
5
|
+
import { bytesOf } from "./binary-frame.js";
|
|
5
6
|
import { channelRefusals } from "./channel-refusals.js";
|
|
7
|
+
import { packSent } from "./sent.js";
|
|
6
8
|
import { Buffer } from "node:buffer";
|
|
7
9
|
//#region src/channels/channel-sender.ts
|
|
8
10
|
const invalid = (refusal) => new HttpProblem(422, refusal.code, "Unprocessable Content", refusal.message);
|
|
11
|
+
const invalidPayload = {
|
|
12
|
+
code: "invalid-payload",
|
|
13
|
+
message: `A payload is JSON or bytes of at most ${realtimeLimits.maxFrameBytes} bytes`
|
|
14
|
+
};
|
|
15
|
+
const payloadBytes = (payload) => {
|
|
16
|
+
if (payload instanceof ArrayBuffer || ArrayBuffer.isView(payload)) return payload.byteLength;
|
|
17
|
+
const json = JSON.stringify(payload);
|
|
18
|
+
return json === void 0 ? NaN : Buffer.byteLength(json);
|
|
19
|
+
};
|
|
9
20
|
/** The record a broadcast names, its event and payload checked, or a problem. */
|
|
10
21
|
const broadcastRecord = (registry, { resource, key, event, payload }) => {
|
|
11
22
|
const found = channelRecord(registry, resource, key);
|
|
12
23
|
if (found === "too-long") throw invalid(channelRefusals.nameTooLong);
|
|
13
24
|
if (!found) throw notFound(channelRefusals.notFound.message);
|
|
14
25
|
if (!fitsFrame(event)) throw invalid(channelRefusals.eventTooLong);
|
|
15
|
-
|
|
16
|
-
if (json === void 0 || Buffer.byteLength(json) > realtimeLimits.maxFrameBytes) throw invalid({
|
|
17
|
-
code: "invalid-payload",
|
|
18
|
-
message: `A payload is JSON of at most ${realtimeLimits.maxFrameBytes} bytes`
|
|
19
|
-
});
|
|
26
|
+
if (!(payloadBytes(payload) <= realtimeLimits.maxFrameBytes)) throw invalid(invalidPayload);
|
|
20
27
|
return found;
|
|
21
28
|
};
|
|
29
|
+
/** A checked broadcast as the hub publishes it; a typed array's bytes go as an `ArrayBuffer` of their own. */
|
|
30
|
+
const publicationTo = (record, event, sent) => ({
|
|
31
|
+
topic: record.name,
|
|
32
|
+
event,
|
|
33
|
+
payload: packSent({
|
|
34
|
+
...sent,
|
|
35
|
+
payload: bytesOf(sent.payload) ?? sent.payload
|
|
36
|
+
}),
|
|
37
|
+
private: false
|
|
38
|
+
});
|
|
22
39
|
/** Publishes a checked broadcast, refused with 429 when the channel took its broadcasts for this second. */
|
|
23
40
|
const publishTo = async (broadcasts, record, event, sent) => {
|
|
24
|
-
if (await broadcasts.publish([
|
|
25
|
-
topic: record.name,
|
|
26
|
-
event,
|
|
27
|
-
payload: sent,
|
|
28
|
-
private: false
|
|
29
|
-
}])) return;
|
|
41
|
+
if (await broadcasts.publish([publicationTo(record, event, sent)])) return;
|
|
30
42
|
throw new HttpProblem(429, channelRefusals.channelTooBusy.code, "Too Many Requests", channelRefusals.channelTooBusy.message);
|
|
31
43
|
};
|
|
32
44
|
/** The server's own sender over the record channels' hub. */
|
|
@@ -36,4 +48,4 @@ const channelSender = (registry, broadcasts) => ({ broadcast: async (message) =>
|
|
|
36
48
|
payload: message.payload
|
|
37
49
|
}) });
|
|
38
50
|
//#endregion
|
|
39
|
-
export { broadcastRecord, channelSender, publishTo };
|
|
51
|
+
export { broadcastRecord, channelSender, publicationTo, publishTo };
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { fitsFrame } from "../realtime/broadcast-frame.js";
|
|
2
2
|
import { realtimeLimits } from "../realtime/realtime-limits.js";
|
|
3
3
|
import { fixedWindow, fixedWindows } from "../realtime/fixed-window.js";
|
|
4
|
+
import { encodeBinaryFrame } from "./binary-frame.js";
|
|
4
5
|
import { channelRefusals } from "./channel-refusals.js";
|
|
6
|
+
import { packSent, senderOf, unpackSent } from "./sent.js";
|
|
5
7
|
import { channelBounds } from "./channel-bounds.js";
|
|
6
|
-
import { decodeClientMessage } from "./channel-protocol.js";
|
|
8
|
+
import { decodeBinaryMessage, decodeClientMessage } from "./channel-protocol.js";
|
|
7
9
|
import { entriesOf, entryDiff } from "./presence-entries.js";
|
|
8
|
-
import { senderOf } from "./sent.js";
|
|
9
10
|
import { randomUUID } from "node:crypto";
|
|
10
11
|
import { Buffer } from "node:buffer";
|
|
11
12
|
//#region src/channels/channel-socket.ts
|
|
@@ -56,6 +57,10 @@ const channelSocket = ({ send, buffered, close, session: first, reauthenticate,
|
|
|
56
57
|
const push = (message) => {
|
|
57
58
|
if (!open) return;
|
|
58
59
|
if (!(buffered() <= realtimeLimits.maxUnsentBytes)) return shut(policyViolation);
|
|
60
|
+
if (message.type === "broadcast" && message.payload instanceof ArrayBuffer) {
|
|
61
|
+
const { payload, ...header } = message;
|
|
62
|
+
return send(encodeBinaryFrame(header, payload));
|
|
63
|
+
}
|
|
59
64
|
send(JSON.stringify(message));
|
|
60
65
|
};
|
|
61
66
|
const ok = (ref, channel) => {
|
|
@@ -105,7 +110,7 @@ const channelSocket = ({ send, buffered, close, session: first, reauthenticate,
|
|
|
105
110
|
const member = {
|
|
106
111
|
self: false,
|
|
107
112
|
private: false,
|
|
108
|
-
receive: (event, payload) => hear(name, event, payload)
|
|
113
|
+
receive: (event, payload) => hear(name, event, unpackSent(payload))
|
|
109
114
|
};
|
|
110
115
|
const seat = presences.join(name, {
|
|
111
116
|
key: randomUUID(),
|
|
@@ -161,7 +166,7 @@ const channelSocket = ({ send, buffered, close, session: first, reauthenticate,
|
|
|
161
166
|
user: userOf(session),
|
|
162
167
|
payload: message.payload
|
|
163
168
|
};
|
|
164
|
-
if (!await broadcasts.send(message.channel, channel.member, message.event, sent)) return refuse(message.ref, channelRefusals.channelTooBusy);
|
|
169
|
+
if (!await broadcasts.send(message.channel, channel.member, message.event, packSent(sent))) return refuse(message.ref, channelRefusals.channelTooBusy);
|
|
165
170
|
if (message.self === true) hear(message.channel, message.event, sent);
|
|
166
171
|
if (message.ack === true) ok(message.ref);
|
|
167
172
|
};
|
|
@@ -224,15 +229,15 @@ const channelSocket = ({ send, buffered, close, session: first, reauthenticate,
|
|
|
224
229
|
};
|
|
225
230
|
return {
|
|
226
231
|
/**
|
|
227
|
-
* Handles a
|
|
228
|
-
* over the size cap, or of a kind whose size is unknown, closes the socket.
|
|
232
|
+
* Handles a frame after those before it: a text one, or a binary `ArrayBuffer` of a send of bytes; one that is not
|
|
233
|
+
* a message gets an error. A frame over the size cap, or of a kind whose size is unknown, closes the socket.
|
|
229
234
|
*/
|
|
230
235
|
message: (frame) => {
|
|
231
236
|
if (!open) return;
|
|
232
237
|
const size = frameBytes(frame);
|
|
233
238
|
if (!(size <= realtimeLimits.maxFrameBytes)) return shut(messageTooBig);
|
|
234
239
|
stayAwake();
|
|
235
|
-
const message = typeof frame === "string" ? decodeClientMessage(frame) : void 0;
|
|
240
|
+
const message = typeof frame === "string" ? decodeClientMessage(frame) : frame instanceof ArrayBuffer ? decodeBinaryMessage(frame) : void 0;
|
|
236
241
|
if (!message) return push({
|
|
237
242
|
type: "error",
|
|
238
243
|
error: channelRefusals.invalidMessage
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { HttpProblem } from "../problem.js";
|
|
2
|
+
import { writeTransaction } from "../transactions.js";
|
|
3
|
+
import { followedPoll } from "../realtime/followed-poll.js";
|
|
4
|
+
import { broadcastRecord, publicationTo } from "./channel-sender.js";
|
|
5
|
+
import { sendsSchema } from "./sends-schema-sql.js";
|
|
6
|
+
import { sql } from "kysely";
|
|
7
|
+
//#region src/channels/database-sends.ts
|
|
8
|
+
/** Sends taken in one transaction at most. */
|
|
9
|
+
const batchSize = 1e3;
|
|
10
|
+
/**
|
|
11
|
+
* The broadcast a row sends, as the server, under the checks of `channels.broadcast` in API functions; `undefined` for
|
|
12
|
+
* one they refuse, such as for a record the app does not have or a payload over a frame, which is dropped.
|
|
13
|
+
*/
|
|
14
|
+
const publicationOfSend = (registry, row) => {
|
|
15
|
+
try {
|
|
16
|
+
const record = broadcastRecord(registry, row);
|
|
17
|
+
return publicationTo(record, row.event, {
|
|
18
|
+
user: null,
|
|
19
|
+
server: true,
|
|
20
|
+
payload: row.payload
|
|
21
|
+
});
|
|
22
|
+
} catch (error) {
|
|
23
|
+
if (error instanceof HttpProblem) return;
|
|
24
|
+
throw error;
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
const takeSends = (db, statementTimeoutMs) => writeTransaction(db, statementTimeoutMs, async (trx) => {
|
|
28
|
+
const { rows } = await sql`with taken as (
|
|
29
|
+
delete from ${sql.id(sendsSchema, "sends")} where id in (
|
|
30
|
+
select id from ${sql.id(sendsSchema, "sends")} order by id limit ${batchSize} for update skip locked
|
|
31
|
+
) returning id, resource, key, event, payload
|
|
32
|
+
)
|
|
33
|
+
select id::text as id, resource, key, event, payload from taken order by id`.execute(trx);
|
|
34
|
+
return rows;
|
|
35
|
+
});
|
|
36
|
+
/**
|
|
37
|
+
* SQL's broadcasts to the record channels: `channels.send()` inserts a row in the caller's transaction, so only a
|
|
38
|
+
* committed one is seen, and while a record channel's socket of this process is open the table is read every
|
|
39
|
+
* `intervalMs`, its rows taken out of it and sent to the channels of their records, as the server, under the channel's
|
|
40
|
+
* broadcast rate. A send nobody hears is dropped, as is one the checks refuse. With several processes, one takes a row
|
|
41
|
+
* and the relay carries it to the others.
|
|
42
|
+
*/
|
|
43
|
+
const databaseSends = ({ db, statementTimeoutMs, registry, broadcasts, intervalMs = 1e3, report }) => {
|
|
44
|
+
const read = async () => {
|
|
45
|
+
for (;;) {
|
|
46
|
+
const rows = await takeSends(db, statementTimeoutMs);
|
|
47
|
+
for (const row of rows) {
|
|
48
|
+
const publication = publicationOfSend(registry, row);
|
|
49
|
+
if (publication) await broadcasts.publish([publication]);
|
|
50
|
+
}
|
|
51
|
+
if (rows.length < batchSize) return;
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
return followedPoll(read, intervalMs, report);
|
|
55
|
+
};
|
|
56
|
+
//#endregion
|
|
57
|
+
export { databaseSends, publicationOfSend };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { writeTransaction } from "../transactions.js";
|
|
2
|
+
import { sendsSchema } from "./sends-schema-sql.js";
|
|
3
|
+
import { inBatches, pruneBatchSize } from "../realtime/broadcast-prune.js";
|
|
4
|
+
import { sql } from "kysely";
|
|
5
|
+
//#region src/channels/sends-prune.ts
|
|
6
|
+
const deleteOldSends = (db, statementTimeoutMs, limit) => writeTransaction(db, statementTimeoutMs, async (trx) => {
|
|
7
|
+
const { numAffectedRows } = await sql`delete from ${sql.id(sendsSchema, "sends")} where id in (
|
|
8
|
+
select id from ${sql.id(sendsSchema, "sends")} where sent_at < now() - interval '48 hours' limit ${limit} for update skip locked
|
|
9
|
+
)`.execute(trx);
|
|
10
|
+
return Number(numAffectedRows ?? 0n);
|
|
11
|
+
});
|
|
12
|
+
/** Deletes the sends of `channels.sends` more than 48 hours old, which no reader took because no socket was open, in batches. */
|
|
13
|
+
const pruneSends = (db, statementTimeoutMs) => inBatches(pruneBatchSize, (limit) => deleteOldSends(db, statementTimeoutMs, limit));
|
|
14
|
+
//#endregion
|
|
15
|
+
export { pruneSends };
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
//#region src/channels/sends-schema-sql.ts
|
|
2
|
+
/** The schema of SQL's broadcasts to the record channels. */
|
|
3
|
+
const sendsSchema = "channels";
|
|
4
|
+
/**
|
|
5
|
+
* - `sends`: each broadcast SQL sent, inserted in the sender's transaction, taken out by a serving process once
|
|
6
|
+
* committed, or pruned after 48 hours when none read it.
|
|
7
|
+
* - `send(resource, key, event, payload)`: inserts one. It runs as its owner, the app's role, which alone may call it,
|
|
8
|
+
* so that no other role, such as one of `/api/sb/rest/v1`'s, sends as the server; a trigger function that sends for
|
|
9
|
+
* them is `security definer`.
|
|
10
|
+
*/
|
|
11
|
+
const sendsSchemaSql = `create schema if not exists ${sendsSchema};
|
|
12
|
+
|
|
13
|
+
create table if not exists ${sendsSchema}.sends (
|
|
14
|
+
id bigint generated always as identity primary key,
|
|
15
|
+
resource text not null,
|
|
16
|
+
key text not null,
|
|
17
|
+
event text not null,
|
|
18
|
+
payload jsonb not null,
|
|
19
|
+
sent_at timestamptz not null default now()
|
|
20
|
+
);
|
|
21
|
+
|
|
22
|
+
create index if not exists sends_sent_at on ${sendsSchema}.sends (sent_at);
|
|
23
|
+
|
|
24
|
+
create or replace function ${sendsSchema}.send(resource text, key text, event text, payload jsonb) returns void
|
|
25
|
+
language sql security definer set search_path = ''
|
|
26
|
+
as $$ insert into ${sendsSchema}.sends (resource, key, event, payload) values (send.resource, send.key, send.event, send.payload) $$;
|
|
27
|
+
|
|
28
|
+
revoke execute on function ${sendsSchema}.send(text, text, text, jsonb) from public;`;
|
|
29
|
+
//#endregion
|
|
30
|
+
export { sendsSchema, sendsSchemaSql };
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { Db } from "@protobase/query";
|
|
2
|
+
//#region src/channels/sends-schema.d.ts
|
|
3
|
+
/** The SQL that adds the `channels` schema `realtime: { sqlSends: true }` needs; `undefined` when it is there. */
|
|
4
|
+
export declare const channelSendsMigration: (db: Db) => Promise<"create schema if not exists channels;\n\ncreate table if not exists channels.sends (\n id bigint generated always as identity primary key,\n resource text not null,\n key text not null,\n event text not null,\n payload jsonb not null,\n sent_at timestamptz not null default now()\n);\n\ncreate index if not exists sends_sent_at on channels.sends (sent_at);\n\ncreate or replace function channels.send(resource text, key text, event text, payload jsonb) returns void\n language sql security definer set search_path = ''\n as $$ insert into channels.sends (resource, key, event, payload) values (send.resource, send.key, send.event, send.payload) $$;\n\nrevoke execute on function channels.send(text, text, text, jsonb) from public;" | undefined>;
|
|
5
|
+
/** Refuses to go on while the `channels` schema is missing; the project's migrations add it. */
|
|
6
|
+
export declare const checkChannelSends: (db: Db) => Promise<void>;
|
|
7
|
+
//#endregion
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { sendsSchema, sendsSchemaSql } from "./sends-schema-sql.js";
|
|
2
|
+
import { migrationCommand } from "../changes/capture-schema.js";
|
|
3
|
+
import { sql } from "kysely";
|
|
4
|
+
//#region src/channels/sends-schema.ts
|
|
5
|
+
const sendsCurrent = async (db) => {
|
|
6
|
+
const { rows } = await sql`select to_regclass(${`${sendsSchema}.sends`}) is not null
|
|
7
|
+
and to_regprocedure(${`${sendsSchema}.send(text, text, text, jsonb)`}) is not null as found`.execute(db);
|
|
8
|
+
return rows[0].found;
|
|
9
|
+
};
|
|
10
|
+
/** The SQL that adds the `channels` schema `realtime: { sqlSends: true }` needs; `undefined` when it is there. */
|
|
11
|
+
const channelSendsMigration = async (db) => await sendsCurrent(db) ? void 0 : sendsSchemaSql;
|
|
12
|
+
/** Refuses to go on while the `channels` schema is missing; the project's migrations add it. */
|
|
13
|
+
const checkChannelSends = async (db) => {
|
|
14
|
+
if (await sendsCurrent(db)) return;
|
|
15
|
+
throw new Error(`The ${sendsSchema} schema that realtime.sqlSends needs is missing. Write it as a migration with \`${migrationCommand}\`, apply it with the project's migrations, then start again.`);
|
|
16
|
+
};
|
|
17
|
+
//#endregion
|
|
18
|
+
export { channelSendsMigration, checkChannelSends };
|
package/dist/channels/sent.js
CHANGED
|
@@ -1,8 +1,24 @@
|
|
|
1
|
+
import { decodeBinaryFrame, encodeBinaryFrame } from "./binary-frame.js";
|
|
1
2
|
//#region src/channels/sent.ts
|
|
2
3
|
/** The sender as a broadcast message names it. */
|
|
3
4
|
const senderOf = (sent) => ({
|
|
4
5
|
user: sent.user,
|
|
5
6
|
...sent.server && { server: true }
|
|
6
7
|
});
|
|
8
|
+
/**
|
|
9
|
+
* A broadcast as the hub's payload. One of bytes goes as one binary frame, its sender the header, since the relay
|
|
10
|
+
* carries a payload of bytes as they are and any other as JSON, which has no bytes.
|
|
11
|
+
*/
|
|
12
|
+
const packSent = (sent) => sent.payload instanceof ArrayBuffer ? encodeBinaryFrame(senderOf(sent), sent.payload) : sent;
|
|
13
|
+
/** The broadcast a hub's payload carries (`packSent`). */
|
|
14
|
+
const unpackSent = (packed) => {
|
|
15
|
+
if (!(packed instanceof ArrayBuffer)) return packed;
|
|
16
|
+
const { header, bytes } = decodeBinaryFrame(packed);
|
|
17
|
+
return {
|
|
18
|
+
user: header.user,
|
|
19
|
+
...header.server === true && { server: true },
|
|
20
|
+
payload: bytes
|
|
21
|
+
};
|
|
22
|
+
};
|
|
7
23
|
//#endregion
|
|
8
|
-
export { senderOf };
|
|
24
|
+
export { packSent, senderOf, unpackSent };
|
package/dist/create-admin.d.ts
CHANGED
|
@@ -10,6 +10,7 @@ import { HistoryOptions } from "./history/options.js";
|
|
|
10
10
|
import { WebhooksOptions } from "./webhooks/table/options.js";
|
|
11
11
|
import { NotificationsOptions } from "./notifications/options.js";
|
|
12
12
|
import { CallsOptions } from "./calls/calls-options.js";
|
|
13
|
+
import { StatusOptions } from "./status/options.js";
|
|
13
14
|
import { SubscriptionDefinition } from "./webhooks/subscription.js";
|
|
14
15
|
import { FilesOptions } from "./files/options.js";
|
|
15
16
|
import { Difficulty } from "./forms/challenge.js";
|
|
@@ -51,10 +52,12 @@ export type CreateAdminInput = {
|
|
|
51
52
|
subscriptions?: SubscriptionDefinition[];
|
|
52
53
|
/** Webhook subscriptions made in the admin app, from `webhooks: {}` (see `WebhooksOptions`). */
|
|
53
54
|
webhooks?: WebhooksOptions;
|
|
54
|
-
/** How `/api/sb/realtime/v1`
|
|
55
|
+
/** How `/api/sb/realtime/v1` and `/api/channels` are served: by one process or several, and whether SQL sends to channels (see `RealtimeOptions`). */
|
|
55
56
|
realtime?: RealtimeOptions;
|
|
56
57
|
/** Each user's devices and the endpoints that register them, from `notifications: {}` (see `NotificationsOptions`). */
|
|
57
58
|
notifications?: NotificationsOptions;
|
|
59
|
+
/** People's own status, from `status: {}` (see `StatusOptions`): it needs `auth` with organizations. */
|
|
60
|
+
status?: StatusOptions;
|
|
58
61
|
/** The media server behind the calls functions admit members to with `calls.join` (see `CallsOptions`). */
|
|
59
62
|
calls?: CallsOptions;
|
|
60
63
|
/**
|
|
@@ -96,5 +99,5 @@ export type CreateAdminInput = {
|
|
|
96
99
|
upgradeWebSocket?: UpgradeWebSocket;
|
|
97
100
|
};
|
|
98
101
|
/** The admin REST API as a Hono app. It uses only Web APIs, so it runs on Node, Workers and anywhere else Hono does. */
|
|
99
|
-
export declare const createAdmin: ({ resources: appResources, views: appViews, pages, userMenu, bottomBars, db, authenticate, auth, functions, secret, files: filesOptions, events, history: historyOptions, jobs, subscriptions, webhooks: webhooksOptions, realtime, notifications, calls, mobile, app: identity, icons, bundlePlugins, fieldTypes, options, formsDifficulty, commentUsers: givenCommentUsers, verifyToken, upgradeWebSocket }: CreateAdminInput) => Admin;
|
|
102
|
+
export declare const createAdmin: ({ resources: appResources, views: appViews, pages, userMenu, bottomBars, db, authenticate, auth, functions, secret, files: filesOptions, events, history: historyOptions, jobs, subscriptions, webhooks: webhooksOptions, realtime, notifications, status, calls, mobile, app: identity, icons, bundlePlugins, fieldTypes, options, formsDifficulty, commentUsers: givenCommentUsers, verifyToken, upgradeWebSocket }: CreateAdminInput) => Admin;
|
|
100
103
|
//#endregion
|
package/dist/create-admin.js
CHANGED
|
@@ -17,10 +17,11 @@ import { apiKeySettings } from "./api-keys/api-key-settings.js";
|
|
|
17
17
|
import { apiKeyCaller } from "./api-keys/resolve-caller.js";
|
|
18
18
|
import { jwtVerifier } from "./auth/jwt.js";
|
|
19
19
|
import { realtimeApp } from "./realtime/realtime-app.js";
|
|
20
|
-
import { relayed } from "./realtime/relay/realtime-options.js";
|
|
20
|
+
import { relayed, sqlSends } from "./realtime/relay/realtime-options.js";
|
|
21
21
|
import { realtimeHubs } from "./realtime/realtime-hubs.js";
|
|
22
22
|
import { channelSender } from "./channels/channel-sender.js";
|
|
23
23
|
import { channelRoutes } from "./channels/channel-routes.js";
|
|
24
|
+
import { databaseSends } from "./channels/database-sends.js";
|
|
24
25
|
import { startRealtimePrune } from "./realtime/realtime-prune.js";
|
|
25
26
|
import { restRoutes } from "./rest/rest-routes.js";
|
|
26
27
|
import { storageRoutes } from "./storage/storage-routes.js";
|
|
@@ -65,6 +66,10 @@ import { historyRoutes } from "./notifications/history-routes.js";
|
|
|
65
66
|
import { deviceRoutes } from "./notifications/routes.js";
|
|
66
67
|
import { notificationSenders, webPushKey } from "./notifications/senders-of.js";
|
|
67
68
|
import { pushJob } from "./notifications/push-job.js";
|
|
69
|
+
import { onDoNotDisturb } from "./status/do-not-disturb.js";
|
|
70
|
+
import { checkStatusAuth, statusNames } from "./status/options.js";
|
|
71
|
+
import { statusRoutes } from "./status/routes.js";
|
|
72
|
+
import { storeSchema } from "./better-auth/store/store-names.js";
|
|
68
73
|
import { liveSessions } from "./better-auth/live-sessions.js";
|
|
69
74
|
import { checkBottomBars, checkShell } from "./check-shell.js";
|
|
70
75
|
import { checkOwnerFields } from "./check-owners.js";
|
|
@@ -105,13 +110,14 @@ import { createScanGuard } from "./scan-guard.js";
|
|
|
105
110
|
import { failedJobs, failedJobsView } from "./jobs/failed-jobs.js";
|
|
106
111
|
import { Hono } from "hono";
|
|
107
112
|
import { accountPage, accountPageName, checkPages, settingsResources, withSettingsFields } from "@protobase/layout";
|
|
113
|
+
import { statusPath } from "@protobase/schema/internal";
|
|
108
114
|
//#region src/create-admin.ts
|
|
109
115
|
const declaredRoles = (auth, roles) => {
|
|
110
116
|
const names = [...auth?.roles ?? [], ...Object.keys(roles?.definitions ?? {})];
|
|
111
117
|
return names.length > 0 ? [...new Set(names)] : void 0;
|
|
112
118
|
};
|
|
113
119
|
/** The admin REST API as a Hono app. It uses only Web APIs, so it runs on Node, Workers and anywhere else Hono does. */
|
|
114
|
-
const createAdmin = ({ resources: appResources, views: appViews = [], pages = [], userMenu, bottomBars = [], db, authenticate, auth, functions = {}, secret, files: filesOptions, events, history: historyOptions, jobs = [], subscriptions = [], webhooks: webhooksOptions, realtime, notifications, calls, mobile, app: identity, icons, bundlePlugins = [], fieldTypes = {}, options = {}, formsDifficulty, commentUsers: givenCommentUsers, verifyToken, upgradeWebSocket }) => {
|
|
120
|
+
const createAdmin = ({ resources: appResources, views: appViews = [], pages = [], userMenu, bottomBars = [], db, authenticate, auth, functions = {}, secret, files: filesOptions, events, history: historyOptions, jobs = [], subscriptions = [], webhooks: webhooksOptions, realtime, notifications, status, calls, mobile, app: identity, icons, bundlePlugins = [], fieldTypes = {}, options = {}, formsDifficulty, commentUsers: givenCommentUsers, verifyToken, upgradeWebSocket }) => {
|
|
115
121
|
const basePath = options.basePath ?? "/api/v1";
|
|
116
122
|
const appModels = appResources.map((resource) => resource.toModel());
|
|
117
123
|
const withComments = appModels.some((model) => model.comments);
|
|
@@ -172,6 +178,14 @@ const createAdmin = ({ resources: appResources, views: appViews = [], pages = []
|
|
|
172
178
|
const errorReporting = errorReportingSettings(options.errorReporting, globalThis.process?.env ?? {});
|
|
173
179
|
const maps = mapSettings(options.maps, globalThis.process?.env ?? {});
|
|
174
180
|
const senders = notifications ? notificationSenders(notifications) : [];
|
|
181
|
+
if (status) checkStatusAuth(auth);
|
|
182
|
+
const statusTables = status && auth && {
|
|
183
|
+
statuses: statusNames(status),
|
|
184
|
+
members: {
|
|
185
|
+
schema: storeSchema(auth),
|
|
186
|
+
table: "member"
|
|
187
|
+
}
|
|
188
|
+
};
|
|
175
189
|
const pushKey = webPushKey(senders);
|
|
176
190
|
const locale = localeSetting(options.locale);
|
|
177
191
|
const plugins = pluginOptions(options.plugins, globalThis.process?.env ?? {}, bundlePlugins);
|
|
@@ -230,6 +244,7 @@ const createAdmin = ({ resources: appResources, views: appViews = [], pages = []
|
|
|
230
244
|
...maps && { maps },
|
|
231
245
|
...notifications && notifications.ui !== false && { bell: true },
|
|
232
246
|
...pushKey && { webPushKey: pushKey },
|
|
247
|
+
...statusTables && { status: true },
|
|
233
248
|
...locale && { locale },
|
|
234
249
|
plugins,
|
|
235
250
|
sources,
|
|
@@ -399,6 +414,18 @@ const createAdmin = ({ resources: appResources, views: appViews = [], pages = []
|
|
|
399
414
|
}));
|
|
400
415
|
if (upgradeWebSocket) {
|
|
401
416
|
const visitors = (caller) => withAnonymous(caller, options.roles);
|
|
417
|
+
const sends = sqlSends(realtime) ? databaseSends({
|
|
418
|
+
db,
|
|
419
|
+
statementTimeoutMs: deps.statementTimeoutMs,
|
|
420
|
+
registry,
|
|
421
|
+
broadcasts: hubs.records.broadcasts,
|
|
422
|
+
report
|
|
423
|
+
}) : void 0;
|
|
424
|
+
const follow = () => {
|
|
425
|
+
const unrelay = hubs.follow();
|
|
426
|
+
const unsend = sends?.follow();
|
|
427
|
+
return () => (unsend?.(), unrelay());
|
|
428
|
+
};
|
|
402
429
|
app.route(systemPath || "/", channelRoutes({
|
|
403
430
|
deps,
|
|
404
431
|
upgradeWebSocket,
|
|
@@ -406,7 +433,7 @@ const createAdmin = ({ resources: appResources, views: appViews = [], pages = []
|
|
|
406
433
|
authenticateUpgrade: visitors(upgradeAuthenticate),
|
|
407
434
|
onError,
|
|
408
435
|
hubs: hubs.records,
|
|
409
|
-
follow
|
|
436
|
+
follow,
|
|
410
437
|
report
|
|
411
438
|
}));
|
|
412
439
|
}
|
|
@@ -431,6 +458,7 @@ const createAdmin = ({ resources: appResources, views: appViews = [], pages = []
|
|
|
431
458
|
if (auth) removeHistoryWithUsers(auth, history);
|
|
432
459
|
app.route(historyPath(systemPath), guarded((routes) => void routes.route("/", historyRoutes(history, auth?.origins ?? []))));
|
|
433
460
|
}
|
|
461
|
+
if (statusTables) app.route(statusPath(systemPath), guarded((routes) => void routes.route("/", statusRoutes(db, statusTables))));
|
|
434
462
|
if (forms) app.route(`${systemPath}/forms`, formRoutes(deps, forms));
|
|
435
463
|
const feed = tableFeed({
|
|
436
464
|
db,
|
|
@@ -490,6 +518,7 @@ const createAdmin = ({ resources: appResources, views: appViews = [], pages = []
|
|
|
490
518
|
db,
|
|
491
519
|
statementTimeoutMs: deps.statementTimeoutMs,
|
|
492
520
|
relayed: relayed(realtime),
|
|
521
|
+
sqlSends: sqlSends(realtime),
|
|
493
522
|
report
|
|
494
523
|
});
|
|
495
524
|
return Object.assign(app, {
|
|
@@ -500,7 +529,7 @@ const createAdmin = ({ resources: appResources, views: appViews = [], pages = []
|
|
|
500
529
|
...withHistory && days !== void 0 ? [pruneHistoryJob(changes, days)] : [],
|
|
501
530
|
...mentions ? [mentions] : [],
|
|
502
531
|
...fill ? [fill] : [],
|
|
503
|
-
...notifications ? [pushJob(notifications, senders, auth && liveSessions(auth))] : []
|
|
532
|
+
...notifications ? [pushJob(notifications, senders, auth && liveSessions(auth), statusTables ? (jobDb, userIds) => onDoNotDisturb(jobDb, statusTables.statuses, userIds) : void 0)] : []
|
|
504
533
|
]
|
|
505
534
|
});
|
|
506
535
|
};
|
package/dist/deps.d.ts
CHANGED
|
@@ -53,6 +53,8 @@ export type Deps = {
|
|
|
53
53
|
maps?: MapSettings;
|
|
54
54
|
/** Whether the shell draws the notification bell: notifications are on and their `ui` is not false. */
|
|
55
55
|
bell?: boolean;
|
|
56
|
+
/** Whether people set their own status (`status: {}`), which `/meta` names to everyone. */
|
|
57
|
+
status?: boolean;
|
|
56
58
|
/** The VAPID public key browsers subscribe for Web Push with, when notifications have a web sender, which `/meta` names to everyone. */
|
|
57
59
|
webPushKey?: string;
|
|
58
60
|
/** The language the admin app's screens are in for everyone, which `/meta` names to everyone. */
|