matrix-nodejs-homeserver 0.3.2 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +58 -20
- package/dist/api/longpoll.js +61 -0
- package/dist/api/slidingsync.js +30 -32
- package/dist/api/sync.js +331 -183
- package/dist/core/admin.js +1 -0
- package/dist/core/changes.d.ts +79 -0
- package/dist/core/changes.js +332 -0
- package/dist/core/cluster.d.ts +19 -3
- package/dist/core/cluster.js +44 -4
- package/dist/core/keys.js +4 -2
- package/dist/core/notifier.d.ts +49 -3
- package/dist/core/notifier.js +123 -26
- package/dist/core/push.d.ts +12 -11
- package/dist/core/push.js +32 -24
- package/dist/core/pushrules.js +24 -5
- package/dist/core/rooms.js +7 -1
- package/dist/core/semaphore.js +1 -1
- package/dist/homeserver.d.ts +7 -1
- package/dist/homeserver.js +17 -7
- package/dist/storage/batch.js +72 -1
- package/dist/storage/schema.js +7 -2
- package/dist/storage/sql.d.ts +22 -3
- package/dist/storage/sql.js +126 -28
- package/dist/storage/types.d.ts +15 -1
- package/package.json +1 -1
package/dist/core/admin.js
CHANGED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { DatabaseAdapter } from "../storage/types.js";
|
|
2
|
+
//#region src/core/changes.d.ts
|
|
3
|
+
/**
|
|
4
|
+
* Where keys last changed, for a homeserver that is the only instance on its database.
|
|
5
|
+
*
|
|
6
|
+
* https://github.com/k18k/matrix-nodejs-homeserver/issues/51: most of a `/sync` build asked the
|
|
7
|
+
* database whether something had changed since the token (to-device messages, presence, account
|
|
8
|
+
* data, the user's rooms) and almost always found nothing. Every write passes through the adapter
|
|
9
|
+
* returned by `ChangeTracker.wrap`, which notes, per key, the stream position (or, for cached
|
|
10
|
+
* values, the write) at which it last changed, so `/sync` knows without asking.
|
|
11
|
+
*
|
|
12
|
+
* It is only used without a cluster adapter: another instance's writes would not pass through it.
|
|
13
|
+
* Memory is bounded: each log keeps its most recently changed keys, and forgetting a key raises
|
|
14
|
+
* the log's floor, at or below which every key counts as changed (and the database is asked).
|
|
15
|
+
*/
|
|
16
|
+
export declare class ChangeLog {
|
|
17
|
+
private latest;
|
|
18
|
+
private floor;
|
|
19
|
+
private max;
|
|
20
|
+
constructor(max: number, floor?: number);
|
|
21
|
+
/** `key` changed at `pos`. */
|
|
22
|
+
record(key: string, pos: number): void;
|
|
23
|
+
/** Every key may have changed at or below `pos` (for writes whose keys are not known). */
|
|
24
|
+
raiseFloor(pos: number): void;
|
|
25
|
+
/** Set the floor (also lower it: from "nothing is known" to a starting position). */
|
|
26
|
+
setFloor(pos: number): void;
|
|
27
|
+
/** Whether `key` may have changed after `pos`; false only when it is known not to have. */
|
|
28
|
+
changedAfter(key: string, pos: number): boolean;
|
|
29
|
+
/** The keys among `keys` that may have changed after `pos`. */
|
|
30
|
+
filterChanged(keys: Iterable<string>, pos: number): string[];
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Values read from the database, kept until a write changes their key. A value is only kept if
|
|
34
|
+
* no write to its key started while it was being read; in a transaction, the keys are marked
|
|
35
|
+
* again once it commits, so a value read before the commit is not kept either.
|
|
36
|
+
*/
|
|
37
|
+
export declare class ValueCache<V> {
|
|
38
|
+
private values;
|
|
39
|
+
private writes;
|
|
40
|
+
private clock;
|
|
41
|
+
private max;
|
|
42
|
+
constructor(max: number, clock: () => number);
|
|
43
|
+
/** A write to `key` (or, without a key, to any) has started or finished at `tick`. */
|
|
44
|
+
written(key: string | null, tick: number): void;
|
|
45
|
+
/** The kept value of `key`, if there is one and no write has changed it since it was read. */
|
|
46
|
+
peek(key: string): V | undefined;
|
|
47
|
+
/**
|
|
48
|
+
* Keep `value`, read by a read that began at `readAt` (the clock then): unless a write to `key`
|
|
49
|
+
* began since, which may or may not be in `value`.
|
|
50
|
+
*/
|
|
51
|
+
store(key: string, value: V, readAt: number): void;
|
|
52
|
+
get(key: string, load: () => Promise<V>): Promise<V>;
|
|
53
|
+
}
|
|
54
|
+
/** Stream logs: keys that changed at a stream position. */
|
|
55
|
+
export type StreamLogName = "toDevice" | "presence" | "deviceLists" | "accountData" | "roomEvents" | "roomReceipts" | "roomMembers" | "roomPartialState";
|
|
56
|
+
/** Value caches: values kept until their key is written. */
|
|
57
|
+
export type ValueName = "userMemberships" | "ignoredUsers" | "oneTimeKeys" | "fallbackKeys" | "presenceRow" | "readReceipts" | "unreadCounts";
|
|
58
|
+
export declare class ChangeTracker {
|
|
59
|
+
readonly logs: Record<StreamLogName, ChangeLog>;
|
|
60
|
+
readonly values: Record<ValueName, ValueCache<any>>;
|
|
61
|
+
private tick;
|
|
62
|
+
constructor();
|
|
63
|
+
/**
|
|
64
|
+
* Begin answering for changes after `position`. Until then, every key counts as changed. Every
|
|
65
|
+
* write after the adapter was wrapped is noted, so the logs know all changes after `position`
|
|
66
|
+
* if it was read after wrapping.
|
|
67
|
+
*/
|
|
68
|
+
start(position: number): void;
|
|
69
|
+
/** The clock to pass to `ValueCache.store` for a read that begins now. */
|
|
70
|
+
now(): number;
|
|
71
|
+
/** Note a write to `model`: `rows` written, or rows matching `where` changed or deleted. */
|
|
72
|
+
private note;
|
|
73
|
+
/** Mark cached values written again (after a transaction committed). */
|
|
74
|
+
private rewritten;
|
|
75
|
+
/** An adapter that notes every write before passing it on to `db`. */
|
|
76
|
+
wrap(db: DatabaseAdapter): DatabaseAdapter;
|
|
77
|
+
private wrapWith;
|
|
78
|
+
}
|
|
79
|
+
//#endregion
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
import { readBatch } from "../storage/batch.js";
|
|
2
|
+
//#region src/core/changes.ts
|
|
3
|
+
/** Rows kept in a cache are shared between requests: freeze them, so a change fails loudly. */
|
|
4
|
+
function freeze(value) {
|
|
5
|
+
if (value && typeof value === "object" && !Object.isFrozen(value)) {
|
|
6
|
+
Object.freeze(value);
|
|
7
|
+
for (const v of Object.values(value)) freeze(v);
|
|
8
|
+
}
|
|
9
|
+
return value;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Run `reads` like `readBatch`, but answer from the change tracker what it can (#51); the others
|
|
13
|
+
* still go to the database together. Without a tracker (in a cluster), all go to the database.
|
|
14
|
+
*/
|
|
15
|
+
async function trackedReads(db, tracker, reads) {
|
|
16
|
+
if (!tracker) return readBatch(db, reads.map((r) => r.query));
|
|
17
|
+
const out = [];
|
|
18
|
+
const pending = [];
|
|
19
|
+
reads.forEach((read, i) => {
|
|
20
|
+
const skip = read.unlessChanged;
|
|
21
|
+
if (skip && !tracker.logs[skip.log].changedAfter(skip.key, skip.after)) out[i] = [];
|
|
22
|
+
else if (read.cached && (out[i] = tracker.values[read.cached.name].peek(read.cached.key)) !== void 0) return;
|
|
23
|
+
else pending.push(i);
|
|
24
|
+
});
|
|
25
|
+
if (!pending.length) return out;
|
|
26
|
+
const readAt = tracker.now();
|
|
27
|
+
const rows = await readBatch(db, pending.map((i) => reads[i].query));
|
|
28
|
+
pending.forEach((i, j) => {
|
|
29
|
+
out[i] = rows[j];
|
|
30
|
+
const cached = reads[i].cached;
|
|
31
|
+
if (cached) tracker.values[cached.name].store(cached.key, freeze(rows[j]), readAt);
|
|
32
|
+
});
|
|
33
|
+
return out;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Where keys last changed, for a homeserver that is the only instance on its database.
|
|
37
|
+
*
|
|
38
|
+
* https://github.com/k18k/matrix-nodejs-homeserver/issues/51: most of a `/sync` build asked the
|
|
39
|
+
* database whether something had changed since the token (to-device messages, presence, account
|
|
40
|
+
* data, the user's rooms) and almost always found nothing. Every write passes through the adapter
|
|
41
|
+
* returned by `ChangeTracker.wrap`, which notes, per key, the stream position (or, for cached
|
|
42
|
+
* values, the write) at which it last changed, so `/sync` knows without asking.
|
|
43
|
+
*
|
|
44
|
+
* It is only used without a cluster adapter: another instance's writes would not pass through it.
|
|
45
|
+
* Memory is bounded: each log keeps its most recently changed keys, and forgetting a key raises
|
|
46
|
+
* the log's floor, at or below which every key counts as changed (and the database is asked).
|
|
47
|
+
*/
|
|
48
|
+
var ChangeLog = class {
|
|
49
|
+
latest = /* @__PURE__ */ new Map();
|
|
50
|
+
floor;
|
|
51
|
+
max;
|
|
52
|
+
constructor(max, floor = 0) {
|
|
53
|
+
this.max = max;
|
|
54
|
+
this.floor = floor;
|
|
55
|
+
}
|
|
56
|
+
/** `key` changed at `pos`. */
|
|
57
|
+
record(key, pos) {
|
|
58
|
+
const previous = this.latest.get(key);
|
|
59
|
+
this.latest.delete(key);
|
|
60
|
+
this.latest.set(key, previous === void 0 ? pos : Math.max(previous, pos));
|
|
61
|
+
if (this.latest.size > this.max) {
|
|
62
|
+
const [oldest, at] = this.latest.entries().next().value;
|
|
63
|
+
this.latest.delete(oldest);
|
|
64
|
+
if (at > this.floor) this.floor = at;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
/** Every key may have changed at or below `pos` (for writes whose keys are not known). */
|
|
68
|
+
raiseFloor(pos) {
|
|
69
|
+
if (pos > this.floor) this.floor = pos;
|
|
70
|
+
}
|
|
71
|
+
/** Set the floor (also lower it: from "nothing is known" to a starting position). */
|
|
72
|
+
setFloor(pos) {
|
|
73
|
+
this.floor = pos;
|
|
74
|
+
}
|
|
75
|
+
/** Whether `key` may have changed after `pos`; false only when it is known not to have. */
|
|
76
|
+
changedAfter(key, pos) {
|
|
77
|
+
return pos < this.floor || (this.latest.get(key) ?? Number.NEGATIVE_INFINITY) > pos;
|
|
78
|
+
}
|
|
79
|
+
/** The keys among `keys` that may have changed after `pos`. */
|
|
80
|
+
filterChanged(keys, pos) {
|
|
81
|
+
const out = [];
|
|
82
|
+
for (const k of keys) if (this.changedAfter(k, pos)) out.push(k);
|
|
83
|
+
return out;
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
/**
|
|
87
|
+
* Values read from the database, kept until a write changes their key. A value is only kept if
|
|
88
|
+
* no write to its key started while it was being read; in a transaction, the keys are marked
|
|
89
|
+
* again once it commits, so a value read before the commit is not kept either.
|
|
90
|
+
*/
|
|
91
|
+
var ValueCache = class {
|
|
92
|
+
values = /* @__PURE__ */ new Map();
|
|
93
|
+
writes;
|
|
94
|
+
clock;
|
|
95
|
+
max;
|
|
96
|
+
constructor(max, clock) {
|
|
97
|
+
this.max = max;
|
|
98
|
+
this.clock = clock;
|
|
99
|
+
this.writes = new ChangeLog(max * 2);
|
|
100
|
+
}
|
|
101
|
+
/** A write to `key` (or, without a key, to any) has started or finished at `tick`. */
|
|
102
|
+
written(key, tick) {
|
|
103
|
+
if (key === null) {
|
|
104
|
+
this.writes.raiseFloor(tick);
|
|
105
|
+
this.values.clear();
|
|
106
|
+
} else {
|
|
107
|
+
this.writes.record(key, tick);
|
|
108
|
+
this.values.delete(key);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/** The kept value of `key`, if there is one and no write has changed it since it was read. */
|
|
112
|
+
peek(key) {
|
|
113
|
+
const hit = this.values.get(key);
|
|
114
|
+
if (!hit) return void 0;
|
|
115
|
+
if (this.writes.changedAfter(key, hit.readAt)) {
|
|
116
|
+
this.values.delete(key);
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
this.values.delete(key);
|
|
120
|
+
this.values.set(key, hit);
|
|
121
|
+
return hit.value;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Keep `value`, read by a read that began at `readAt` (the clock then): unless a write to `key`
|
|
125
|
+
* began since, which may or may not be in `value`.
|
|
126
|
+
*/
|
|
127
|
+
store(key, value, readAt) {
|
|
128
|
+
if (this.writes.changedAfter(key, readAt)) return;
|
|
129
|
+
this.values.delete(key);
|
|
130
|
+
this.values.set(key, {
|
|
131
|
+
value,
|
|
132
|
+
readAt
|
|
133
|
+
});
|
|
134
|
+
if (this.values.size > this.max) this.values.delete(this.values.keys().next().value);
|
|
135
|
+
}
|
|
136
|
+
async get(key, load) {
|
|
137
|
+
const hit = this.peek(key);
|
|
138
|
+
if (hit !== void 0) return hit;
|
|
139
|
+
const readAt = this.clock();
|
|
140
|
+
const value = await load();
|
|
141
|
+
this.store(key, value, readAt);
|
|
142
|
+
return value;
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
const LOG_SIZE = 1e5;
|
|
146
|
+
const CACHE_SIZE = 5e4;
|
|
147
|
+
const join = (...parts) => parts.every((p) => typeof p === "string") ? parts.join("|") : null;
|
|
148
|
+
const GLOBAL = () => "*";
|
|
149
|
+
const RULES = {
|
|
150
|
+
to_device: [{
|
|
151
|
+
stream: "toDevice",
|
|
152
|
+
key: (f) => join(f.user_id, f.device_id)
|
|
153
|
+
}],
|
|
154
|
+
presence: [{
|
|
155
|
+
stream: "presence",
|
|
156
|
+
key: GLOBAL
|
|
157
|
+
}, {
|
|
158
|
+
value: "presenceRow",
|
|
159
|
+
key: (f) => join(f.user_id)
|
|
160
|
+
}],
|
|
161
|
+
device_list_changes: [{
|
|
162
|
+
stream: "deviceLists",
|
|
163
|
+
key: GLOBAL
|
|
164
|
+
}],
|
|
165
|
+
account_data: [{
|
|
166
|
+
stream: "accountData",
|
|
167
|
+
key: (f) => join(f.user_id)
|
|
168
|
+
}, {
|
|
169
|
+
value: "ignoredUsers",
|
|
170
|
+
key: (f) => join(f.user_id)
|
|
171
|
+
}],
|
|
172
|
+
events: [{
|
|
173
|
+
stream: "roomEvents",
|
|
174
|
+
key: (f) => join(f.room_id)
|
|
175
|
+
}],
|
|
176
|
+
receipts: [
|
|
177
|
+
{
|
|
178
|
+
stream: "roomReceipts",
|
|
179
|
+
key: (f) => join(f.room_id)
|
|
180
|
+
},
|
|
181
|
+
{
|
|
182
|
+
value: "readReceipts",
|
|
183
|
+
key: (f) => join(f.room_id, f.user_id)
|
|
184
|
+
},
|
|
185
|
+
{
|
|
186
|
+
value: "unreadCounts",
|
|
187
|
+
key: (f) => join(f.user_id, f.room_id)
|
|
188
|
+
}
|
|
189
|
+
],
|
|
190
|
+
memberships: [{
|
|
191
|
+
stream: "roomMembers",
|
|
192
|
+
key: (f) => join(f.room_id)
|
|
193
|
+
}, {
|
|
194
|
+
value: "userMemberships",
|
|
195
|
+
key: (f) => join(f.user_id)
|
|
196
|
+
}],
|
|
197
|
+
partial_state_rooms: [{
|
|
198
|
+
stream: "roomPartialState",
|
|
199
|
+
key: (f) => join(f.room_id),
|
|
200
|
+
pos: "completed_stream"
|
|
201
|
+
}],
|
|
202
|
+
notifications: [{
|
|
203
|
+
value: "unreadCounts",
|
|
204
|
+
key: (f) => join(f.user_id, f.room_id)
|
|
205
|
+
}],
|
|
206
|
+
one_time_keys: [{
|
|
207
|
+
value: "oneTimeKeys",
|
|
208
|
+
key: (f) => join(f.user_id, f.device_id)
|
|
209
|
+
}],
|
|
210
|
+
fallback_keys: [{
|
|
211
|
+
value: "fallbackKeys",
|
|
212
|
+
key: (f) => join(f.user_id, f.device_id)
|
|
213
|
+
}]
|
|
214
|
+
};
|
|
215
|
+
/** The fields a where clause pins to one value. */
|
|
216
|
+
function pinned(where) {
|
|
217
|
+
const out = {};
|
|
218
|
+
for (const w of where) if ((w.operator ?? "eq") === "eq") out[w.field] = w.value;
|
|
219
|
+
return out;
|
|
220
|
+
}
|
|
221
|
+
var ChangeTracker = class {
|
|
222
|
+
logs;
|
|
223
|
+
values;
|
|
224
|
+
tick = 0;
|
|
225
|
+
constructor() {
|
|
226
|
+
const log = () => new ChangeLog(LOG_SIZE, Number.POSITIVE_INFINITY);
|
|
227
|
+
this.logs = {
|
|
228
|
+
toDevice: log(),
|
|
229
|
+
presence: log(),
|
|
230
|
+
deviceLists: log(),
|
|
231
|
+
accountData: log(),
|
|
232
|
+
roomEvents: log(),
|
|
233
|
+
roomReceipts: log(),
|
|
234
|
+
roomMembers: log(),
|
|
235
|
+
roomPartialState: log()
|
|
236
|
+
};
|
|
237
|
+
const clock = () => this.tick;
|
|
238
|
+
const cache = () => new ValueCache(CACHE_SIZE, clock);
|
|
239
|
+
this.values = {
|
|
240
|
+
userMemberships: cache(),
|
|
241
|
+
ignoredUsers: cache(),
|
|
242
|
+
oneTimeKeys: cache(),
|
|
243
|
+
fallbackKeys: cache(),
|
|
244
|
+
presenceRow: cache(),
|
|
245
|
+
readReceipts: cache(),
|
|
246
|
+
unreadCounts: cache()
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Begin answering for changes after `position`. Until then, every key counts as changed. Every
|
|
251
|
+
* write after the adapter was wrapped is noted, so the logs know all changes after `position`
|
|
252
|
+
* if it was read after wrapping.
|
|
253
|
+
*/
|
|
254
|
+
start(position) {
|
|
255
|
+
for (const log of Object.values(this.logs)) log.setFloor(position);
|
|
256
|
+
}
|
|
257
|
+
/** The clock to pass to `ValueCache.store` for a read that begins now. */
|
|
258
|
+
now() {
|
|
259
|
+
return this.tick;
|
|
260
|
+
}
|
|
261
|
+
/** Note a write to `model`: `rows` written, or rows matching `where` changed or deleted. */
|
|
262
|
+
note(model, rows, where, data) {
|
|
263
|
+
const rules = RULES[model];
|
|
264
|
+
if (!rules) return [];
|
|
265
|
+
const tick = ++this.tick;
|
|
266
|
+
const touched = [];
|
|
267
|
+
const sources = rows ?? [{
|
|
268
|
+
...pinned(where ?? []),
|
|
269
|
+
...data
|
|
270
|
+
}];
|
|
271
|
+
for (const fields of sources) for (const rule of rules) {
|
|
272
|
+
const key = rule.key(fields);
|
|
273
|
+
if (rule.value) {
|
|
274
|
+
this.values[rule.value].written(key, tick);
|
|
275
|
+
if (key !== null) touched.push(`${rule.value}\0${key}`);
|
|
276
|
+
}
|
|
277
|
+
if (rule.stream) {
|
|
278
|
+
const pos = fields[rule.pos ?? "stream"];
|
|
279
|
+
if (typeof pos !== "number") continue;
|
|
280
|
+
if (key === null) this.logs[rule.stream].raiseFloor(pos);
|
|
281
|
+
else this.logs[rule.stream].record(key, pos);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
return touched;
|
|
285
|
+
}
|
|
286
|
+
/** Mark cached values written again (after a transaction committed). */
|
|
287
|
+
rewritten(touched) {
|
|
288
|
+
const tick = ++this.tick;
|
|
289
|
+
for (const t of touched) {
|
|
290
|
+
const [name, key] = t.split("\0");
|
|
291
|
+
this.values[name].written(key, tick);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
/** An adapter that notes every write before passing it on to `db`. */
|
|
295
|
+
wrap(db) {
|
|
296
|
+
return this.wrapWith(db, null);
|
|
297
|
+
}
|
|
298
|
+
wrapWith(db, inTx) {
|
|
299
|
+
const around = async (touched, run) => {
|
|
300
|
+
try {
|
|
301
|
+
return await run();
|
|
302
|
+
} finally {
|
|
303
|
+
if (inTx) inTx.push(...touched);
|
|
304
|
+
else this.rewritten(touched);
|
|
305
|
+
}
|
|
306
|
+
};
|
|
307
|
+
const adapter = {
|
|
308
|
+
findMany: (model, opts) => db.findMany(model, opts),
|
|
309
|
+
findOne: (model, where) => db.findOne(model, where),
|
|
310
|
+
...db.findManyBatch ? { findManyBatch: (queries, options) => db.findManyBatch(queries, options) } : {},
|
|
311
|
+
create: (model, data) => around(this.note(model, [data], null), () => db.create(model, data)),
|
|
312
|
+
upsert: (model, data) => around(this.note(model, [data], null), () => db.upsert(model, data)),
|
|
313
|
+
...db.upsertMany ? { upsertMany: (model, rows) => around(this.note(model, rows, null), () => db.upsertMany(model, rows)) } : {},
|
|
314
|
+
update: (model, where, data) => around(this.note(model, null, where, data), () => db.update(model, where, data)),
|
|
315
|
+
delete: (model, where) => around(this.note(model, null, where), () => db.delete(model, where)),
|
|
316
|
+
max: (model, field) => db.max(model, field),
|
|
317
|
+
...db.migrate ? { migrate: (schema) => db.migrate(schema) } : {}
|
|
318
|
+
};
|
|
319
|
+
if (db.transaction) adapter.transaction = async (fn) => {
|
|
320
|
+
if (inTx) return db.transaction((tx) => fn(this.wrapWith(tx, inTx)));
|
|
321
|
+
const touched = [];
|
|
322
|
+
try {
|
|
323
|
+
return await db.transaction((tx) => fn(this.wrapWith(tx, touched)));
|
|
324
|
+
} finally {
|
|
325
|
+
this.rewritten(touched);
|
|
326
|
+
}
|
|
327
|
+
};
|
|
328
|
+
return adapter;
|
|
329
|
+
}
|
|
330
|
+
};
|
|
331
|
+
//#endregion
|
|
332
|
+
export { ChangeLog, ChangeTracker, ValueCache, freeze, trackedReads };
|
package/dist/core/cluster.d.ts
CHANGED
|
@@ -52,9 +52,13 @@ export declare class StreamManager {
|
|
|
52
52
|
private lastPos;
|
|
53
53
|
private heartbeat;
|
|
54
54
|
private positionCache;
|
|
55
|
+
private highestAllocated;
|
|
56
|
+
/** Called after a write of this instance completed, when the position may have moved (#50). */
|
|
57
|
+
onAdvance: () => void;
|
|
55
58
|
private publishing;
|
|
56
59
|
private nextPublish;
|
|
57
|
-
|
|
60
|
+
private warn;
|
|
61
|
+
constructor(cluster: ClusterAdapter, instanceId: string, warn?: (...args: unknown[]) => void);
|
|
58
62
|
private key;
|
|
59
63
|
/** Publish this writer's state as it is when the publish starts; concurrent callers share it. */
|
|
60
64
|
private flush;
|
|
@@ -64,8 +68,20 @@ export declare class StreamManager {
|
|
|
64
68
|
write<T>(fn: (id: number) => Promise<T>): Promise<T>;
|
|
65
69
|
/** A fresh decreasing position for backfilled (historical) events. */
|
|
66
70
|
backfillId(): Promise<number>;
|
|
67
|
-
/** The highest position
|
|
68
|
-
|
|
71
|
+
/** The highest position this instance has allocated: an upper bound of its changes so far. */
|
|
72
|
+
get lastAllocated(): number;
|
|
73
|
+
/**
|
|
74
|
+
* The highest position below which everything is persisted, cluster-wide. `fresh` skips the
|
|
75
|
+
* 20 ms cache, for a caller that knows a change has completed elsewhere.
|
|
76
|
+
*/
|
|
77
|
+
position(fresh?: boolean): Promise<number>;
|
|
78
|
+
/**
|
|
79
|
+
* Resolve once the position has reached `pos` (true), or false after `timeoutMs` or on abort.
|
|
80
|
+
* For a request woken by a change that another write in flight still keeps out of sight
|
|
81
|
+
* (https://github.com/k18k/matrix-nodejs-homeserver/issues/50). It checks again at growing
|
|
82
|
+
* intervals: writes finish in milliseconds, and the position may be held by another instance.
|
|
83
|
+
*/
|
|
84
|
+
waitFor(pos: number, timeoutMs: number, signal?: AbortSignal): Promise<boolean>;
|
|
69
85
|
advanceTo(value: number): Promise<void>;
|
|
70
86
|
lowerBackfillTo(value: number): Promise<void>;
|
|
71
87
|
stop(): Promise<void>;
|
package/dist/core/cluster.js
CHANGED
|
@@ -80,6 +80,11 @@ function localCluster() {
|
|
|
80
80
|
}
|
|
81
81
|
const WRITER_TTL_MS = 3e4;
|
|
82
82
|
/**
|
|
83
|
+
* A stream position held longer than this holds back /sync for everyone: it is logged with where
|
|
84
|
+
* it was allocated (https://github.com/k18k/matrix-nodejs-homeserver/issues/48).
|
|
85
|
+
*/
|
|
86
|
+
const SLOW_LEASE_MS = 5e3;
|
|
87
|
+
/**
|
|
83
88
|
* Stream positions with multiple writers. Positions are allocated from a global
|
|
84
89
|
* sequence; readers only see positions up to which every writer has finished, so
|
|
85
90
|
* an event committed late with a lower position can never be skipped by /sync.
|
|
@@ -99,11 +104,16 @@ var StreamManager = class {
|
|
|
99
104
|
lastPos = 0;
|
|
100
105
|
heartbeat;
|
|
101
106
|
positionCache = null;
|
|
107
|
+
highestAllocated = 0;
|
|
108
|
+
/** Called after a write of this instance completed, when the position may have moved (#50). */
|
|
109
|
+
onAdvance = () => {};
|
|
102
110
|
publishing = Promise.resolve();
|
|
103
111
|
nextPublish = null;
|
|
104
|
-
|
|
112
|
+
warn;
|
|
113
|
+
constructor(cluster, instanceId, warn = () => {}) {
|
|
105
114
|
this.cluster = cluster;
|
|
106
115
|
this.instanceId = instanceId;
|
|
116
|
+
this.warn = warn;
|
|
107
117
|
this.heartbeat = setInterval(() => void this.flush().catch(() => {}), WRITER_TTL_MS / 3);
|
|
108
118
|
this.heartbeat.unref?.();
|
|
109
119
|
}
|
|
@@ -146,10 +156,14 @@ var StreamManager = class {
|
|
|
146
156
|
await this.flush();
|
|
147
157
|
id = await this.cluster.nextId("stream");
|
|
148
158
|
this.inflight.add(id);
|
|
159
|
+
if (id > this.highestAllocated) this.highestAllocated = id;
|
|
149
160
|
} finally {
|
|
150
161
|
this.allocating--;
|
|
151
162
|
}
|
|
152
163
|
let finished = false;
|
|
164
|
+
const site = /* @__PURE__ */ new Error("stream position allocated here");
|
|
165
|
+
const slow = setTimeout(() => this.warn(`Stream position ${id} held for over ${SLOW_LEASE_MS} ms; /sync waits for it`, site), SLOW_LEASE_MS);
|
|
166
|
+
slow.unref?.();
|
|
153
167
|
const timer = setTimeout(() => void lease.done(), 6e4);
|
|
154
168
|
timer.unref?.();
|
|
155
169
|
const lease = {
|
|
@@ -158,9 +172,11 @@ var StreamManager = class {
|
|
|
158
172
|
if (finished) return;
|
|
159
173
|
finished = true;
|
|
160
174
|
clearTimeout(timer);
|
|
175
|
+
clearTimeout(slow);
|
|
161
176
|
this.inflight.delete(id);
|
|
162
177
|
await this.flush();
|
|
163
178
|
this.positionCache = null;
|
|
179
|
+
this.onAdvance();
|
|
164
180
|
}
|
|
165
181
|
};
|
|
166
182
|
return lease;
|
|
@@ -178,9 +194,16 @@ var StreamManager = class {
|
|
|
178
194
|
async backfillId() {
|
|
179
195
|
return -await this.cluster.nextId("backfill");
|
|
180
196
|
}
|
|
181
|
-
/** The highest position
|
|
182
|
-
|
|
183
|
-
|
|
197
|
+
/** The highest position this instance has allocated: an upper bound of its changes so far. */
|
|
198
|
+
get lastAllocated() {
|
|
199
|
+
return this.highestAllocated;
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* The highest position below which everything is persisted, cluster-wide. `fresh` skips the
|
|
203
|
+
* 20 ms cache, for a caller that knows a change has completed elsewhere.
|
|
204
|
+
*/
|
|
205
|
+
async position(fresh = false) {
|
|
206
|
+
if (!fresh && this.positionCache && Date.now() - this.positionCache.at < 20) return this.positionCache.value;
|
|
184
207
|
let pos = await this.cluster.currentId("stream");
|
|
185
208
|
for (const [, w] of await this.cluster.entries("writer:")) if (w?.busy) pos = Math.min(pos, w.pos);
|
|
186
209
|
if (this.inflight.size) pos = Math.min(pos, Math.min(...this.inflight) - 1);
|
|
@@ -190,6 +213,23 @@ var StreamManager = class {
|
|
|
190
213
|
};
|
|
191
214
|
return pos;
|
|
192
215
|
}
|
|
216
|
+
/**
|
|
217
|
+
* Resolve once the position has reached `pos` (true), or false after `timeoutMs` or on abort.
|
|
218
|
+
* For a request woken by a change that another write in flight still keeps out of sight
|
|
219
|
+
* (https://github.com/k18k/matrix-nodejs-homeserver/issues/50). It checks again at growing
|
|
220
|
+
* intervals: writes finish in milliseconds, and the position may be held by another instance.
|
|
221
|
+
*/
|
|
222
|
+
async waitFor(pos, timeoutMs, signal) {
|
|
223
|
+
const deadline = Date.now() + timeoutMs;
|
|
224
|
+
let delay = 5;
|
|
225
|
+
for (;;) {
|
|
226
|
+
if (await this.position(true) >= pos) return true;
|
|
227
|
+
const left = deadline - Date.now();
|
|
228
|
+
if (left <= 0 || signal?.aborted) return false;
|
|
229
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(delay, left)));
|
|
230
|
+
delay = Math.min(delay * 2, 200);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
193
233
|
async advanceTo(value) {
|
|
194
234
|
await this.cluster.advance("stream", value);
|
|
195
235
|
}
|
package/dist/core/keys.js
CHANGED
|
@@ -438,8 +438,10 @@ var KeyService = class {
|
|
|
438
438
|
if (m.membership === "join") {
|
|
439
439
|
for (const u of members) if (u !== userId) changed.add(u);
|
|
440
440
|
} else for (const u of members) if (!sharing.has(u)) left.add(u);
|
|
441
|
-
} else if (m.membership === "join")
|
|
442
|
-
|
|
441
|
+
} else if (m.membership === "join") {
|
|
442
|
+
const ev = await this.hs.rooms.getEvent(m.event_id);
|
|
443
|
+
if (ev && ev.stream > since) changed.add(m.user_id);
|
|
444
|
+
} else if (!sharing.has(m.user_id)) left.add(m.user_id);
|
|
443
445
|
}
|
|
444
446
|
for (const u of left) changed.delete(u);
|
|
445
447
|
return {
|
package/dist/core/notifier.d.ts
CHANGED
|
@@ -1,17 +1,63 @@
|
|
|
1
1
|
import { ClusterAdapter } from "./cluster.js";
|
|
2
2
|
//#region src/core/notifier.d.ts
|
|
3
|
+
/** A registration for wake-ups from the moment it was made (see `Notifier.listen`). */
|
|
4
|
+
export interface Listener {
|
|
5
|
+
/**
|
|
6
|
+
* The stream position of this user's latest wake-up before the listener was registered (0 if
|
|
7
|
+
* none is remembered): the change may not have been visible to the build that preceded it.
|
|
8
|
+
*/
|
|
9
|
+
readonly recent: number;
|
|
10
|
+
/**
|
|
11
|
+
* Resolve with the position of the first wake-up since registration (at once if there was one
|
|
12
|
+
* already), or null after `timeoutMs` or on abort.
|
|
13
|
+
*/
|
|
14
|
+
wait(timeoutMs: number, signal?: AbortSignal): Promise<number | null>;
|
|
15
|
+
/** Stop listening; a pending `wait` resolves with null. */
|
|
16
|
+
cancel(): void;
|
|
17
|
+
}
|
|
3
18
|
/**
|
|
4
19
|
* Wakes long-polling `/sync` requests when something relevant happens. Notifications
|
|
5
20
|
* are broadcast through the cluster adapter so waiters on other instances wake too.
|
|
21
|
+
*
|
|
22
|
+
* https://github.com/k18k/matrix-nodejs-homeserver/issues/50: every wake-up carries the stream
|
|
23
|
+
* position of its change, or an upper bound of it. Writes can complete out of order, so a change
|
|
24
|
+
* may be committed while an earlier write still holds the position below it. As in Synapse
|
|
25
|
+
* (`Notifier._notify_pending_new_room_events`), such a wake-up is held here until the position
|
|
26
|
+
* reaches it: a request is only woken for changes it can see.
|
|
6
27
|
*/
|
|
7
28
|
export declare class Notifier {
|
|
8
|
-
private
|
|
29
|
+
private listeners;
|
|
30
|
+
/** Each user's latest wake-up position, oldest first (re-inserted on update). */
|
|
31
|
+
private recent;
|
|
9
32
|
private cluster;
|
|
10
33
|
private instanceId;
|
|
11
|
-
|
|
12
|
-
|
|
34
|
+
private lastAllocated;
|
|
35
|
+
private position;
|
|
36
|
+
/** Wake-ups whose change is beyond the position known to be visible. */
|
|
37
|
+
private held;
|
|
38
|
+
/** The highest position known to be visible. */
|
|
39
|
+
private visible;
|
|
40
|
+
private release;
|
|
41
|
+
/**
|
|
42
|
+
* `lastAllocated`: the highest stream position this instance has allocated (an upper bound of
|
|
43
|
+
* its changes); `position`: the position below which everything is visible.
|
|
44
|
+
*/
|
|
45
|
+
constructor(cluster: ClusterAdapter, instanceId: string, lastAllocated?: () => number, position?: (fresh: boolean) => Promise<number>);
|
|
46
|
+
/**
|
|
47
|
+
* Wake `userIds`. `pos` is the stream position of the change; without it, the highest position
|
|
48
|
+
* this instance has allocated stands in, which is never below it.
|
|
49
|
+
*/
|
|
50
|
+
notify(userIds: Iterable<string>, pos?: number): void;
|
|
51
|
+
/** The position may have moved (a write of this instance completed): release what it covers. */
|
|
52
|
+
advanced(): void;
|
|
53
|
+
/** Wake now if the change is visible, otherwise once the position reaches it. */
|
|
54
|
+
private hold;
|
|
55
|
+
private scheduleRelease;
|
|
56
|
+
private releaseHeld;
|
|
13
57
|
private wakeLocal;
|
|
14
58
|
notifyAll(): void;
|
|
59
|
+
listen(userId: string): Listener;
|
|
60
|
+
/** Wait for a wake-up of `userId` (for callers that do not track positions). */
|
|
15
61
|
wait(userId: string, timeoutMs: number, signal?: AbortSignal): Promise<void>;
|
|
16
62
|
}
|
|
17
63
|
//#endregion
|