@3sln/trove 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/core/src/index.js +4 -2
- package/packages/core/src/notifications/channel.js +68 -0
- package/packages/core/src/notifications/index.js +72 -43
- package/packages/core/src/notifications/webpush.js +110 -0
- package/packages/core/src/sqlite-d1.js +14 -1
- package/packages/core/src/sqlite-driver.js +73 -1
- package/packages/server/src/adapters/worker-tasks.js +14 -4
- package/packages/server/src/adapters/worker.js +7 -1
- package/packages/server/src/engine/providers/core.js +31 -4
- package/packages/server/src/index.js +23 -2
- package/packages/server/src/routes.js +21 -14
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@3sln/trove",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.4",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Trove — a self-hostable, plugin-extensible Google Drive. Semantic search, pluggable storage (S3 / filesystem / NAS), and a VS Code-style contribution system with sandboxed plugins.",
|
|
6
6
|
"repository": {
|
|
@@ -66,9 +66,11 @@ export {
|
|
|
66
66
|
export { SidecarService, SidecarStore, SidecarManager } from './sidecar/index.js';
|
|
67
67
|
export * as sidecarOps from './sidecar/document.js';
|
|
68
68
|
|
|
69
|
-
// Notifications: mention batching
|
|
69
|
+
// Notifications: mention batching and the inbox, plus the channels that deliver.
|
|
70
|
+
// `NotificationChannel` is the extension point — subclass it for email or chat.
|
|
70
71
|
export { NotificationCenter } from './notifications/index.js';
|
|
71
|
-
export {
|
|
72
|
+
export { NotificationChannel } from './notifications/channel.js';
|
|
73
|
+
export { WebPushService, WebPushChannel, generateVapidKeys } from './notifications/webpush.js';
|
|
72
74
|
|
|
73
75
|
import { Vfs } from './vfs.js';
|
|
74
76
|
import { MemoryStorage } from './storage/memory.js';
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// A way a notification reaches a person.
|
|
2
|
+
//
|
|
3
|
+
// The inbox is not one of these. NotificationCenter batches mentions, collapses each
|
|
4
|
+
// batch into one notification, and writes it to the user's inbox — that is the durable
|
|
5
|
+
// record, it works with no channel configured at all, and /api/notifications serves it.
|
|
6
|
+
// A channel is the part that goes and TELLS someone: a web push, an email, a message
|
|
7
|
+
// into a chat workspace. Nothing here is allowed to be the only copy of anything.
|
|
8
|
+
//
|
|
9
|
+
// Two constraints worth knowing before writing one:
|
|
10
|
+
//
|
|
11
|
+
// It must be fetch-based. On Workers the drain runs inside a cron slice, where there
|
|
12
|
+
// are no sockets — an email channel has to be an HTTP API (Resend, SES, Postmark),
|
|
13
|
+
// not SMTP. A channel that opens a socket works on a self-hosted drive and fails on
|
|
14
|
+
// the runtime the drive most often ships to.
|
|
15
|
+
//
|
|
16
|
+
// Delivery is best-effort and isolated. A channel that throws is logged and the rest
|
|
17
|
+
// still run; the notification is already in the inbox by then, so a failed send loses
|
|
18
|
+
// the ping and not the notification.
|
|
19
|
+
//
|
|
20
|
+
// Channels may also own routes — the endpoints a client needs to REGISTER with them, of
|
|
21
|
+
// which a VAPID key and a push subscription are the obvious example. Those endpoints
|
|
22
|
+
// live with the channel rather than in the core route table, so the drive's API does
|
|
23
|
+
// not grow a permanent `/api/push/*` whether or not push exists.
|
|
24
|
+
|
|
25
|
+
import { TroveError } from '../errors.js';
|
|
26
|
+
|
|
27
|
+
export class NotificationChannel {
|
|
28
|
+
/**
|
|
29
|
+
* Stable identifier, used in logs and to find a channel among its peers.
|
|
30
|
+
* @returns {string}
|
|
31
|
+
*/
|
|
32
|
+
get id() {
|
|
33
|
+
throw TroveError.unsupported('A NotificationChannel must have an id');
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Deliver one notification to one user. Called once per user per drain, after the
|
|
38
|
+
* notification is already in their inbox.
|
|
39
|
+
*
|
|
40
|
+
* @param {string} userId the principal id
|
|
41
|
+
* @param {object} note the collapsed notification — id, kind, count, items, title
|
|
42
|
+
* @returns {Promise<void>}
|
|
43
|
+
*/
|
|
44
|
+
async deliver(userId, note) { // eslint-disable-line no-unused-vars
|
|
45
|
+
throw TroveError.unsupported(`Channel "${this.id}" cannot deliver`);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Endpoints this channel needs, mounted under the drive's router.
|
|
50
|
+
*
|
|
51
|
+
* Each is `{ method, path, deps, handler }` with the same meaning the router gives
|
|
52
|
+
* them — `deps` names the resources the handler is leased, and the handler receives
|
|
53
|
+
* them alongside `principal`, `req`, `params` and `query`. Returning nothing is the
|
|
54
|
+
* common case: a channel that reads an address off the identity has nothing to
|
|
55
|
+
* register.
|
|
56
|
+
*
|
|
57
|
+
* `helpers` is the request plumbing the router owns and a channel should not
|
|
58
|
+
* reimplement: `body(req)` parses JSON under the server's size cap — the cap is the
|
|
59
|
+
* point, a channel parsing the body itself is an unbounded read — and
|
|
60
|
+
* `requirePrincipal(principal)` throws the same 401 every other route throws.
|
|
61
|
+
*
|
|
62
|
+
* @param {{body: Function, requirePrincipal: Function}} helpers
|
|
63
|
+
* @returns {Array<{method: string, path: string, deps?: string[], handler: Function}>}
|
|
64
|
+
*/
|
|
65
|
+
routes(helpers) { // eslint-disable-line no-unused-vars
|
|
66
|
+
return [];
|
|
67
|
+
}
|
|
68
|
+
}
|
|
@@ -1,17 +1,25 @@
|
|
|
1
|
-
// NotificationCenter — batches @mentions and
|
|
2
|
-
// interval
|
|
3
|
-
//
|
|
4
|
-
// inbox, and
|
|
5
|
-
//
|
|
6
|
-
//
|
|
1
|
+
// NotificationCenter — batches @mentions and drains them on a configurable
|
|
2
|
+
// interval: as conversations mutate, mentions accumulate per user; every
|
|
3
|
+
// `flushIntervalMs` we drain the batch, collapse it into one notification, drop that
|
|
4
|
+
// into the user's inbox, and hand it to each delivery channel. Pending batches and
|
|
5
|
+
// inboxes live in the pluggable KeyValueStore, so this survives restarts and works
|
|
6
|
+
// multi-instance.
|
|
7
7
|
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
8
|
+
// The split worth keeping straight: the INBOX is the record, and it is written whether
|
|
9
|
+
// or not anything can be delivered — /api/notifications serves it and a drive with no
|
|
10
|
+
// channels configured still notifies people perfectly well. A CHANNEL is the part that
|
|
11
|
+
// goes and tells someone, and it is allowed to fail. Web push is one channel (see
|
|
12
|
+
// webpush.js); email or a chat workspace would be others, and nothing here knows the
|
|
13
|
+
// difference between them.
|
|
14
|
+
//
|
|
15
|
+
// Bodyless push means mention text never reaches a third-party push service — the
|
|
16
|
+
// client fetches /api/notifications over its authenticated channel instead. That is a
|
|
17
|
+
// property of the web-push channel rather than of this file, but it is the reason the
|
|
18
|
+
// inbox has to exist independently of delivery.
|
|
11
19
|
|
|
12
|
-
import {
|
|
20
|
+
import { TroveError } from '../errors.js';
|
|
21
|
+
import { WebPushChannel } from './webpush.js';
|
|
13
22
|
|
|
14
|
-
const NS_SUBS = 'push-subs'; // userId -> [subscription]
|
|
15
23
|
const NS_PENDING = 'mentions-pending'; // userId -> [mention]
|
|
16
24
|
const NS_INBOX = 'notifications-inbox'; // userId -> [notification]
|
|
17
25
|
|
|
@@ -19,21 +27,31 @@ export class NotificationCenter {
|
|
|
19
27
|
/**
|
|
20
28
|
* @param {object} deps
|
|
21
29
|
* @param {import('../kv.js').KeyValueStore} deps.kv
|
|
22
|
-
|
|
23
|
-
* @param {import('./webpush.js').WebPushService} [deps.push]
|
|
30
|
+
* @param {import('./channel.js').NotificationChannel[]} [deps.channels] delivery
|
|
31
|
+
* @param {import('./webpush.js').WebPushService} [deps.push] the pre-channel
|
|
32
|
+
* spelling of "web push": a bare service, wrapped into a channel here so there is
|
|
33
|
+
* still exactly one delivery path.
|
|
24
34
|
* @param {number} [deps.flushIntervalMs] default 30s
|
|
25
35
|
* @param {number} [deps.inboxCap] keep at most N per user (default 200)
|
|
26
36
|
*/
|
|
27
|
-
constructor({ kv, push, flushIntervalMs = 30_000, inboxCap = 200 }) {
|
|
37
|
+
constructor({ kv, push, channels, flushIntervalMs = 30_000, inboxCap = 200 }) {
|
|
28
38
|
this.kv = kv;
|
|
29
|
-
this.
|
|
39
|
+
this.channels = [
|
|
40
|
+
...(channels || []),
|
|
41
|
+
...(push ? [new WebPushChannel({ kv, service: push })] : []),
|
|
42
|
+
];
|
|
30
43
|
this.flushIntervalMs = flushIntervalMs;
|
|
31
44
|
this.inboxCap = inboxCap;
|
|
32
45
|
this._timer = null;
|
|
33
46
|
}
|
|
34
47
|
|
|
48
|
+
/** A channel by id, for the callers that need a specific one. */
|
|
49
|
+
channel(id) {
|
|
50
|
+
return this.channels.find((c) => c.id === id) || null;
|
|
51
|
+
}
|
|
52
|
+
|
|
35
53
|
vapidPublicKey() {
|
|
36
|
-
return this.push?.publicKey || null;
|
|
54
|
+
return this.channel('web-push')?.publicKey || null;
|
|
37
55
|
}
|
|
38
56
|
|
|
39
57
|
/** Queue mention events (from SidecarService.onMentions). */
|
|
@@ -45,8 +63,25 @@ import { assertPublicUrl } from '../util.js';
|
|
|
45
63
|
}
|
|
46
64
|
}
|
|
47
65
|
|
|
48
|
-
/**
|
|
66
|
+
/**
|
|
67
|
+
* Drain all pending batches: inbox + push. Returns how many users notified.
|
|
68
|
+
*
|
|
69
|
+
* Concurrent calls collapse onto one drain. There are two callers — the interval
|
|
70
|
+
* timer, and maintenance on a runtime whose timers do not survive a request — and
|
|
71
|
+
* both can be live at once. Two drains reading the same pending batch would deliver
|
|
72
|
+
* it twice, because the read and the delete are not one operation.
|
|
73
|
+
*
|
|
74
|
+
* This makes one PROCESS safe, not a cluster: two servers over a shared KV can still
|
|
75
|
+
* both read a batch before either deletes it. That race predates this and wants a
|
|
76
|
+
* claim in the store to fix properly.
|
|
77
|
+
*/
|
|
49
78
|
async flush(now = Date.now()) {
|
|
79
|
+
if (this._draining) return this._draining;
|
|
80
|
+
this._draining = this.#drain(now).finally(() => { this._draining = null; });
|
|
81
|
+
return this._draining;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
async #drain(now) {
|
|
50
85
|
const pendingUsers = await this.kv.list(NS_PENDING);
|
|
51
86
|
let notified = 0;
|
|
52
87
|
for (const { key: userId, value: mentions } of pendingUsers) {
|
|
@@ -70,47 +105,41 @@ import { assertPublicUrl } from '../util.js';
|
|
|
70
105
|
inbox.unshift(note);
|
|
71
106
|
await this.kv.set(NS_INBOX, userId, inbox.slice(0, this.inboxCap));
|
|
72
107
|
await this.kv.delete(NS_PENDING, userId);
|
|
73
|
-
await this.#
|
|
108
|
+
await this.#deliver(userId, note);
|
|
74
109
|
notified++;
|
|
75
110
|
}
|
|
76
111
|
return notified;
|
|
77
112
|
}
|
|
78
113
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
114
|
+
/**
|
|
115
|
+
* Hand one notification to every channel.
|
|
116
|
+
*
|
|
117
|
+
* Sequential and forgiving: a channel that throws is logged and the others still run.
|
|
118
|
+
* The inbox was written above, so a failed send costs the ping and not the
|
|
119
|
+
* notification — which is the whole reason the inbox is not itself a channel.
|
|
120
|
+
*/
|
|
121
|
+
async #deliver(userId, note) {
|
|
122
|
+
for (const channel of this.channels) {
|
|
84
123
|
try {
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
alive.push(sub); // transient — keep the subscription, retry next flush
|
|
124
|
+
await channel.deliver(userId, note);
|
|
125
|
+
} catch (err) {
|
|
126
|
+
console.error(`[trove] notification channel "${channel.id}" failed for ${userId}`, err);
|
|
89
127
|
}
|
|
90
128
|
}
|
|
91
|
-
if (alive.length !== subs.length) await this.kv.set(NS_SUBS, userId, alive);
|
|
92
129
|
}
|
|
93
130
|
|
|
94
131
|
// --- subscriptions & inbox (called by routes) ------------------------------
|
|
95
132
|
|
|
96
133
|
async subscribePush(userId, subscription) {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
// instance metadata being the obvious target. A real push service is on the public
|
|
101
|
-
// internet, so nothing legitimate is lost by refusing the rest.
|
|
102
|
-
assertPublicUrl(subscription.endpoint, 'Push endpoint');
|
|
103
|
-
const subs = (await this.kv.get(NS_SUBS, userId)) || [];
|
|
104
|
-
if (!subs.some((s) => s.endpoint === subscription.endpoint)) {
|
|
105
|
-
subs.push(subscription);
|
|
106
|
-
await this.kv.set(NS_SUBS, userId, subs);
|
|
107
|
-
}
|
|
108
|
-
return { ok: true };
|
|
134
|
+
const channel = this.channel('web-push');
|
|
135
|
+
if (!channel) throw TroveError.unsupported('Web push is not configured');
|
|
136
|
+
return channel.subscribe(userId, subscription);
|
|
109
137
|
}
|
|
138
|
+
|
|
110
139
|
async unsubscribePush(userId, endpoint) {
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
return
|
|
140
|
+
const channel = this.channel('web-push');
|
|
141
|
+
if (!channel) throw TroveError.unsupported('Web push is not configured');
|
|
142
|
+
return channel.unsubscribe(userId, endpoint);
|
|
114
143
|
}
|
|
115
144
|
|
|
116
145
|
async inbox(userId) {
|
|
@@ -18,6 +18,8 @@
|
|
|
18
18
|
|
|
19
19
|
import { TroveError } from '../errors.js';
|
|
20
20
|
import { withRetry } from '../retry.js';
|
|
21
|
+
import { assertPublicUrl } from '../util.js';
|
|
22
|
+
import { NotificationChannel } from './channel.js';
|
|
21
23
|
|
|
22
24
|
const enc = new TextEncoder();
|
|
23
25
|
|
|
@@ -215,3 +217,111 @@ export class WebPushService {
|
|
|
215
217
|
);
|
|
216
218
|
}
|
|
217
219
|
}
|
|
220
|
+
|
|
221
|
+
// --- as a notification channel --------------------------------------------------
|
|
222
|
+
|
|
223
|
+
const NS_SUBS = 'push-subs'; // userId -> [subscription]
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Web push, as one way of reaching someone.
|
|
227
|
+
*
|
|
228
|
+
* Owns the subscriptions as well as the sending, because they are the same concern:
|
|
229
|
+
* a subscription is a push endpoint and means nothing to any other channel. It also
|
|
230
|
+
* owns the endpoints a browser uses to register one — the VAPID public key and the
|
|
231
|
+
* subscribe/unsubscribe pair — so a drive with no push configured does not answer on
|
|
232
|
+
* `/api/push/*` at all, rather than answering with a null.
|
|
233
|
+
*/
|
|
234
|
+
export class WebPushChannel extends NotificationChannel {
|
|
235
|
+
/**
|
|
236
|
+
* @param {object} o
|
|
237
|
+
* @param {import('../kv.js').KeyValueStore} o.kv where subscriptions live
|
|
238
|
+
* @param {WebPushService} [o.service] a ready service; otherwise built from the keys
|
|
239
|
+
* @param {string} [o.publicKey]
|
|
240
|
+
* @param {string} [o.privateKey]
|
|
241
|
+
* @param {string} [o.subject]
|
|
242
|
+
*/
|
|
243
|
+
constructor({ kv, service, publicKey, privateKey, subject } = {}) {
|
|
244
|
+
super();
|
|
245
|
+
if (!kv) throw TroveError.invalid('WebPushChannel requires a kv store');
|
|
246
|
+
this.kv = kv;
|
|
247
|
+
this.service = service ?? new WebPushService({ publicKey, privateKey, subject });
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
get id() { return 'web-push'; }
|
|
251
|
+
|
|
252
|
+
/** The application server key a browser needs to subscribe. */
|
|
253
|
+
get publicKey() { return this.service.publicKey; }
|
|
254
|
+
|
|
255
|
+
async deliver(userId, note) {
|
|
256
|
+
const subs = (await this.kv.get(NS_SUBS, userId)) || [];
|
|
257
|
+
const alive = [];
|
|
258
|
+
for (const sub of subs) {
|
|
259
|
+
try {
|
|
260
|
+
const res = await this.service.send(sub, { topic: 'mentions', urgency: 'normal' });
|
|
261
|
+
if (!res.gone) alive.push(sub);
|
|
262
|
+
} catch {
|
|
263
|
+
alive.push(sub); // transient — keep the subscription, retry next drain
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
if (alive.length !== subs.length) await this.kv.set(NS_SUBS, userId, alive);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
async subscribe(userId, subscription) {
|
|
270
|
+
if (!subscription?.endpoint) throw TroveError.invalid('Invalid push subscription');
|
|
271
|
+
// The server POSTs to this endpoint, from inside its own network, on every drain.
|
|
272
|
+
// Left unchecked it is a request forgery primitive any user can register — cloud
|
|
273
|
+
// instance metadata being the obvious target. A real push service is on the public
|
|
274
|
+
// internet, so nothing legitimate is lost by refusing the rest.
|
|
275
|
+
assertPublicUrl(subscription.endpoint, 'Push endpoint');
|
|
276
|
+
const subs = (await this.kv.get(NS_SUBS, userId)) || [];
|
|
277
|
+
if (!subs.some((s) => s.endpoint === subscription.endpoint)) {
|
|
278
|
+
subs.push(subscription);
|
|
279
|
+
await this.kv.set(NS_SUBS, userId, subs);
|
|
280
|
+
}
|
|
281
|
+
return { ok: true };
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
async unsubscribe(userId, endpoint) {
|
|
285
|
+
const subs = (await this.kv.get(NS_SUBS, userId)) || [];
|
|
286
|
+
await this.kv.set(NS_SUBS, userId, subs.filter((s) => s.endpoint !== endpoint));
|
|
287
|
+
return { ok: true };
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* The registration endpoints, owned here rather than by the core route table.
|
|
292
|
+
*
|
|
293
|
+
* They exist only when this channel does, which is the improvement over declaring
|
|
294
|
+
* them centrally: a drive with no VAPID keys used to answer `/api/push/vapid` with
|
|
295
|
+
* `{ publicKey: null }` and accept subscriptions it could never send to. Now there is
|
|
296
|
+
* no route at all, and a client that asks gets a 404 meaning what it says.
|
|
297
|
+
*
|
|
298
|
+
* The paths are unchanged, because the shipped web app calls them by name.
|
|
299
|
+
*/
|
|
300
|
+
routes({ body, requirePrincipal }) {
|
|
301
|
+
return [
|
|
302
|
+
{
|
|
303
|
+
method: 'GET',
|
|
304
|
+
path: '/api/push/vapid',
|
|
305
|
+
handler: () => ({ publicKey: this.publicKey }),
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
method: 'POST',
|
|
309
|
+
path: '/api/push/subscribe',
|
|
310
|
+
handler: async ({ principal, req }) => {
|
|
311
|
+
requirePrincipal(principal);
|
|
312
|
+
const b = await body(req);
|
|
313
|
+
return this.subscribe(principal.id, b.subscription);
|
|
314
|
+
},
|
|
315
|
+
},
|
|
316
|
+
{
|
|
317
|
+
method: 'DELETE',
|
|
318
|
+
path: '/api/push/subscribe',
|
|
319
|
+
handler: async ({ principal, req }) => {
|
|
320
|
+
requirePrincipal(principal);
|
|
321
|
+
const b = await body(req);
|
|
322
|
+
return this.unsubscribe(principal.id, b.endpoint);
|
|
323
|
+
},
|
|
324
|
+
},
|
|
325
|
+
];
|
|
326
|
+
}
|
|
327
|
+
}
|
|
@@ -23,6 +23,19 @@ import { SqliteDatabase, SqliteProvider } from './sqlite.js';
|
|
|
23
23
|
// The keys the server co-locates in one database — see LocalSqliteProvider.
|
|
24
24
|
const CORE_KEYS = new Set(['metadata', 'kv', 'plugins', 'search']);
|
|
25
25
|
|
|
26
|
+
// D1 answers `PRAGMA journal_mode = WAL` with SQLITE_AUTH: setting a pragma is not
|
|
27
|
+
// something a D1 client may do. The shared metadata schema opens with two of them, and
|
|
28
|
+
// that runs in SqliteStore.init() — so the first request a Workers drive ever served
|
|
29
|
+
// died, and every one after it, on a statement that is pure local-file housekeeping and
|
|
30
|
+
// means nothing here. Dropped rather than made conditional at the call site: this is the
|
|
31
|
+
// one place that knows it is talking to D1.
|
|
32
|
+
//
|
|
33
|
+
// Only ASSIGNMENTS. `PRAGMA table_info(nodes)` is a query, D1 supports it, and the trash
|
|
34
|
+
// migration reads it to decide whether it has already run — filtering that too would
|
|
35
|
+
// silently re-run migrations.
|
|
36
|
+
const PRAGMA_ASSIGNMENT = /^\s*PRAGMA\s+[\w.]+\s*=/i;
|
|
37
|
+
const notAPragmaAssignment = (statement) => !PRAGMA_ASSIGNMENT.test(statement);
|
|
38
|
+
|
|
26
39
|
/** Split a multi-statement DDL script into individual statements. */
|
|
27
40
|
function splitStatements(sql) {
|
|
28
41
|
// Good enough for the schema DDL Trove ships, which contains no semicolons inside
|
|
@@ -51,7 +64,7 @@ class D1Database extends SqliteDatabase {
|
|
|
51
64
|
// D1's own exec() is documented as slow and unsuitable for anything hot; it is also
|
|
52
65
|
// inconsistent about multi-statement input across versions. Batching prepared
|
|
53
66
|
// statements is both faster and atomic, which is what schema setup wants.
|
|
54
|
-
const statements = splitStatements(sql);
|
|
67
|
+
const statements = splitStatements(sql).filter(notAPragmaAssignment);
|
|
55
68
|
if (!statements.length) return;
|
|
56
69
|
if (statements.length === 1) {
|
|
57
70
|
await this.d1.prepare(statements[0]).run();
|
|
@@ -3,6 +3,18 @@
|
|
|
3
3
|
// expose the same statement API — prepare().run/get/all(...params), exec() for
|
|
4
4
|
// raw/multi-statement SQL, close() — and differ only in the constructor, so
|
|
5
5
|
// callers get one `db` handle that behaves identically under either runtime.
|
|
6
|
+
//
|
|
7
|
+
// Both also load extensions, which is what sqlite-vec needs for a durable vector index:
|
|
8
|
+
// node:sqlite via `allowExtension` at construction, bun:sqlite natively. The one place
|
|
9
|
+
// that is not true is macOS under Bun, where the system libsqlite3 Bun links is built
|
|
10
|
+
// without extension support — see useExtensionCapableSqlite below.
|
|
11
|
+
//
|
|
12
|
+
// better-sqlite3 is deliberately not used here. It would add a native dependency that
|
|
13
|
+
// every consumer installs, including Workers deployments that can never call it, in
|
|
14
|
+
// exchange for a feature set both built-ins already have — and it does not survive the
|
|
15
|
+
// trade: constructing one under Bun 1.2.23 and 1.3.14 on macOS arm64 aborts the process
|
|
16
|
+
// with `NAPI FATAL ERROR`, which is a panic rather than a throw and so cannot even be
|
|
17
|
+
// caught and fallen back from.
|
|
6
18
|
|
|
7
19
|
import { TroveError } from './errors.js';
|
|
8
20
|
|
|
@@ -22,9 +34,69 @@ export async function openDatabase(pathOrMemory = ':memory:') {
|
|
|
22
34
|
return db;
|
|
23
35
|
}
|
|
24
36
|
|
|
37
|
+
// Held in a variable so a bundler cannot read it.
|
|
38
|
+
//
|
|
39
|
+
// `await import('bun:sqlite')` behind a `typeof Bun` guard still fails to BUILD for
|
|
40
|
+
// Workers: esbuild resolves a literal specifier statically, and a guard is a runtime
|
|
41
|
+
// thing that says nothing about what the bundler does with the module graph. The result
|
|
42
|
+
// was a wrangler build that could not link, from an import that would never have run.
|
|
43
|
+
// A non-literal specifier is not resolvable at build time, so it survives as a runtime
|
|
44
|
+
// import — which the guard then never reaches.
|
|
45
|
+
const BUN_SQLITE = 'bun:sqlite';
|
|
46
|
+
|
|
47
|
+
// Where a SQLite that can load extensions tends to live on macOS. Ordered: an explicit
|
|
48
|
+
// setting wins, then Homebrew on Apple Silicon, then Homebrew on Intel.
|
|
49
|
+
const MACOS_SQLITE_CANDIDATES = [
|
|
50
|
+
'/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib',
|
|
51
|
+
'/usr/local/opt/sqlite/lib/libsqlite3.dylib',
|
|
52
|
+
];
|
|
53
|
+
|
|
54
|
+
let customSqliteTried = false;
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Point Bun at a SQLite that can load extensions, on the one platform where it cannot.
|
|
58
|
+
*
|
|
59
|
+
* `bun:sqlite` supports extensions natively — `loadExtension` is right there and works
|
|
60
|
+
* on Linux with nothing special. macOS is the exception, and not because of Bun: the
|
|
61
|
+
* SYSTEM libsqlite3 that Bun links by default is built without
|
|
62
|
+
* SQLITE_ENABLE_LOAD_EXTENSION, so the call comes back "This build of sqlite3 does not
|
|
63
|
+
* support dynamic extension loading". The effect is that sqlite-vec cannot load, and
|
|
64
|
+
* semantic search silently degrades to an in-memory index rebuilt on every restart —
|
|
65
|
+
* on developer laptops specifically, which is where it is least likely to be noticed.
|
|
66
|
+
*
|
|
67
|
+
* `setCustomSQLite` is Bun's documented answer. It has to happen before any database is
|
|
68
|
+
* opened, hence doing it here rather than at the point extensions are wanted.
|
|
69
|
+
*
|
|
70
|
+
* Best-effort throughout: if no capable library is installed we leave Bun on the system
|
|
71
|
+
* one and the sqlite-vec store degrades exactly as it did before, with its own warning.
|
|
72
|
+
* Set TROVE_SQLITE_LIB to override the search.
|
|
73
|
+
*/
|
|
74
|
+
async function useExtensionCapableSqlite(Database) {
|
|
75
|
+
if (customSqliteTried || process.platform !== 'darwin') return;
|
|
76
|
+
customSqliteTried = true;
|
|
77
|
+
if (typeof Database.setCustomSQLite !== 'function') return;
|
|
78
|
+
|
|
79
|
+
const configured = process.env.TROVE_SQLITE_LIB;
|
|
80
|
+
const { existsSync } = await import('node:fs');
|
|
81
|
+
const candidates = configured ? [configured] : MACOS_SQLITE_CANDIDATES;
|
|
82
|
+
for (const lib of candidates) {
|
|
83
|
+
if (!existsSync(lib)) continue;
|
|
84
|
+
try {
|
|
85
|
+
Database.setCustomSQLite(lib);
|
|
86
|
+
return;
|
|
87
|
+
} catch { /* keep looking; the default is still a working database */ }
|
|
88
|
+
}
|
|
89
|
+
// Only worth saying when it was asked for explicitly — otherwise this is the ordinary
|
|
90
|
+
// case of a machine without Homebrew sqlite, and the vector store says its own piece.
|
|
91
|
+
if (configured) {
|
|
92
|
+
console.warn(`[trove] TROVE_SQLITE_LIB=${configured} could not be used; falling back to the system SQLite`);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
25
96
|
async function open(pathOrMemory) {
|
|
26
97
|
if (typeof Bun !== 'undefined') {
|
|
27
|
-
const { Database } = await import(
|
|
98
|
+
const { Database } = await import(BUN_SQLITE);
|
|
99
|
+
await useExtensionCapableSqlite(Database);
|
|
28
100
|
return new Database(pathOrMemory);
|
|
29
101
|
}
|
|
30
102
|
try {
|
|
@@ -149,12 +149,14 @@ export function createTaskHost(getServer) {
|
|
|
149
149
|
* gets the truth without knowing where it lives.
|
|
150
150
|
*/
|
|
151
151
|
export class RemoteTasks extends TaskRegistry {
|
|
152
|
+
/** @param {() => object} stub resolves the Durable Object stub — see remoteBackground
|
|
153
|
+
* for why this is a function and not the stub itself. */
|
|
152
154
|
constructor(stub) {
|
|
153
155
|
super();
|
|
154
156
|
this.stub = stub;
|
|
155
157
|
}
|
|
156
158
|
async #rpc(path, payload) {
|
|
157
|
-
const res = await this.stub.fetch(`https://trove.tasks${path}`, {
|
|
159
|
+
const res = await this.stub().fetch(`https://trove.tasks${path}`, {
|
|
158
160
|
method: 'POST',
|
|
159
161
|
headers: { 'content-type': 'application/json' },
|
|
160
162
|
body: JSON.stringify(payload || {}),
|
|
@@ -181,8 +183,16 @@ export function remoteBackground(namespace) {
|
|
|
181
183
|
// One instance for the whole drive, by name. Tasks are few and long, so there is no
|
|
182
184
|
// throughput argument for sharding — and one instance is what makes GET /api/tasks a
|
|
183
185
|
// complete answer rather than a per-shard sample.
|
|
184
|
-
|
|
185
|
-
|
|
186
|
+
//
|
|
187
|
+
// Resolved per call, never held. A stub belongs to the I/O context of the request that
|
|
188
|
+
// created it, and the server that owns this one is cached at module scope for the life
|
|
189
|
+
// of the isolate — so the stub outlived its request and every later use threw. The
|
|
190
|
+
// first request worked and the second did not, which reads like a fluke and is not:
|
|
191
|
+
// GET /api/tasks was broken for the entire life of every isolate after its first
|
|
192
|
+
// request. `idFromName` is a pure hash, so re-deriving it costs nothing and always
|
|
193
|
+
// names the same object.
|
|
194
|
+
const stub = () => namespace.get(namespace.idFromName('trove-tasks'));
|
|
195
|
+
const begin = (payload) => stub()
|
|
186
196
|
.fetch('https://trove.tasks/begin', {
|
|
187
197
|
method: 'POST',
|
|
188
198
|
headers: { 'content-type': 'application/json' },
|
|
@@ -195,7 +205,7 @@ export function remoteBackground(namespace) {
|
|
|
195
205
|
beginScan: (collectionId, { reason } = {}) => begin({ kind: 'scan', collectionId, reason }),
|
|
196
206
|
beginReindex: ({ reason } = {}) => begin({ kind: 'index', reason }),
|
|
197
207
|
},
|
|
198
|
-
maintain: (budgetMs) => stub
|
|
208
|
+
maintain: (budgetMs) => stub()
|
|
199
209
|
.fetch('https://trove.tasks/maintain', {
|
|
200
210
|
method: 'POST',
|
|
201
211
|
headers: { 'content-type': 'application/json' },
|
|
@@ -53,7 +53,13 @@ async function getServer(env, buildVfs, { delegate = true } = {}) {
|
|
|
53
53
|
config.metadata = { driver: 'sqlite' };
|
|
54
54
|
}
|
|
55
55
|
// Cloudflare Vectorize binding → first-class vector store (no REST creds needed).
|
|
56
|
-
|
|
56
|
+
//
|
|
57
|
+
// An explicit TROVE_VECTOR wins. The binding used to be taken unconditionally, which
|
|
58
|
+
// left no way to opt out: `wrangler dev` against a real Vectorize index is the only
|
|
59
|
+
// shape a local run could have, and setting TROVE_VECTOR=memory in .dev.vars did
|
|
60
|
+
// nothing. Naming a store and being given a different one is the kind of override that
|
|
61
|
+
// should never be silent.
|
|
62
|
+
if (env.VECTORIZE && (!env.TROVE_VECTOR || env.TROVE_VECTOR === 'vectorize')) {
|
|
57
63
|
config.vectorStore = { driver: 'vectorize', binding: env.VECTORIZE };
|
|
58
64
|
}
|
|
59
65
|
// Cloudflare Workers AI binding → LLM-assisted search transformer (human text →
|
|
@@ -32,7 +32,7 @@ import {
|
|
|
32
32
|
cloudflareAccess,
|
|
33
33
|
KeyValueStore, MemoryKV, SqliteKV,
|
|
34
34
|
SqliteProvider, LocalSqliteProvider,
|
|
35
|
-
SidecarService, NotificationCenter, WebPushService,
|
|
35
|
+
SidecarService, NotificationCenter, WebPushService, WebPushChannel, NotificationChannel,
|
|
36
36
|
CollectionService,
|
|
37
37
|
PluginService, PackageStore, StoragePackageStore, SqlitePluginInstallStore,
|
|
38
38
|
IndexerRuntime, InProcessIndexerRuntime, PluginIndexers,
|
|
@@ -285,17 +285,44 @@ export function coreProviders(config, lifecycleState) {
|
|
|
285
285
|
})
|
|
286
286
|
: null))),
|
|
287
287
|
|
|
288
|
-
notifications
|
|
288
|
+
// How notifications actually reach people. A list, because there is no reason for
|
|
289
|
+
// it to be one: a drive can push to browsers and mail a digest and post into a chat
|
|
290
|
+
// workspace, and none of those knows about the others. Web push is the default and
|
|
291
|
+
// only when VAPID is configured, so a drive that sets nothing gets an inbox and no
|
|
292
|
+
// delivery — which is what it got before.
|
|
293
|
+
//
|
|
294
|
+
// `config.notificationChannels` replaces the list wholesale rather than adding to
|
|
295
|
+
// it, so a caller who wants email INSTEAD of push says so by saying so.
|
|
296
|
+
notificationChannels: Provider.fromLazySingleton(
|
|
289
297
|
async (deps) => {
|
|
290
298
|
const { kv, push } = await need(deps, ['kv', 'push']);
|
|
291
|
-
|
|
299
|
+
if (config.notificationChannels) {
|
|
300
|
+
return config.notificationChannels.filter(Boolean).map((c) => {
|
|
301
|
+
if (!(c instanceof NotificationChannel)) {
|
|
302
|
+
throw TroveError.invalid('Every notificationChannels entry must be a NotificationChannel');
|
|
303
|
+
}
|
|
304
|
+
return c;
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
return push ? [new WebPushChannel({ kv, service: push })] : [];
|
|
308
|
+
},
|
|
309
|
+
null,
|
|
310
|
+
{ deps: ['kv', 'push'] },
|
|
311
|
+
),
|
|
312
|
+
|
|
313
|
+
notifications: Provider.fromLazySingleton(
|
|
314
|
+
async (deps) => {
|
|
315
|
+
const { kv, notificationChannels } = await need(deps, ['kv', 'notificationChannels']);
|
|
316
|
+
const center = new NotificationCenter({
|
|
317
|
+
kv, channels: notificationChannels, flushIntervalMs: config.mentionFlushMs ?? 30_000,
|
|
318
|
+
});
|
|
292
319
|
if (config.startFlusher !== false) center.start();
|
|
293
320
|
return center;
|
|
294
321
|
},
|
|
295
322
|
// Stopping the flusher used to be a line in close() that had to remember this
|
|
296
323
|
// existed. Now it is attached to the thing it stops.
|
|
297
324
|
(center) => center.stop(),
|
|
298
|
-
{ deps: ['kv', '
|
|
325
|
+
{ deps: ['kv', 'notificationChannels'] },
|
|
299
326
|
),
|
|
300
327
|
|
|
301
328
|
sidecar: Provider.fromLazySingleton(
|
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
accessHost, TroveError,
|
|
17
17
|
protectedResourceMetadata, challengeHeaders, publicOrigin,
|
|
18
18
|
} from '@3sln/trove/core';
|
|
19
|
-
import { createRouter } from './routes.js';
|
|
19
|
+
import { createRouter, routeHelpers } from './routes.js';
|
|
20
20
|
import { createDriveEngine, scanStarter, BACKBONE } from './engine/index.js';
|
|
21
21
|
import { createMcpHandler } from './mcp/index.js';
|
|
22
22
|
import { cacheControlFor } from './cachePolicy.js';
|
|
@@ -210,6 +210,17 @@ export async function createServer(config = {}) {
|
|
|
210
210
|
|
|
211
211
|
const router = createRouter();
|
|
212
212
|
|
|
213
|
+
// Routes contributed by delivery channels — the endpoints a client uses to REGISTER
|
|
214
|
+
// with one, of which a VAPID key and a push subscription are the obvious example.
|
|
215
|
+
// Mounted here rather than declared in routes.js so the drive's API reflects what is
|
|
216
|
+
// actually configured: no web push, no /api/push/*. Added after the core table, so a
|
|
217
|
+
// channel cannot shadow a built-in route by claiming its path.
|
|
218
|
+
for (const channel of notifications?.channels || []) {
|
|
219
|
+
for (const route of channel.routes?.(routeHelpers) || []) {
|
|
220
|
+
router.add(route.method, route.path, route.deps || [], route.handler);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
213
224
|
// Said at boot, because that is when someone is looking and can still fix it. The
|
|
214
225
|
// alternative is discovering it from a client that can't sign in and a 401 that
|
|
215
226
|
// doesn't say why.
|
|
@@ -341,9 +352,19 @@ export async function createServer(config = {}) {
|
|
|
341
352
|
* does, so it does as much as it can and stores where it got to.
|
|
342
353
|
*/
|
|
343
354
|
async function runMaintenance({ budgetMs = 20_000, scan = true } = {}) {
|
|
344
|
-
const out = { swept: false, purged: 0, scans: [] };
|
|
355
|
+
const out = { swept: false, purged: 0, scans: [], notified: 0 };
|
|
345
356
|
await vfs.uploads.sweepExpired(Date.now());
|
|
346
357
|
await sidecar.sweep();
|
|
358
|
+
// Mentions are batched and drained on an interval — a timer, and a timer registered
|
|
359
|
+
// during a request does not outlive it on Workers, where the adapter switches the
|
|
360
|
+
// flusher off for exactly that reason. Nothing else called flush, so on that runtime
|
|
361
|
+
// mentions piled up in the pending store and were never delivered at all: no inbox
|
|
362
|
+
// entry, no push, no error. Maintenance runs from a cron there, which is the one
|
|
363
|
+
// thing that does fire. Harmless where the timer works — concurrent drains collapse.
|
|
364
|
+
out.notified = await notifications.flush().catch((e) => {
|
|
365
|
+
console.error('[trove] mention flush failed', e);
|
|
366
|
+
return 0;
|
|
367
|
+
});
|
|
347
368
|
const trashMs = (config.trashRetentionDays ?? 30) * 86400_000;
|
|
348
369
|
if (trashMs > 0) out.purged = (await vfs.purgeTrash({ before: Date.now() - trashMs }))?.purged || 0;
|
|
349
370
|
out.swept = true;
|
|
@@ -442,6 +442,14 @@ export function createRouter() {
|
|
|
442
442
|
|
|
443
443
|
// --- search & indexing -----------------------------------------------------
|
|
444
444
|
|
|
445
|
+
// Raw search: the query string is passed through as text, with NO transformer.
|
|
446
|
+
//
|
|
447
|
+
// Which means the `#tag` grammar does not apply here. `/api/capabilities` advertises a
|
|
448
|
+
// searchPrompt telling people `#tag` narrows by tag, and against this endpoint that
|
|
449
|
+
// degrades into a text search for the literal string "#tag" — no error, just quietly
|
|
450
|
+
// different results. POST /api/query is the one that runs the transformer and is what
|
|
451
|
+
// the workbench uses; this stays as the lower-level endpoint for callers that have
|
|
452
|
+
// already resolved their own query.
|
|
445
453
|
r.get('/api/search', ['collections', 'vfs'], async (ctx) => {
|
|
446
454
|
const { vfs, query } = ctx;
|
|
447
455
|
if (!query.q) throw TroveError.invalid('q is required');
|
|
@@ -748,20 +756,10 @@ export function createRouter() {
|
|
|
748
756
|
return notifications.markRead(principal.id, b.ids);
|
|
749
757
|
});
|
|
750
758
|
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
requirePrincipal(principal);
|
|
756
|
-
const b = await body(req);
|
|
757
|
-
return notifications.subscribePush(principal.id, b.subscription);
|
|
758
|
-
});
|
|
759
|
-
r.delete('/api/push/subscribe', ['notifications'], async ({ notifications, principal, req }) => {
|
|
760
|
-
requireNotifications(notifications);
|
|
761
|
-
requirePrincipal(principal);
|
|
762
|
-
const b = await body(req);
|
|
763
|
-
return notifications.unsubscribePush(principal.id, b.endpoint);
|
|
764
|
-
});
|
|
759
|
+
// /api/push/* is not here. Registering with a delivery channel is the channel's own
|
|
760
|
+
// business — see WebPushChannel.routes() — so the drive's route table does not carry
|
|
761
|
+
// endpoints for a transport it may not have configured, and adding email or chat does
|
|
762
|
+
// not mean editing this file.
|
|
765
763
|
|
|
766
764
|
// --- plugins: domain verification proxy + per-plugin server storage --------
|
|
767
765
|
|
|
@@ -1064,3 +1062,12 @@ function requirePrincipal(principal) {
|
|
|
1064
1062
|
function requireNotifications(n) {
|
|
1065
1063
|
if (!n) throw TroveError.unsupported('Notifications are not enabled on this server');
|
|
1066
1064
|
}
|
|
1065
|
+
|
|
1066
|
+
/**
|
|
1067
|
+
* The request plumbing handed to routes contributed from outside this file.
|
|
1068
|
+
*
|
|
1069
|
+
* `body` is the capped JSON read — the cap is the reason to share it rather than let a
|
|
1070
|
+
* channel call `req.json()` and accept an unbounded body — and `requirePrincipal` is
|
|
1071
|
+
* the same 401 every route here throws.
|
|
1072
|
+
*/
|
|
1073
|
+
export const routeHelpers = { body, requirePrincipal };
|