@3sln/trove 0.0.3 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/core/src/apiKeys.js +326 -0
- package/packages/core/src/collections/index.js +83 -13
- package/packages/core/src/index.js +18 -4
- package/packages/core/src/issues.js +4 -0
- package/packages/core/src/notifications/channel.js +68 -0
- package/packages/core/src/notifications/index.js +72 -43
- package/packages/core/src/notifications/webpush.js +110 -0
- package/packages/core/src/sqlite-d1.js +14 -1
- package/packages/core/src/sqlite-driver.js +73 -1
- package/packages/core/src/storage/diagnose.js +234 -0
- package/packages/core/src/storage/drivers.js +74 -0
- package/packages/core/src/storage/filesystem.js +22 -0
- package/packages/core/src/storage/registry.js +129 -0
- package/packages/server/src/adapters/bun.js +6 -0
- package/packages/server/src/adapters/node.js +6 -0
- package/packages/server/src/adapters/worker-tasks.js +14 -4
- package/packages/server/src/adapters/worker.js +7 -1
- package/packages/server/src/engine/index.js +1 -1
- package/packages/server/src/engine/providers/access.js +47 -5
- package/packages/server/src/engine/providers/core.js +109 -16
- package/packages/server/src/index.js +137 -12
- package/packages/server/src/mcp/tools.js +40 -8
- package/packages/server/src/router.js +1 -1
- package/packages/server/src/routes.js +156 -46
- package/packages/server/src/scope.js +2 -2
- package/packages/web/dist/assets/main-f0f2tfhp.js +356 -0
- package/packages/web/dist/assets/{main-4cxs7prw.js.map → main-f0f2tfhp.js.map} +17 -16
- package/packages/web/dist/assets/{styles-kcx1x337.css → styles-d3cyysgp.css} +1 -1
- package/packages/web/dist/index.html +2 -2
- package/packages/web/dist/sw.js +58 -9
- package/packages/web/src/bl/actions.js +112 -13
- package/packages/web/src/bl/activity.js +32 -0
- package/packages/web/src/bl/commands.js +41 -2
- package/packages/web/src/bl/index.js +9 -4
- package/packages/web/src/bl/services.js +78 -1
- package/packages/web/src/platform/api.js +57 -14
- package/packages/web/src/platform/pluginRpc.js +7 -4
- package/packages/web/src/styles.css +137 -0
- package/packages/web/src/ui/components/activityPanel.js +28 -1
- package/packages/web/src/ui/components/collectionGate.js +81 -0
- package/packages/web/src/ui/components/overlays.js +64 -34
- package/packages/web/src/ui/components/phoneChrome.js +2 -2
- package/packages/web/src/ui/components/settingsView.js +197 -1
- package/packages/web/src/ui/components/statusBar.js +19 -2
- package/packages/web/src/ui/compositions/workbench.js +9 -2
- package/packages/web/dist/assets/main-4cxs7prw.js +0 -356
|
@@ -1,17 +1,25 @@
|
|
|
1
|
-
// NotificationCenter — batches @mentions and
|
|
2
|
-
// interval
|
|
3
|
-
//
|
|
4
|
-
// inbox, and
|
|
5
|
-
//
|
|
6
|
-
//
|
|
1
|
+
// NotificationCenter — batches @mentions and drains them on a configurable
|
|
2
|
+
// interval: as conversations mutate, mentions accumulate per user; every
|
|
3
|
+
// `flushIntervalMs` we drain the batch, collapse it into one notification, drop that
|
|
4
|
+
// into the user's inbox, and hand it to each delivery channel. Pending batches and
|
|
5
|
+
// inboxes live in the pluggable KeyValueStore, so this survives restarts and works
|
|
6
|
+
// multi-instance.
|
|
7
7
|
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
8
|
+
// The split worth keeping straight: the INBOX is the record, and it is written whether
|
|
9
|
+
// or not anything can be delivered — /api/notifications serves it and a drive with no
|
|
10
|
+
// channels configured still notifies people perfectly well. A CHANNEL is the part that
|
|
11
|
+
// goes and tells someone, and it is allowed to fail. Web push is one channel (see
|
|
12
|
+
// webpush.js); email or a chat workspace would be others, and nothing here knows the
|
|
13
|
+
// difference between them.
|
|
14
|
+
//
|
|
15
|
+
// Bodyless push means mention text never reaches a third-party push service — the
|
|
16
|
+
// client fetches /api/notifications over its authenticated channel instead. That is a
|
|
17
|
+
// property of the web-push channel rather than of this file, but it is the reason the
|
|
18
|
+
// inbox has to exist independently of delivery.
|
|
11
19
|
|
|
12
|
-
import {
|
|
20
|
+
import { TroveError } from '../errors.js';
|
|
21
|
+
import { WebPushChannel } from './webpush.js';
|
|
13
22
|
|
|
14
|
-
const NS_SUBS = 'push-subs'; // userId -> [subscription]
|
|
15
23
|
const NS_PENDING = 'mentions-pending'; // userId -> [mention]
|
|
16
24
|
const NS_INBOX = 'notifications-inbox'; // userId -> [notification]
|
|
17
25
|
|
|
@@ -19,21 +27,31 @@ export class NotificationCenter {
|
|
|
19
27
|
/**
|
|
20
28
|
* @param {object} deps
|
|
21
29
|
* @param {import('../kv.js').KeyValueStore} deps.kv
|
|
22
|
-
|
|
23
|
-
* @param {import('./webpush.js').WebPushService} [deps.push]
|
|
30
|
+
* @param {import('./channel.js').NotificationChannel[]} [deps.channels] delivery
|
|
31
|
+
* @param {import('./webpush.js').WebPushService} [deps.push] the pre-channel
|
|
32
|
+
* spelling of "web push": a bare service, wrapped into a channel here so there is
|
|
33
|
+
* still exactly one delivery path.
|
|
24
34
|
* @param {number} [deps.flushIntervalMs] default 30s
|
|
25
35
|
* @param {number} [deps.inboxCap] keep at most N per user (default 200)
|
|
26
36
|
*/
|
|
27
|
-
constructor({ kv, push, flushIntervalMs = 30_000, inboxCap = 200 }) {
|
|
37
|
+
constructor({ kv, push, channels, flushIntervalMs = 30_000, inboxCap = 200 }) {
|
|
28
38
|
this.kv = kv;
|
|
29
|
-
this.
|
|
39
|
+
this.channels = [
|
|
40
|
+
...(channels || []),
|
|
41
|
+
...(push ? [new WebPushChannel({ kv, service: push })] : []),
|
|
42
|
+
];
|
|
30
43
|
this.flushIntervalMs = flushIntervalMs;
|
|
31
44
|
this.inboxCap = inboxCap;
|
|
32
45
|
this._timer = null;
|
|
33
46
|
}
|
|
34
47
|
|
|
48
|
+
/** A channel by id, for the callers that need a specific one. */
|
|
49
|
+
channel(id) {
|
|
50
|
+
return this.channels.find((c) => c.id === id) || null;
|
|
51
|
+
}
|
|
52
|
+
|
|
35
53
|
vapidPublicKey() {
|
|
36
|
-
return this.push?.publicKey || null;
|
|
54
|
+
return this.channel('web-push')?.publicKey || null;
|
|
37
55
|
}
|
|
38
56
|
|
|
39
57
|
/** Queue mention events (from SidecarService.onMentions). */
|
|
@@ -45,8 +63,25 @@ import { assertPublicUrl } from '../util.js';
|
|
|
45
63
|
}
|
|
46
64
|
}
|
|
47
65
|
|
|
48
|
-
/**
|
|
66
|
+
/**
|
|
67
|
+
* Drain all pending batches: inbox + push. Returns how many users notified.
|
|
68
|
+
*
|
|
69
|
+
* Concurrent calls collapse onto one drain. There are two callers — the interval
|
|
70
|
+
* timer, and maintenance on a runtime whose timers do not survive a request — and
|
|
71
|
+
* both can be live at once. Two drains reading the same pending batch would deliver
|
|
72
|
+
* it twice, because the read and the delete are not one operation.
|
|
73
|
+
*
|
|
74
|
+
* This makes one PROCESS safe, not a cluster: two servers over a shared KV can still
|
|
75
|
+
* both read a batch before either deletes it. That race predates this and wants a
|
|
76
|
+
* claim in the store to fix properly.
|
|
77
|
+
*/
|
|
49
78
|
async flush(now = Date.now()) {
|
|
79
|
+
if (this._draining) return this._draining;
|
|
80
|
+
this._draining = this.#drain(now).finally(() => { this._draining = null; });
|
|
81
|
+
return this._draining;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
async #drain(now) {
|
|
50
85
|
const pendingUsers = await this.kv.list(NS_PENDING);
|
|
51
86
|
let notified = 0;
|
|
52
87
|
for (const { key: userId, value: mentions } of pendingUsers) {
|
|
@@ -70,47 +105,41 @@ import { assertPublicUrl } from '../util.js';
|
|
|
70
105
|
inbox.unshift(note);
|
|
71
106
|
await this.kv.set(NS_INBOX, userId, inbox.slice(0, this.inboxCap));
|
|
72
107
|
await this.kv.delete(NS_PENDING, userId);
|
|
73
|
-
await this.#
|
|
108
|
+
await this.#deliver(userId, note);
|
|
74
109
|
notified++;
|
|
75
110
|
}
|
|
76
111
|
return notified;
|
|
77
112
|
}
|
|
78
113
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
114
|
+
/**
|
|
115
|
+
* Hand one notification to every channel.
|
|
116
|
+
*
|
|
117
|
+
* Sequential and forgiving: a channel that throws is logged and the others still run.
|
|
118
|
+
* The inbox was written above, so a failed send costs the ping and not the
|
|
119
|
+
* notification — which is the whole reason the inbox is not itself a channel.
|
|
120
|
+
*/
|
|
121
|
+
async #deliver(userId, note) {
|
|
122
|
+
for (const channel of this.channels) {
|
|
84
123
|
try {
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
alive.push(sub); // transient — keep the subscription, retry next flush
|
|
124
|
+
await channel.deliver(userId, note);
|
|
125
|
+
} catch (err) {
|
|
126
|
+
console.error(`[trove] notification channel "${channel.id}" failed for ${userId}`, err);
|
|
89
127
|
}
|
|
90
128
|
}
|
|
91
|
-
if (alive.length !== subs.length) await this.kv.set(NS_SUBS, userId, alive);
|
|
92
129
|
}
|
|
93
130
|
|
|
94
131
|
// --- subscriptions & inbox (called by routes) ------------------------------
|
|
95
132
|
|
|
96
133
|
async subscribePush(userId, subscription) {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
// instance metadata being the obvious target. A real push service is on the public
|
|
101
|
-
// internet, so nothing legitimate is lost by refusing the rest.
|
|
102
|
-
assertPublicUrl(subscription.endpoint, 'Push endpoint');
|
|
103
|
-
const subs = (await this.kv.get(NS_SUBS, userId)) || [];
|
|
104
|
-
if (!subs.some((s) => s.endpoint === subscription.endpoint)) {
|
|
105
|
-
subs.push(subscription);
|
|
106
|
-
await this.kv.set(NS_SUBS, userId, subs);
|
|
107
|
-
}
|
|
108
|
-
return { ok: true };
|
|
134
|
+
const channel = this.channel('web-push');
|
|
135
|
+
if (!channel) throw TroveError.unsupported('Web push is not configured');
|
|
136
|
+
return channel.subscribe(userId, subscription);
|
|
109
137
|
}
|
|
138
|
+
|
|
110
139
|
async unsubscribePush(userId, endpoint) {
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
return
|
|
140
|
+
const channel = this.channel('web-push');
|
|
141
|
+
if (!channel) throw TroveError.unsupported('Web push is not configured');
|
|
142
|
+
return channel.unsubscribe(userId, endpoint);
|
|
114
143
|
}
|
|
115
144
|
|
|
116
145
|
async inbox(userId) {
|
|
@@ -18,6 +18,8 @@
|
|
|
18
18
|
|
|
19
19
|
import { TroveError } from '../errors.js';
|
|
20
20
|
import { withRetry } from '../retry.js';
|
|
21
|
+
import { assertPublicUrl } from '../util.js';
|
|
22
|
+
import { NotificationChannel } from './channel.js';
|
|
21
23
|
|
|
22
24
|
const enc = new TextEncoder();
|
|
23
25
|
|
|
@@ -215,3 +217,111 @@ export class WebPushService {
|
|
|
215
217
|
);
|
|
216
218
|
}
|
|
217
219
|
}
|
|
220
|
+
|
|
221
|
+
// --- as a notification channel --------------------------------------------------
|
|
222
|
+
|
|
223
|
+
const NS_SUBS = 'push-subs'; // userId -> [subscription]
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Web push, as one way of reaching someone.
|
|
227
|
+
*
|
|
228
|
+
* Owns the subscriptions as well as the sending, because they are the same concern:
|
|
229
|
+
* a subscription is a push endpoint and means nothing to any other channel. It also
|
|
230
|
+
* owns the endpoints a browser uses to register one — the VAPID public key and the
|
|
231
|
+
* subscribe/unsubscribe pair — so a drive with no push configured does not answer on
|
|
232
|
+
* `/api/push/*` at all, rather than answering with a null.
|
|
233
|
+
*/
|
|
234
|
+
export class WebPushChannel extends NotificationChannel {
|
|
235
|
+
/**
|
|
236
|
+
* @param {object} o
|
|
237
|
+
* @param {import('../kv.js').KeyValueStore} o.kv where subscriptions live
|
|
238
|
+
* @param {WebPushService} [o.service] a ready service; otherwise built from the keys
|
|
239
|
+
* @param {string} [o.publicKey]
|
|
240
|
+
* @param {string} [o.privateKey]
|
|
241
|
+
* @param {string} [o.subject]
|
|
242
|
+
*/
|
|
243
|
+
constructor({ kv, service, publicKey, privateKey, subject } = {}) {
|
|
244
|
+
super();
|
|
245
|
+
if (!kv) throw TroveError.invalid('WebPushChannel requires a kv store');
|
|
246
|
+
this.kv = kv;
|
|
247
|
+
this.service = service ?? new WebPushService({ publicKey, privateKey, subject });
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
get id() { return 'web-push'; }
|
|
251
|
+
|
|
252
|
+
/** The application server key a browser needs to subscribe. */
|
|
253
|
+
get publicKey() { return this.service.publicKey; }
|
|
254
|
+
|
|
255
|
+
async deliver(userId, note) {
|
|
256
|
+
const subs = (await this.kv.get(NS_SUBS, userId)) || [];
|
|
257
|
+
const alive = [];
|
|
258
|
+
for (const sub of subs) {
|
|
259
|
+
try {
|
|
260
|
+
const res = await this.service.send(sub, { topic: 'mentions', urgency: 'normal' });
|
|
261
|
+
if (!res.gone) alive.push(sub);
|
|
262
|
+
} catch {
|
|
263
|
+
alive.push(sub); // transient — keep the subscription, retry next drain
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
if (alive.length !== subs.length) await this.kv.set(NS_SUBS, userId, alive);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
async subscribe(userId, subscription) {
|
|
270
|
+
if (!subscription?.endpoint) throw TroveError.invalid('Invalid push subscription');
|
|
271
|
+
// The server POSTs to this endpoint, from inside its own network, on every drain.
|
|
272
|
+
// Left unchecked it is a request forgery primitive any user can register — cloud
|
|
273
|
+
// instance metadata being the obvious target. A real push service is on the public
|
|
274
|
+
// internet, so nothing legitimate is lost by refusing the rest.
|
|
275
|
+
assertPublicUrl(subscription.endpoint, 'Push endpoint');
|
|
276
|
+
const subs = (await this.kv.get(NS_SUBS, userId)) || [];
|
|
277
|
+
if (!subs.some((s) => s.endpoint === subscription.endpoint)) {
|
|
278
|
+
subs.push(subscription);
|
|
279
|
+
await this.kv.set(NS_SUBS, userId, subs);
|
|
280
|
+
}
|
|
281
|
+
return { ok: true };
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
async unsubscribe(userId, endpoint) {
|
|
285
|
+
const subs = (await this.kv.get(NS_SUBS, userId)) || [];
|
|
286
|
+
await this.kv.set(NS_SUBS, userId, subs.filter((s) => s.endpoint !== endpoint));
|
|
287
|
+
return { ok: true };
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* The registration endpoints, owned here rather than by the core route table.
|
|
292
|
+
*
|
|
293
|
+
* They exist only when this channel does, which is the improvement over declaring
|
|
294
|
+
* them centrally: a drive with no VAPID keys used to answer `/api/push/vapid` with
|
|
295
|
+
* `{ publicKey: null }` and accept subscriptions it could never send to. Now there is
|
|
296
|
+
* no route at all, and a client that asks gets a 404 meaning what it says.
|
|
297
|
+
*
|
|
298
|
+
* The paths are unchanged, because the shipped web app calls them by name.
|
|
299
|
+
*/
|
|
300
|
+
routes({ body, requirePrincipal }) {
|
|
301
|
+
return [
|
|
302
|
+
{
|
|
303
|
+
method: 'GET',
|
|
304
|
+
path: '/api/push/vapid',
|
|
305
|
+
handler: () => ({ publicKey: this.publicKey }),
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
method: 'POST',
|
|
309
|
+
path: '/api/push/subscribe',
|
|
310
|
+
handler: async ({ principal, req }) => {
|
|
311
|
+
requirePrincipal(principal);
|
|
312
|
+
const b = await body(req);
|
|
313
|
+
return this.subscribe(principal.id, b.subscription);
|
|
314
|
+
},
|
|
315
|
+
},
|
|
316
|
+
{
|
|
317
|
+
method: 'DELETE',
|
|
318
|
+
path: '/api/push/subscribe',
|
|
319
|
+
handler: async ({ principal, req }) => {
|
|
320
|
+
requirePrincipal(principal);
|
|
321
|
+
const b = await body(req);
|
|
322
|
+
return this.unsubscribe(principal.id, b.endpoint);
|
|
323
|
+
},
|
|
324
|
+
},
|
|
325
|
+
];
|
|
326
|
+
}
|
|
327
|
+
}
|
|
@@ -23,6 +23,19 @@ import { SqliteDatabase, SqliteProvider } from './sqlite.js';
|
|
|
23
23
|
// The keys the server co-locates in one database — see LocalSqliteProvider.
|
|
24
24
|
const CORE_KEYS = new Set(['metadata', 'kv', 'plugins', 'search']);
|
|
25
25
|
|
|
26
|
+
// D1 answers `PRAGMA journal_mode = WAL` with SQLITE_AUTH: setting a pragma is not
|
|
27
|
+
// something a D1 client may do. The shared metadata schema opens with two of them, and
|
|
28
|
+
// that runs in SqliteStore.init() — so the first request a Workers drive ever served
|
|
29
|
+
// died, and every one after it, on a statement that is pure local-file housekeeping and
|
|
30
|
+
// means nothing here. Dropped rather than made conditional at the call site: this is the
|
|
31
|
+
// one place that knows it is talking to D1.
|
|
32
|
+
//
|
|
33
|
+
// Only ASSIGNMENTS. `PRAGMA table_info(nodes)` is a query, D1 supports it, and the trash
|
|
34
|
+
// migration reads it to decide whether it has already run — filtering that too would
|
|
35
|
+
// silently re-run migrations.
|
|
36
|
+
const PRAGMA_ASSIGNMENT = /^\s*PRAGMA\s+[\w.]+\s*=/i;
|
|
37
|
+
const notAPragmaAssignment = (statement) => !PRAGMA_ASSIGNMENT.test(statement);
|
|
38
|
+
|
|
26
39
|
/** Split a multi-statement DDL script into individual statements. */
|
|
27
40
|
function splitStatements(sql) {
|
|
28
41
|
// Good enough for the schema DDL Trove ships, which contains no semicolons inside
|
|
@@ -51,7 +64,7 @@ class D1Database extends SqliteDatabase {
|
|
|
51
64
|
// D1's own exec() is documented as slow and unsuitable for anything hot; it is also
|
|
52
65
|
// inconsistent about multi-statement input across versions. Batching prepared
|
|
53
66
|
// statements is both faster and atomic, which is what schema setup wants.
|
|
54
|
-
const statements = splitStatements(sql);
|
|
67
|
+
const statements = splitStatements(sql).filter(notAPragmaAssignment);
|
|
55
68
|
if (!statements.length) return;
|
|
56
69
|
if (statements.length === 1) {
|
|
57
70
|
await this.d1.prepare(statements[0]).run();
|
|
@@ -3,6 +3,18 @@
|
|
|
3
3
|
// expose the same statement API — prepare().run/get/all(...params), exec() for
|
|
4
4
|
// raw/multi-statement SQL, close() — and differ only in the constructor, so
|
|
5
5
|
// callers get one `db` handle that behaves identically under either runtime.
|
|
6
|
+
//
|
|
7
|
+
// Both also load extensions, which is what sqlite-vec needs for a durable vector index:
|
|
8
|
+
// node:sqlite via `allowExtension` at construction, bun:sqlite natively. The one place
|
|
9
|
+
// that is not true is macOS under Bun, where the system libsqlite3 Bun links is built
|
|
10
|
+
// without extension support — see useExtensionCapableSqlite below.
|
|
11
|
+
//
|
|
12
|
+
// better-sqlite3 is deliberately not used here. It would add a native dependency that
|
|
13
|
+
// every consumer installs, including Workers deployments that can never call it, in
|
|
14
|
+
// exchange for a feature set both built-ins already have — and it does not survive the
|
|
15
|
+
// trade: constructing one under Bun 1.2.23 and 1.3.14 on macOS arm64 aborts the process
|
|
16
|
+
// with `NAPI FATAL ERROR`, which is a panic rather than a throw and so cannot even be
|
|
17
|
+
// caught and fallen back from.
|
|
6
18
|
|
|
7
19
|
import { TroveError } from './errors.js';
|
|
8
20
|
|
|
@@ -22,9 +34,69 @@ export async function openDatabase(pathOrMemory = ':memory:') {
|
|
|
22
34
|
return db;
|
|
23
35
|
}
|
|
24
36
|
|
|
37
|
+
// Held in a variable so a bundler cannot read it.
|
|
38
|
+
//
|
|
39
|
+
// `await import('bun:sqlite')` behind a `typeof Bun` guard still fails to BUILD for
|
|
40
|
+
// Workers: esbuild resolves a literal specifier statically, and a guard is a runtime
|
|
41
|
+
// thing that says nothing about what the bundler does with the module graph. The result
|
|
42
|
+
// was a wrangler build that could not link, from an import that would never have run.
|
|
43
|
+
// A non-literal specifier is not resolvable at build time, so it survives as a runtime
|
|
44
|
+
// import — which the guard then never reaches.
|
|
45
|
+
const BUN_SQLITE = 'bun:sqlite';
|
|
46
|
+
|
|
47
|
+
// Where a SQLite that can load extensions tends to live on macOS. Ordered: an explicit
|
|
48
|
+
// setting wins, then Homebrew on Apple Silicon, then Homebrew on Intel.
|
|
49
|
+
const MACOS_SQLITE_CANDIDATES = [
|
|
50
|
+
'/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib',
|
|
51
|
+
'/usr/local/opt/sqlite/lib/libsqlite3.dylib',
|
|
52
|
+
];
|
|
53
|
+
|
|
54
|
+
let customSqliteTried = false;
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Point Bun at a SQLite that can load extensions, on the one platform where it cannot.
|
|
58
|
+
*
|
|
59
|
+
* `bun:sqlite` supports extensions natively — `loadExtension` is right there and works
|
|
60
|
+
* on Linux with nothing special. macOS is the exception, and not because of Bun: the
|
|
61
|
+
* SYSTEM libsqlite3 that Bun links by default is built without
|
|
62
|
+
* SQLITE_ENABLE_LOAD_EXTENSION, so the call comes back "This build of sqlite3 does not
|
|
63
|
+
* support dynamic extension loading". The effect is that sqlite-vec cannot load, and
|
|
64
|
+
* semantic search silently degrades to an in-memory index rebuilt on every restart —
|
|
65
|
+
* on developer laptops specifically, which is where it is least likely to be noticed.
|
|
66
|
+
*
|
|
67
|
+
* `setCustomSQLite` is Bun's documented answer. It has to happen before any database is
|
|
68
|
+
* opened, hence doing it here rather than at the point extensions are wanted.
|
|
69
|
+
*
|
|
70
|
+
* Best-effort throughout: if no capable library is installed we leave Bun on the system
|
|
71
|
+
* one and the sqlite-vec store degrades exactly as it did before, with its own warning.
|
|
72
|
+
* Set TROVE_SQLITE_LIB to override the search.
|
|
73
|
+
*/
|
|
74
|
+
async function useExtensionCapableSqlite(Database) {
|
|
75
|
+
if (customSqliteTried || process.platform !== 'darwin') return;
|
|
76
|
+
customSqliteTried = true;
|
|
77
|
+
if (typeof Database.setCustomSQLite !== 'function') return;
|
|
78
|
+
|
|
79
|
+
const configured = process.env.TROVE_SQLITE_LIB;
|
|
80
|
+
const { existsSync } = await import('node:fs');
|
|
81
|
+
const candidates = configured ? [configured] : MACOS_SQLITE_CANDIDATES;
|
|
82
|
+
for (const lib of candidates) {
|
|
83
|
+
if (!existsSync(lib)) continue;
|
|
84
|
+
try {
|
|
85
|
+
Database.setCustomSQLite(lib);
|
|
86
|
+
return;
|
|
87
|
+
} catch { /* keep looking; the default is still a working database */ }
|
|
88
|
+
}
|
|
89
|
+
// Only worth saying when it was asked for explicitly — otherwise this is the ordinary
|
|
90
|
+
// case of a machine without Homebrew sqlite, and the vector store says its own piece.
|
|
91
|
+
if (configured) {
|
|
92
|
+
console.warn(`[trove] TROVE_SQLITE_LIB=${configured} could not be used; falling back to the system SQLite`);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
25
96
|
async function open(pathOrMemory) {
|
|
26
97
|
if (typeof Bun !== 'undefined') {
|
|
27
|
-
const { Database } = await import(
|
|
98
|
+
const { Database } = await import(BUN_SQLITE);
|
|
99
|
+
await useExtensionCapableSqlite(Database);
|
|
28
100
|
return new Database(pathOrMemory);
|
|
29
101
|
}
|
|
30
102
|
try {
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
// Is the backing store actually usable from a browser?
|
|
2
|
+
//
|
|
3
|
+
// This exists because of a failure that cost a day. A drive was serving its file list
|
|
4
|
+
// fine, and every file opened to a spinner that never resolved. The server was healthy,
|
|
5
|
+
// the storage was healthy, the download endpoint returned 200 in 300ms — and the browser
|
|
6
|
+
// still could not read a single byte, because the R2 bucket had no CORS policy. Nothing
|
|
7
|
+
// in the system knew: CORS is enforced in the browser, so the server's own request to
|
|
8
|
+
// the same bucket succeeded, and the only evidence was a console message in one tab.
|
|
9
|
+
//
|
|
10
|
+
// So the check has to be the browser's check. A CORS preflight is an ordinary OPTIONS
|
|
11
|
+
// request that anyone can send, including us — and a preflight never touches the object,
|
|
12
|
+
// so it works against a key that does not exist and costs nothing. Sending it against a
|
|
13
|
+
// presigned URL and reading the response headers is exactly what a browser does before
|
|
14
|
+
// it will hand a response to a page. If it fails for us it fails for them.
|
|
15
|
+
//
|
|
16
|
+
// The point is not detection for its own sake. Each finding carries the command that
|
|
17
|
+
// fixes it: a diagnostic that says "CORS is misconfigured" to someone who did not know
|
|
18
|
+
// buckets had a CORS policy has told them nothing they can act on.
|
|
19
|
+
|
|
20
|
+
/** A key that need not exist — a preflight is answered without looking one up. */
|
|
21
|
+
const PROBE_KEY = '.trove-cors-probe';
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Every code this module can produce.
|
|
25
|
+
*
|
|
26
|
+
* Exported because the caller that raises these as issues also has to CLEAR the ones a
|
|
27
|
+
* later check no longer reports — that is what makes fixing the bucket make the warning
|
|
28
|
+
* go away. Deriving the clear-set from the same list the checks use means a new finding
|
|
29
|
+
* cannot be added without becoming clearable.
|
|
30
|
+
*/
|
|
31
|
+
export const STORAGE_ISSUE_CODES = [
|
|
32
|
+
'storage-unreachable',
|
|
33
|
+
'cors-missing',
|
|
34
|
+
'cors-origin',
|
|
35
|
+
'cors-headers',
|
|
36
|
+
'cors-expose',
|
|
37
|
+
'cors-unknown',
|
|
38
|
+
];
|
|
39
|
+
|
|
40
|
+
/** Headers the client sends on a download, so a policy that omits them breaks reads. */
|
|
41
|
+
const NEEDED_REQUEST_HEADERS = ['range'];
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Headers the client must be able to READ off the response.
|
|
45
|
+
*
|
|
46
|
+
* Cross-origin responses expose almost nothing by default, and these are not cosmetic:
|
|
47
|
+
* without `content-range` the text viewer cannot tell a truncated file from a whole one,
|
|
48
|
+
* and without `accept-ranges` seeking in audio and video is not offered at all.
|
|
49
|
+
*
|
|
50
|
+
* Checked against a real GET rather than the preflight — see where this is used.
|
|
51
|
+
*/
|
|
52
|
+
const NEEDED_EXPOSED_HEADERS = ['content-range', 'content-length'];
|
|
53
|
+
|
|
54
|
+
const csv = (value) => String(value || '').toLowerCase().split(',').map((s) => s.trim()).filter(Boolean);
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Diagnose one collection's backing store.
|
|
58
|
+
*
|
|
59
|
+
* Ordered: an unreachable store short-circuits, because "CORS is not configured" is a
|
|
60
|
+
* misleading thing to say about a bucket whose credentials are wrong.
|
|
61
|
+
*
|
|
62
|
+
* @param {object} deps
|
|
63
|
+
* @param {import('./interface.js').StorageBackend} deps.storage
|
|
64
|
+
* @param {string} [deps.origin] the browser origin to check against — the drive's own
|
|
65
|
+
* public URL. Omitted means the CORS check is skipped rather than guessed at: a policy
|
|
66
|
+
* is allowed to be origin-specific, so checking the wrong origin invents a problem.
|
|
67
|
+
* @param {string} [deps.driver] store driver key, to word the remedy for it
|
|
68
|
+
* @param {typeof fetch} [deps.fetchImpl]
|
|
69
|
+
* @returns {Promise<Array<{code: string, severity: string, title: string, detail: string, remedy?: string}>>}
|
|
70
|
+
*/
|
|
71
|
+
export async function diagnoseStorage({ storage, origin = null, driver = null, fetchImpl = null } = {}) {
|
|
72
|
+
const findings = [];
|
|
73
|
+
if (!storage) return findings;
|
|
74
|
+
const doFetch = fetchImpl || (typeof fetch === 'function' ? fetch : null);
|
|
75
|
+
|
|
76
|
+
// --- can we talk to it at all? ---------------------------------------------
|
|
77
|
+
try {
|
|
78
|
+
await storage.list({ limit: 1 });
|
|
79
|
+
} catch (err) {
|
|
80
|
+
findings.push({
|
|
81
|
+
code: 'storage-unreachable',
|
|
82
|
+
severity: 'error',
|
|
83
|
+
title: 'The backing store could not be reached',
|
|
84
|
+
detail: err?.message || String(err),
|
|
85
|
+
remedy:
|
|
86
|
+
'Check the collection’s store settings: the bucket or directory must exist, and '
|
|
87
|
+
+ 'for an S3-compatible store the endpoint, region and credentials must all be for '
|
|
88
|
+
+ 'that bucket. A wrong endpoint and a wrong key look identical from here.',
|
|
89
|
+
});
|
|
90
|
+
return findings;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// --- does the browser get to read it? --------------------------------------
|
|
94
|
+
// Only when downloads go straight from the browser to the store. A drive that proxies
|
|
95
|
+
// its bytes through the server is same-origin all the way, and CORS never applies —
|
|
96
|
+
// reporting a bucket policy as missing there would be a problem the admin cannot have.
|
|
97
|
+
const caps = storage.capabilities || {};
|
|
98
|
+
if (!caps.presignDownload) return findings;
|
|
99
|
+
if (!origin || !doFetch) return findings;
|
|
100
|
+
|
|
101
|
+
let url;
|
|
102
|
+
try {
|
|
103
|
+
url = await storage.presignGet(PROBE_KEY, { expiresIn: 60 });
|
|
104
|
+
} catch {
|
|
105
|
+
// A store that claims presignDownload and cannot presign is a bug, not a
|
|
106
|
+
// configuration problem, and it will surface far more loudly elsewhere.
|
|
107
|
+
return findings;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
let res;
|
|
111
|
+
try {
|
|
112
|
+
res = await doFetch(url, {
|
|
113
|
+
method: 'OPTIONS',
|
|
114
|
+
headers: {
|
|
115
|
+
origin,
|
|
116
|
+
'access-control-request-method': 'GET',
|
|
117
|
+
'access-control-request-headers': NEEDED_REQUEST_HEADERS.join(','),
|
|
118
|
+
},
|
|
119
|
+
});
|
|
120
|
+
} catch (err) {
|
|
121
|
+
findings.push({
|
|
122
|
+
code: 'cors-unknown',
|
|
123
|
+
severity: 'warning',
|
|
124
|
+
title: 'Could not check whether the store allows browser access',
|
|
125
|
+
detail: `The preflight request to the store failed: ${err?.message || err}`,
|
|
126
|
+
remedy: 'This is usually a network or endpoint problem rather than a CORS one — '
|
|
127
|
+
+ 'confirm the store’s endpoint is reachable from the server.',
|
|
128
|
+
});
|
|
129
|
+
return findings;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const allowOrigin = res.headers.get('access-control-allow-origin');
|
|
133
|
+
if (!allowOrigin) {
|
|
134
|
+
findings.push({
|
|
135
|
+
code: 'cors-missing',
|
|
136
|
+
severity: 'error',
|
|
137
|
+
title: 'The store does not allow browser access, so files will not open',
|
|
138
|
+
detail:
|
|
139
|
+
`A CORS preflight from ${origin} was answered without an `
|
|
140
|
+
+ `Access-Control-Allow-Origin header (HTTP ${res.status}). Browsers will refuse `
|
|
141
|
+
+ 'every download, including previews and thumbnails, while the file list and '
|
|
142
|
+
+ 'search keep working normally.',
|
|
143
|
+
remedy: corsRemedy(origin, driver),
|
|
144
|
+
});
|
|
145
|
+
return findings;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
if (allowOrigin !== '*' && allowOrigin.toLowerCase() !== origin.toLowerCase()) {
|
|
149
|
+
findings.push({
|
|
150
|
+
code: 'cors-origin',
|
|
151
|
+
severity: 'error',
|
|
152
|
+
title: 'The store allows a different origin than this drive',
|
|
153
|
+
detail: `It allows "${allowOrigin}", but this drive is served from "${origin}". `
|
|
154
|
+
+ 'A policy naming the wrong origin is refused exactly like no policy at all.',
|
|
155
|
+
remedy: corsRemedy(origin, driver),
|
|
156
|
+
});
|
|
157
|
+
return findings;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Allowed, but possibly not for everything we send or need back. These are warnings:
|
|
161
|
+
// opening a small file will work, so the drive is usable — it is seeking in a video
|
|
162
|
+
// and reading the head of a large file that break.
|
|
163
|
+
const allowHeaders = csv(res.headers.get('access-control-allow-headers'));
|
|
164
|
+
const missingRequest = allowHeaders.includes('*')
|
|
165
|
+
? []
|
|
166
|
+
: NEEDED_REQUEST_HEADERS.filter((h) => !allowHeaders.includes(h));
|
|
167
|
+
if (missingRequest.length) {
|
|
168
|
+
findings.push({
|
|
169
|
+
code: 'cors-headers',
|
|
170
|
+
severity: 'warning',
|
|
171
|
+
title: 'The store’s CORS policy blocks ranged reads',
|
|
172
|
+
detail: `It does not allow the ${missingRequest.join(', ')} request header, so seeking `
|
|
173
|
+
+ 'in audio and video, and previewing the start of a large file, will fail. Whole-file '
|
|
174
|
+
+ 'downloads are unaffected.',
|
|
175
|
+
remedy: corsRemedy(origin, driver),
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Exposed headers are read off the ACTUAL response, not the preflight — that is where
|
|
180
|
+
// the spec puts them and where the browser looks. Some stores echo them on a preflight
|
|
181
|
+
// and some do not, so checking the OPTIONS response reports a correctly configured
|
|
182
|
+
// bucket as broken. A GET for a key that does not exist is answered 404 WITH the CORS
|
|
183
|
+
// headers when a policy matches, which is all this needs.
|
|
184
|
+
let actual;
|
|
185
|
+
try {
|
|
186
|
+
actual = await doFetch(url, { method: 'GET', headers: { origin } });
|
|
187
|
+
} catch {
|
|
188
|
+
// The preflight already passed, so the policy is in place; failing to complete this
|
|
189
|
+
// second request says nothing more about it. Warning here would be guessing.
|
|
190
|
+
return findings;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
const exposed = csv(actual.headers.get('access-control-expose-headers'));
|
|
194
|
+
const missingExposed = exposed.includes('*')
|
|
195
|
+
? []
|
|
196
|
+
: NEEDED_EXPOSED_HEADERS.filter((h) => !exposed.includes(h));
|
|
197
|
+
if (missingExposed.length) {
|
|
198
|
+
findings.push({
|
|
199
|
+
code: 'cors-expose',
|
|
200
|
+
severity: 'warning',
|
|
201
|
+
title: 'The store hides response headers the viewer needs',
|
|
202
|
+
detail: `${missingExposed.join(', ')} are not in exposeHeaders, so the browser cannot read `
|
|
203
|
+
+ 'them. Trove cannot then tell a truncated preview from a complete file, and will '
|
|
204
|
+
+ 'not offer seeking.',
|
|
205
|
+
remedy: corsRemedy(origin, driver),
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
return findings;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** The policy this drive needs, as something that can be pasted. */
|
|
213
|
+
export function corsPolicy(origin) {
|
|
214
|
+
return [{
|
|
215
|
+
AllowedOrigins: [origin],
|
|
216
|
+
AllowedMethods: ['GET', 'PUT', 'HEAD'],
|
|
217
|
+
AllowedHeaders: ['content-type', 'range'],
|
|
218
|
+
ExposeHeaders: ['ETag', 'Content-Length', 'Content-Type', 'Content-Range', 'Accept-Ranges'],
|
|
219
|
+
MaxAgeSeconds: 3600,
|
|
220
|
+
}];
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function corsRemedy(origin, driver) {
|
|
224
|
+
const json = JSON.stringify(corsPolicy(origin), null, 2);
|
|
225
|
+
// R2 is the common case for a Workers deployment and its tooling is not the AWS CLI,
|
|
226
|
+
// so name it explicitly rather than leaving the admin to translate.
|
|
227
|
+
const r2 = driver === 's3'
|
|
228
|
+
? '\n\nOn Cloudflare R2, save the JSON above as cors.json and run:\n'
|
|
229
|
+
+ ' wrangler r2 bucket cors put <bucket> --file cors.json\n\n'
|
|
230
|
+
+ 'On AWS S3:\n'
|
|
231
|
+
+ ' aws s3api put-bucket-cors --bucket <bucket> --cors-configuration file://cors.json'
|
|
232
|
+
: '';
|
|
233
|
+
return `Allow this origin on the bucket. The policy Trove needs:\n\n${json}${r2}`;
|
|
234
|
+
}
|