@3sln/trove 0.0.11 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/package.json +2 -1
- package/packages/core/src/collections/index.js +71 -10
- package/packages/core/src/encryption/keys.js +4 -3
- package/packages/core/src/encryption/policy.js +14 -7
- package/packages/core/src/encryption/rotation.js +46 -12
- package/packages/core/src/errors.js +8 -0
- package/packages/core/src/index.js +8 -2
- package/packages/core/src/indexers/contribution.js +18 -17
- package/packages/core/src/indexing.js +8 -4
- package/packages/core/src/issues.js +8 -0
- package/packages/core/src/links.js +21 -9
- package/packages/core/src/metadata/interface.js +22 -0
- package/packages/core/src/metadata/memory.js +15 -0
- package/packages/core/src/metadata/sqlite.js +23 -1
- package/packages/core/src/plugins/contributions.js +13 -0
- package/packages/core/src/rateLimit.js +253 -0
- package/packages/core/src/scan.js +44 -16
- package/packages/core/src/sidecar/document.js +18 -14
- package/packages/core/src/sidecar/index.js +9 -10
- package/packages/core/src/sidecar/manager.js +10 -15
- package/packages/core/src/uploads.js +40 -15
- package/packages/core/src/vfs.js +59 -34
- package/packages/plugin-sdk/src/browser.js +155 -0
- package/packages/plugin-sdk/src/protocol.js +13 -1
- package/packages/server/src/access/externalEvaluation.js +17 -0
- package/packages/server/src/adapters/bun.js +1 -2
- package/packages/server/src/adapters/node.js +1 -2
- package/packages/server/src/engine/index.js +1 -1
- package/packages/server/src/engine/providers/access.js +17 -34
- package/packages/server/src/engine/providers/core.js +112 -14
- package/packages/server/src/index.js +160 -105
- package/packages/server/src/mcp/index.js +7 -12
- package/packages/server/src/mcp/tools.js +24 -29
- package/packages/server/src/router.js +31 -8
- package/packages/server/src/routes.js +74 -113
- package/packages/server/src/scope.js +78 -0
- package/packages/web/dist/assets/main-828yzsr7.js +511 -0
- package/packages/web/dist/assets/main-828yzsr7.js.map +120 -0
- package/packages/web/dist/index.html +1 -1
- package/packages/web/dist/sw.js +1 -1
- package/packages/web/src/bl/actions.js +226 -104
- package/packages/web/src/bl/activity.js +32 -82
- package/packages/web/src/bl/index.js +23 -11
- package/packages/web/src/bl/launcher.js +2 -9
- package/packages/web/src/bl/links.js +31 -18
- package/packages/web/src/bl/offline.js +24 -10
- package/packages/web/src/bl/pluginInstall.js +29 -26
- package/packages/web/src/bl/queries.js +76 -68
- package/packages/web/src/bl/services.js +22 -14
- package/packages/web/src/bl/social.js +39 -25
- package/packages/web/src/bl/state.js +22 -4
- package/packages/web/src/bl/viewState.js +9 -0
- package/packages/web/src/bl/watchQuery.js +1 -8
- package/packages/web/src/dispatch.js +26 -0
- package/packages/web/src/platform/api.js +32 -0
- package/packages/web/src/platform/commands.js +14 -15
- package/packages/web/src/platform/context.js +3 -58
- package/packages/web/src/platform/contributions.js +1 -35
- package/packages/web/src/platform/fileChunks.js +281 -0
- package/packages/web/src/platform/index.js +8 -9
- package/packages/web/src/platform/mediaUrls.js +10 -0
- package/packages/web/src/platform/navigation.js +35 -21
- package/packages/web/src/platform/pickers.js +39 -19
- package/packages/web/src/platform/pluginHost.js +4 -1
- package/packages/web/src/platform/pluginRpc.js +47 -2
- package/packages/web/src/platform/spatialNav.js +1 -1
- package/packages/web/src/platform/viewport.js +14 -6
- package/packages/web/src/platform/voiceSearch.js +17 -9
- package/packages/web/src/platform/whenclause.js +9 -19
- package/packages/web/src/runtime.js +2 -37
- package/packages/web/src/ui/activate.js +18 -2
- package/packages/web/src/ui/components/activityPanel.js +1 -1
- package/packages/web/src/ui/components/adminView.js +4 -7
- package/packages/web/src/ui/components/editorArea.js +10 -1
- package/packages/web/src/ui/components/launcher.js +29 -13
- package/packages/web/src/ui/components/openers/index.js +17 -22
- package/packages/web/src/ui/components/openers/markdown.js +2 -2
- package/packages/web/src/ui/components/overlays.js +31 -5
- package/packages/web/src/ui/components/pluginReview.js +7 -4
- package/packages/web/src/ui/components/pluginsView.js +1 -1
- package/packages/web/src/ui/components/settingsView.js +30 -27
- package/packages/web/src/ui/components/statusBar.js +8 -6
- package/packages/web/src/ui/components/views/grid.js +2 -2
- package/packages/web/src/ui/components/views/parts.js +1 -1
- package/packages/web/src/ui/compositions/workbench.js +20 -6
- package/packages/web/src/ui/sanitize.js +0 -5
- package/packages/web/src/workbench.js +37 -19
- package/packages/web/dist/assets/main-c9dnnnc6.js +0 -356
- package/packages/web/dist/assets/main-c9dnnnc6.js.map +0 -118
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
// means the same thing it always did.
|
|
22
22
|
|
|
23
23
|
import {
|
|
24
|
-
StorageBackend, MemoryStorage,
|
|
24
|
+
StorageBackend, MemoryStorage,
|
|
25
25
|
StorageDriverRegistry, portableDrivers,
|
|
26
26
|
MetadataStore, MemoryStore, SqliteStore,
|
|
27
27
|
SearchService, EmbeddingProvider, LocalHashEmbedding, HttpEmbedding,
|
|
@@ -44,6 +44,8 @@ import {
|
|
|
44
44
|
Vfs, TroveError,
|
|
45
45
|
resolveAuthDiscovery,
|
|
46
46
|
SignedUrls, resolveUrlSecret,
|
|
47
|
+
diagnoseStorage, STORAGE_ISSUE_CODES,
|
|
48
|
+
RateLimiter, MemoryRateStore, KvRateStore, DEFAULT_RATE_LIMITS,
|
|
47
49
|
} from '@3sln/trove/core';
|
|
48
50
|
import { Provider } from '@3sln/ngin';
|
|
49
51
|
import { need } from '../lazy.js';
|
|
@@ -171,6 +173,33 @@ export function coreProviders(config, lifecycleState) {
|
|
|
171
173
|
// reads it has to say so.
|
|
172
174
|
config: Provider.fromSingleton(config),
|
|
173
175
|
|
|
176
|
+
/**
|
|
177
|
+
* What one caller may cost, per class of work.
|
|
178
|
+
*
|
|
179
|
+
* A provider so the route table can lease it like anything else, and so the store is a
|
|
180
|
+
* decision made once from configuration rather than at each call site. Null when
|
|
181
|
+
* limiting is switched off, which is the one case a caller has to handle — and it does
|
|
182
|
+
* so by not asking, since `Router.handle` only enforces for routes that named a class.
|
|
183
|
+
*/
|
|
184
|
+
rateLimiter: Provider.fromLazySingleton(
|
|
185
|
+
async (deps) => {
|
|
186
|
+
// ON unless switched off, including for a library caller who built a config by
|
|
187
|
+
// hand rather than through `configFromEnv`. A limit that only exists when somebody
|
|
188
|
+
// remembers to ask for it is not a limit.
|
|
189
|
+
const rl = config.rateLimit ?? { enabled: true, store: 'memory', limits: DEFAULT_RATE_LIMITS };
|
|
190
|
+
if (!rl.enabled) return null;
|
|
191
|
+
// KV counters are shared across instances and cost a read and a write per limited
|
|
192
|
+
// request; memory counters are free and exact on ONE long-lived process. See
|
|
193
|
+
// rateLimit.js — the choice is a property of the deployment, not of the code.
|
|
194
|
+
const store = rl.store === 'kv'
|
|
195
|
+
? new KvRateStore({ kv: (await need(deps, ['kv'])).kv })
|
|
196
|
+
: new MemoryRateStore();
|
|
197
|
+
return new RateLimiter({ store, limits: rl.limits || DEFAULT_RATE_LIMITS });
|
|
198
|
+
},
|
|
199
|
+
null,
|
|
200
|
+
{ deps: ['kv'] },
|
|
201
|
+
),
|
|
202
|
+
|
|
174
203
|
// Shutdown, likewise. Long work has to be able to ask whether the server is
|
|
175
204
|
// going down, and closing over a `let` made that invisible.
|
|
176
205
|
lifecycle: Provider.fromSingleton({
|
|
@@ -198,12 +227,83 @@ export function coreProviders(config, lifecycleState) {
|
|
|
198
227
|
beginReindex: (opts) => lifecycleState.background.beginReindex(opts),
|
|
199
228
|
}),
|
|
200
229
|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
230
|
+
/**
|
|
231
|
+
* The storage self-check.
|
|
232
|
+
*
|
|
233
|
+
* The failure this exists for: a bucket with no CORS policy serves the SERVER fine and
|
|
234
|
+
* serves the browser nothing, so the drive looks healthy and every file opens to a
|
|
235
|
+
* spinner. See core/storage/diagnose.js for why the check has to be a real preflight.
|
|
236
|
+
*
|
|
237
|
+
* `origin` is the browser origin to check the policy against, and there is no guessing
|
|
238
|
+
* it: a policy may legitimately name one origin, so checking the wrong one would invent
|
|
239
|
+
* a problem. A request supplies its own; a cron firing has only `config.publicUrl`, and
|
|
240
|
+
* without either the CORS half is skipped rather than assumed.
|
|
241
|
+
*
|
|
242
|
+
* A PROVIDER, not a seam stamped onto `lifecycleState` after construction. The stated
|
|
243
|
+
* reason for that shape was "it needs `collections` and `issues` from this container",
|
|
244
|
+
* which is exactly what `fromLazySingleton` with deps is for — and unlike
|
|
245
|
+
* `backgroundWork`, whose comment names a real circularity (dispatching needs the
|
|
246
|
+
* engine that owns the container), there is none here.
|
|
247
|
+
*/
|
|
248
|
+
storageCheck: Provider.fromLazySingleton(
|
|
249
|
+
async (deps) => {
|
|
250
|
+
const { collections, issues, config: cfg } = await need(deps, ['collections', 'issues', 'config']);
|
|
251
|
+
const KIND = 'storage';
|
|
252
|
+
return {
|
|
253
|
+
async run({ origin = null } = {}) {
|
|
254
|
+
// `all()`, not `list(null)`: this has no user, and asking what the anonymous
|
|
255
|
+
// principal may read means checking nothing at all on a drive that is not public.
|
|
256
|
+
const list = await collections.all().catch(() => []);
|
|
257
|
+
const results = [];
|
|
258
|
+
for (const c of list) {
|
|
259
|
+
let findings;
|
|
260
|
+
try {
|
|
261
|
+
const storage = await collections.storageFor(c.id);
|
|
262
|
+
findings = await diagnoseStorage({
|
|
263
|
+
storage, origin, driver: c.store?.driver || null, fetchImpl: cfg.fetch,
|
|
264
|
+
});
|
|
265
|
+
} catch (err) {
|
|
266
|
+
// Failing to BUILD the store is itself the most severe version of
|
|
267
|
+
// unreachable — an unknown driver, or a config missing a required field,
|
|
268
|
+
// never gets far enough to be asked whether it can be read.
|
|
269
|
+
findings = [{
|
|
270
|
+
code: 'storage-unreachable',
|
|
271
|
+
severity: 'error',
|
|
272
|
+
title: 'This collection\u2019s store could not be opened',
|
|
273
|
+
detail: err?.message || String(err),
|
|
274
|
+
}];
|
|
275
|
+
}
|
|
276
|
+
const found = new Set(findings.map((f) => f.code));
|
|
277
|
+
for (const f of findings) {
|
|
278
|
+
await issues.raise({
|
|
279
|
+
kind: KIND,
|
|
280
|
+
subject: `${c.id}:${f.code}`,
|
|
281
|
+
title: `${c.name || c.id}: ${f.title}`,
|
|
282
|
+
detail: f.detail,
|
|
283
|
+
remedy: f.remedy || null,
|
|
284
|
+
severity: f.severity,
|
|
285
|
+
collectionId: c.id,
|
|
286
|
+
// Re-running the check IS the fix verification, so Retry rechecks against
|
|
287
|
+
// the same origin the finding was made for. Checking a different one
|
|
288
|
+
// would report a pass for a policy the affected browser still cannot use.
|
|
289
|
+
retry: { op: 'storage-check', origin },
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
// Whatever is no longer true stops being listed. Without this, fixing the
|
|
293
|
+
// bucket leaves the warning up, and a problem list that outlives its problems
|
|
294
|
+
// is one people learn to scroll past.
|
|
295
|
+
for (const code of STORAGE_ISSUE_CODES) {
|
|
296
|
+
if (!found.has(code)) await issues.clear(KIND, `${c.id}:${code}`);
|
|
297
|
+
}
|
|
298
|
+
results.push({ collectionId: c.id, name: c.name || c.id, findings });
|
|
299
|
+
}
|
|
300
|
+
return { checked: results.length, corsChecked: !!origin, results };
|
|
301
|
+
},
|
|
302
|
+
};
|
|
303
|
+
},
|
|
304
|
+
null,
|
|
305
|
+
{ deps: ['collections', 'issues', 'config'] },
|
|
306
|
+
),
|
|
207
307
|
|
|
208
308
|
storage: Provider.fromLazySingleton(
|
|
209
309
|
() => resolve(config.storage ?? config.vfs?.storage, StorageBackend, (cfg) => buildStorage(cfg, config)),
|
|
@@ -525,13 +625,11 @@ export function coreProviders(config, lifecycleState) {
|
|
|
525
625
|
indexers: r.indexerRuntime
|
|
526
626
|
? new PluginIndexers({ vfs: r.vfs, runtime: r.indexerRuntime, packages: r.packageStore })
|
|
527
627
|
: null,
|
|
528
|
-
// Who may install a plugin.
|
|
529
|
-
//
|
|
530
|
-
//
|
|
531
|
-
//
|
|
532
|
-
isAdmin: (principal) =>
|
|
533
|
-
? !!principal
|
|
534
|
-
: r.collections.isAdmin(principal)),
|
|
628
|
+
// Who may install a plugin. There is no "any authenticated caller qualifies"
|
|
629
|
+
// fallback: the arm that provided one required `collections: false`, which this
|
|
630
|
+
// provider refuses above and `configFromEnv` refuses again, so it could only ever
|
|
631
|
+
// have fired for a direct-container caller — where it handed out an open drive.
|
|
632
|
+
isAdmin: (principal) => r.collections.isAdmin(principal),
|
|
535
633
|
maxPackageBytes: config.maxUploadBytes ?? undefined,
|
|
536
634
|
strict: config.enforcePluginCaps === true,
|
|
537
635
|
});
|
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
VectorStore, KeywordStore, IndexerRegistry,
|
|
16
16
|
accessHost, TroveError,
|
|
17
17
|
protectedResourceMetadata, challengeHeaders, publicOrigin,
|
|
18
|
-
|
|
18
|
+
DEFAULT_RATE_LIMITS, RATE_CLASSES, describeRateLimits,
|
|
19
19
|
} from '@3sln/trove/core';
|
|
20
20
|
import { createRouter, routeHelpers } from './routes.js';
|
|
21
21
|
import { createDriveEngine, scanStarter, BACKBONE } from './engine/index.js';
|
|
@@ -58,6 +58,15 @@ export async function createServer(config = {}) {
|
|
|
58
58
|
// from it and /api/capabilities describes it, so the form a user sees and the set of
|
|
59
59
|
// things the server can actually construct cannot drift apart.
|
|
60
60
|
config = { ...config, storageRegistry: storageRegistry(config) };
|
|
61
|
+
// Here, not in the adapters. Three of the four called it and the Worker one did not —
|
|
62
|
+
// and a default `wrangler deploy` satisfies the condition exactly: `configFromEnv`
|
|
63
|
+
// leaves identity anonymous when TROVE_AUTH is unset and `defaultOpen` defaults true, so
|
|
64
|
+
// the drive went up world-readable and world-writable on a public URL with a clean log,
|
|
65
|
+
// while the identical config under Bun printed the warning. No adapter can forget it
|
|
66
|
+
// from here. It fires per isolate cold start rather than per process, which is noisier
|
|
67
|
+
// only for the deployments that really are open to everyone.
|
|
68
|
+
warnOnOpenAccess(config);
|
|
69
|
+
warnOnUnenforceableLimits(config);
|
|
61
70
|
const lifecycleState = { closing: false, background: null };
|
|
62
71
|
const engine = createDriveEngine(config, lifecycleState);
|
|
63
72
|
|
|
@@ -68,6 +77,7 @@ export async function createServer(config = {}) {
|
|
|
68
77
|
const {
|
|
69
78
|
storage, sqlite: sqliteProvider, metadata, kv, tasks, issues, notifications,
|
|
70
79
|
sidecar, collections, identity, auth, search, vfs, plugins, apiKeys, capabilities, rotation,
|
|
80
|
+
storageCheck, rateLimiter,
|
|
71
81
|
} = backbone.resources;
|
|
72
82
|
|
|
73
83
|
// Aliased so the rest of this function reads as it did; the container's
|
|
@@ -159,68 +169,12 @@ export async function createServer(config = {}) {
|
|
|
159
169
|
const routeBeginScan = lifecycleState.background.beginScan;
|
|
160
170
|
const routeBeginReindex = lifecycleState.background.beginReindex;
|
|
161
171
|
issues.handle('scan-collection', (issue) => startScan(issue.retry.collectionId, { reason: 'Retrying after a failed scan' }));
|
|
172
|
+
issues.handle('storage-check', (issue) => storageCheck.run({ origin: issue.retry?.origin || config.publicUrl || null }));
|
|
173
|
+
// The one retry that matters most: the user has been told a comment saved and it exists
|
|
174
|
+
// only in memory. The op was raised for years with no handler registered for it — and in
|
|
175
|
+
// a shape `canRetry` rejected, so the button never rendered to reveal that.
|
|
176
|
+
issues.handle('sidecar-flush', () => sidecar.retryPending());
|
|
162
177
|
|
|
163
|
-
// --- storage self-check ----------------------------------------------------
|
|
164
|
-
// The failure this exists for: a bucket with no CORS policy serves the server fine and
|
|
165
|
-
// serves the browser nothing, so the drive looks healthy and every file opens to a
|
|
166
|
-
// spinner. See core/storage/diagnose.js for why the check has to be a real preflight.
|
|
167
|
-
//
|
|
168
|
-
// `origin` is the browser origin to check the policy against, and there is no guessing
|
|
169
|
-
// it: a policy may legitimately name one origin, so checking the wrong one would invent
|
|
170
|
-
// a problem. A request supplies its own; a cron firing has only `config.publicUrl`, and
|
|
171
|
-
// without either the CORS half is skipped rather than assumed.
|
|
172
|
-
const STORAGE_ISSUE_KIND = 'storage';
|
|
173
|
-
async function checkStorage({ origin = null } = {}) {
|
|
174
|
-
// `all()`, not `list(null)`: this has no user, and asking what the anonymous principal
|
|
175
|
-
// may read means checking nothing at all on a drive that is not public.
|
|
176
|
-
const list = collections ? await collections.all().catch(() => []) : [];
|
|
177
|
-
const results = [];
|
|
178
|
-
for (const c of list) {
|
|
179
|
-
let findings;
|
|
180
|
-
try {
|
|
181
|
-
const storage = await collections.storageFor(c.id);
|
|
182
|
-
findings = await diagnoseStorage({
|
|
183
|
-
storage, origin, driver: c.store?.driver || null, fetchImpl: config.fetch,
|
|
184
|
-
});
|
|
185
|
-
} catch (err) {
|
|
186
|
-
// Failing to BUILD the store is itself the most severe version of unreachable —
|
|
187
|
-
// an unknown driver or a config missing a required field never gets far enough
|
|
188
|
-
// to be asked whether it can be read.
|
|
189
|
-
findings = [{
|
|
190
|
-
code: 'storage-unreachable',
|
|
191
|
-
severity: 'error',
|
|
192
|
-
title: 'This collection’s store could not be opened',
|
|
193
|
-
detail: err?.message || String(err),
|
|
194
|
-
}];
|
|
195
|
-
}
|
|
196
|
-
const found = new Set(findings.map((f) => f.code));
|
|
197
|
-
for (const f of findings) {
|
|
198
|
-
await issues.raise({
|
|
199
|
-
kind: STORAGE_ISSUE_KIND,
|
|
200
|
-
subject: `${c.id}:${f.code}`,
|
|
201
|
-
title: `${c.name || c.id}: ${f.title}`,
|
|
202
|
-
detail: f.detail,
|
|
203
|
-
remedy: f.remedy || null,
|
|
204
|
-
severity: f.severity,
|
|
205
|
-
collectionId: c.id,
|
|
206
|
-
// Re-running the check IS the fix verification, so Retry rechecks against the
|
|
207
|
-
// same origin the finding was made for. Checking a different one would report
|
|
208
|
-
// a pass for a policy the affected browser still cannot use.
|
|
209
|
-
retry: { op: 'storage-check', origin },
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
// Whatever is no longer true stops being listed. Without this, fixing the bucket
|
|
213
|
-
// leaves the warning up, and a problem list that outlives its problems is one
|
|
214
|
-
// people learn to scroll past.
|
|
215
|
-
for (const code of STORAGE_ISSUE_CODES) {
|
|
216
|
-
if (!found.has(code)) await issues.clear(STORAGE_ISSUE_KIND, `${c.id}:${code}`);
|
|
217
|
-
}
|
|
218
|
-
results.push({ collectionId: c.id, name: c.name || c.id, findings });
|
|
219
|
-
}
|
|
220
|
-
return { checked: results.length, corsChecked: !!origin, results };
|
|
221
|
-
}
|
|
222
|
-
issues.handle('storage-check', (issue) => checkStorage({ origin: issue.retry?.origin || config.publicUrl || null }));
|
|
223
|
-
lifecycleState.storageCheck = checkStorage;
|
|
224
178
|
|
|
225
179
|
issues.handle('reindex-node', (issue) => tasks.run(
|
|
226
180
|
// Carries the issue's collection, so the person who can see the file can also see
|
|
@@ -263,31 +217,24 @@ export async function createServer(config = {}) {
|
|
|
263
217
|
return out;
|
|
264
218
|
}
|
|
265
219
|
|
|
266
|
-
// Periodic maintenance
|
|
267
|
-
//
|
|
268
|
-
//
|
|
269
|
-
//
|
|
220
|
+
// Periodic maintenance for a runtime whose timers survive the request.
|
|
221
|
+
//
|
|
222
|
+
// The BODY is `runMaintenance`, the same function the Worker's cron calls, because this
|
|
223
|
+
// is where the two schedulers drift. `stepRotations`'s docblock already demanded it —
|
|
224
|
+
// "One function, both callers, so the next thing added to periodic work cannot land in
|
|
225
|
+
// one scheduler and not the other" — and two things had since landed in one and not the
|
|
226
|
+
// other: `checkStorage`, so a bucket whose CORS or credentials changed went unnoticed on
|
|
227
|
+
// the runtime Trove is actually self-hosted on until someone opened Activity and pressed
|
|
228
|
+
// a button, and `notifications.flush()`.
|
|
229
|
+
//
|
|
230
|
+
// `scan: false` keeps the opt-in split below. Everything else — the unbounded upload
|
|
231
|
+
// session and sidecar caches, trash retention, mention delivery, the storage preflight,
|
|
232
|
+
// and finishing a rotation someone started — happens on every firing on every runtime.
|
|
270
233
|
let maintenance = null;
|
|
271
234
|
if (config.startFlusher !== false && config.maintenanceIntervalMs !== 0) {
|
|
272
235
|
const everyMs = config.maintenanceIntervalMs ?? 5 * 60 * 1000;
|
|
273
|
-
// Trash retention. This is the only thing in Trove that destroys data on a timer,
|
|
274
|
-
// so it is opt-outable (TROVE_TRASH_DAYS=0 keeps the trash forever) and it says what
|
|
275
|
-
// it removed. 30 days is the same grace period the drives people are used to give.
|
|
276
|
-
const trashMs = (config.trashRetentionDays ?? 30) * 86400_000;
|
|
277
236
|
maintenance = setInterval(() => {
|
|
278
|
-
|
|
279
|
-
// NOT `sidecar.sweep?.()` — that name did not exist on SidecarService, and the
|
|
280
|
-
// optional call turned "evict idle documents" into a no-op for the process's
|
|
281
|
-
// whole lifetime. It exists now, and the `?.` is gone so a rename shows up.
|
|
282
|
-
.then(() => sidecar.sweep())
|
|
283
|
-
.then(() => (trashMs > 0 ? vfs.purgeTrash({ before: Date.now() - trashMs }) : null))
|
|
284
|
-
.then((r) => { if (r?.purged) console.log(`[trove] purged ${r.purged} item(s) from the trash after ${config.trashRetentionDays ?? 30} days`); })
|
|
285
|
-
// A rotation started through the API finishes on its own here. Unlike a scan this
|
|
286
|
-
// is NOT opt-in: it only touches collections someone has explicitly put into
|
|
287
|
-
// rotation, and leaving one half-moved is worse than the work of finishing it.
|
|
288
|
-
.then(() => (collections ? collections.all().catch(() => []) : []))
|
|
289
|
-
.then((targets) => stepRotations(targets, Math.max(1000, Math.floor(everyMs / 4))))
|
|
290
|
-
.then((moved) => { for (const m of moved) if (m.moved) console.log(`[trove] rotation ${m.collectionId}: ${m.moved} moved, ${m.status}`); })
|
|
237
|
+
runMaintenance({ scan: false, budgetMs: Math.max(4000, Math.floor(everyMs / 4)) })
|
|
291
238
|
.catch((e) => console.error('maintenance sweep failed', e));
|
|
292
239
|
}, everyMs);
|
|
293
240
|
maintenance.unref?.();
|
|
@@ -298,16 +245,18 @@ export async function createServer(config = {}) {
|
|
|
298
245
|
// metered API and real load on a NAS. A deployment that shares its bucket with other
|
|
299
246
|
// tools wants this on; one where Trove is the only writer doesn't need it at all, and
|
|
300
247
|
// can scan on demand instead.
|
|
248
|
+
//
|
|
249
|
+
// The same function again, with scanning on. It repeats the sweeps that the maintenance
|
|
250
|
+
// tick also does — all of them idempotent, and concurrent drains collapse — which is a
|
|
251
|
+
// smaller price than a third hand-written body of periodic work. What it is NOT allowed
|
|
252
|
+
// to do is pick its own collection list: this used to walk `collections.list(null)`,
|
|
253
|
+
// "what may the anonymous principal read", so on any drive that is not open to the
|
|
254
|
+
// public the scheduled scan silently scanned nothing whatsoever.
|
|
301
255
|
let scanTimer = null;
|
|
302
256
|
if (config.startFlusher !== false && config.scanIntervalMs) {
|
|
303
257
|
scanTimer = setInterval(() => {
|
|
304
258
|
if (tasks.list().some((t) => t.kind === 'scan' && t.status === 'running')) return; // still going
|
|
305
|
-
|
|
306
|
-
.then(async (list) => {
|
|
307
|
-
for (const c of list) {
|
|
308
|
-
await startScan(c.id, { reason: 'Scheduled' }).catch(() => {});
|
|
309
|
-
}
|
|
310
|
-
})
|
|
259
|
+
runMaintenance({ scan: true, budgetMs: Math.max(4000, Math.floor(config.scanIntervalMs / 2)) })
|
|
311
260
|
.catch((e) => console.error('scheduled scan failed', e));
|
|
312
261
|
}, config.scanIntervalMs);
|
|
313
262
|
scanTimer.unref?.();
|
|
@@ -322,7 +271,7 @@ export async function createServer(config = {}) {
|
|
|
322
271
|
// channel cannot shadow a built-in route by claiming its path.
|
|
323
272
|
for (const channel of notifications?.channels || []) {
|
|
324
273
|
for (const route of channel.routes?.(routeHelpers) || []) {
|
|
325
|
-
router.add(route.method, route.path, route.deps || [], route.handler);
|
|
274
|
+
router.add(route.method, route.path, route.deps || [], route.handler, { cost: route.cost || 'write' });
|
|
326
275
|
}
|
|
327
276
|
}
|
|
328
277
|
|
|
@@ -330,11 +279,23 @@ export async function createServer(config = {}) {
|
|
|
330
279
|
// exist. A drive that has not configured one has no `/api/access/*` at all, rather than
|
|
331
280
|
// endpoints that exist to answer "no" — which is the difference between a feature that is
|
|
332
281
|
// off and a feature that is broken.
|
|
282
|
+
// Paths a contributed component declared as needing no Trove identity. Exact matches
|
|
283
|
+
// only, and that is deliberate: the check below runs BEFORE routing, so it cannot know
|
|
284
|
+
// which parameterised route would have matched. A component that wants a public route
|
|
285
|
+
// gives it a fixed path.
|
|
286
|
+
const publicPaths = new Set();
|
|
287
|
+
|
|
333
288
|
const accessPolicy = config.accessEvaluation
|
|
334
289
|
? externalEvaluation({ ...config.accessEvaluation, team: config.accessEvaluation.team || config.identity?.access?.team })
|
|
335
290
|
: null;
|
|
336
291
|
for (const route of accessPolicy?.routes?.(routeHelpers) || []) {
|
|
337
|
-
|
|
292
|
+
// `evaluate` by default for a contributed public route: this is the one endpoint a
|
|
293
|
+
// stranger reaches with no credential at all, and each call is a JWKS fetch or cache
|
|
294
|
+
// read and an RSA verify. Refusing to ANSWER an unattributable caller is the right
|
|
295
|
+
// security property and does nothing about the cost of asking.
|
|
296
|
+
router.add(route.method, route.path, route.deps || [], route.handler,
|
|
297
|
+
{ cost: route.cost || (route.public ? 'evaluate' : 'write') });
|
|
298
|
+
if (route.public) publicPaths.add(route.path);
|
|
338
299
|
}
|
|
339
300
|
|
|
340
301
|
// Said at boot, because that is when someone is looking and can still fix it. The
|
|
@@ -345,7 +306,7 @@ export async function createServer(config = {}) {
|
|
|
345
306
|
// MCP: the same drive, the same identity, spoken to by an agent instead of a browser.
|
|
346
307
|
// Null when switched off, and then nothing below routes to it.
|
|
347
308
|
const mcp = createMcpHandler({
|
|
348
|
-
vfs, collections, identity, config, auth,
|
|
309
|
+
vfs, collections, identity, config, auth, rateLimiter,
|
|
349
310
|
// So an agent's tool call obtains the same authorized handles an HTTP route does.
|
|
350
311
|
container: engine.container,
|
|
351
312
|
version: config.version || '0.0.1',
|
|
@@ -409,8 +370,14 @@ export async function createServer(config = {}) {
|
|
|
409
370
|
let principal = null;
|
|
410
371
|
let grant = null;
|
|
411
372
|
try {
|
|
412
|
-
|
|
413
|
-
|
|
373
|
+
// A public route answers to something other than this drive's identity — the
|
|
374
|
+
// external policy endpoints verify a Cloudflare-signed assertion themselves, and
|
|
375
|
+
// the keys endpoint serves a public key. Requiring a session on those is asking
|
|
376
|
+
// the caller to authenticate as a user it is not and does not have.
|
|
377
|
+
if (!publicPaths.has(url.pathname)) {
|
|
378
|
+
grant = await capabilities.resolve(req);
|
|
379
|
+
if (!grant) principal = await identity.authenticate(req);
|
|
380
|
+
}
|
|
414
381
|
} catch (err) {
|
|
415
382
|
const e = err instanceof TroveError ? err : TroveError.unauthorized('Authentication failed');
|
|
416
383
|
return withChallenge(new Response(JSON.stringify(e.toJSON()), { status: e.status, headers: { 'content-type': 'application/json', 'x-content-type-options': 'nosniff' } }), req);
|
|
@@ -422,7 +389,13 @@ export async function createServer(config = {}) {
|
|
|
422
389
|
// locator — nothing recorded what a route used, so nothing stopped it
|
|
423
390
|
// reaching for more.
|
|
424
391
|
container: engine.container,
|
|
425
|
-
|
|
392
|
+
// `mcp` is the one exception, and it says so here rather than being engineered
|
|
393
|
+
// around: it is constructed AFTER the container, so a route that wanted it would
|
|
394
|
+
// need `backgroundWork`-style late binding for a single field. `auth` used to ride
|
|
395
|
+
// along too, and /api/capabilities — its only reader — already declares it as a
|
|
396
|
+
// dep, so it arrived twice by two mechanisms and the route table understated by
|
|
397
|
+
// one what that endpoint touches.
|
|
398
|
+
config, principal, grant, mcp, rateLimiter,
|
|
426
399
|
});
|
|
427
400
|
// A route can refuse on its own (a token that verified but names nobody we know,
|
|
428
401
|
// a session that expired between calls). Whatever refused, the answer to "so
|
|
@@ -482,6 +455,11 @@ export async function createServer(config = {}) {
|
|
|
482
455
|
const out = { swept: false, purged: 0, scans: [], notified: 0, storage: 0 };
|
|
483
456
|
await vfs.uploads.sweepExpired(Date.now());
|
|
484
457
|
await sidecar.sweep();
|
|
458
|
+
// Rate-limit buckets. One key per subject per class per window, and nothing else
|
|
459
|
+
// removes them — the window passing makes a bucket unreachable, not absent. The object
|
|
460
|
+
// guard is real (there is no limiter when limiting is off); the method is not optional,
|
|
461
|
+
// because both stores have one.
|
|
462
|
+
await rateLimiter?.store.sweep().catch(() => {});
|
|
485
463
|
// Mentions are batched and drained on an interval — a timer, and a timer registered
|
|
486
464
|
// during a request does not outlive it on Workers, where the adapter switches the
|
|
487
465
|
// flusher off for exactly that reason. Nothing else called flush, so on that runtime
|
|
@@ -495,7 +473,7 @@ export async function createServer(config = {}) {
|
|
|
495
473
|
// Cheap (one preflight per collection) and the only thing that will ever notice a
|
|
496
474
|
// bucket policy that was fine yesterday, so it runs on every firing rather than
|
|
497
475
|
// waiting for someone to open the Activity panel and press a button.
|
|
498
|
-
out.storage = await
|
|
476
|
+
out.storage = await storageCheck.run({ origin: config.publicUrl || null })
|
|
499
477
|
.then((r) => r.checked)
|
|
500
478
|
.catch((e) => {
|
|
501
479
|
console.error('[trove] storage check failed', e);
|
|
@@ -503,8 +481,14 @@ export async function createServer(config = {}) {
|
|
|
503
481
|
});
|
|
504
482
|
const trashMs = (config.trashRetentionDays ?? 30) * 86400_000;
|
|
505
483
|
if (trashMs > 0) out.purged = (await vfs.purgeTrash({ before: Date.now() - trashMs }))?.purged || 0;
|
|
484
|
+
// Said out loud, on every runtime. This is the only thing in Trove that destroys data
|
|
485
|
+
// on a timer, so it is opt-outable (TROVE_TRASH_DAYS=0 keeps the trash forever) and it
|
|
486
|
+
// reports what it removed.
|
|
487
|
+
if (out.purged) {
|
|
488
|
+
console.log(`[trove] purged ${out.purged} item(s) from the trash after ${config.trashRetentionDays ?? 30} days`);
|
|
489
|
+
}
|
|
506
490
|
out.swept = true;
|
|
507
|
-
|
|
491
|
+
|
|
508
492
|
// `all()` rather than `list(null)`. Maintenance has no user, and `list(null)` answers
|
|
509
493
|
// "what may the anonymous principal read" — which on any drive that is not open to the
|
|
510
494
|
// public is nothing, so the scheduled scan silently scanned no collection whatsoever.
|
|
@@ -515,17 +499,25 @@ export async function createServer(config = {}) {
|
|
|
515
499
|
if (!targets.length) return out;
|
|
516
500
|
// Share the budget across collections so one huge bucket can't starve the rest.
|
|
517
501
|
const each = Math.max(1000, Math.floor(budgetMs / targets.length));
|
|
518
|
-
|
|
519
|
-
const
|
|
520
|
-
|
|
502
|
+
if (scan) {
|
|
503
|
+
for (const c of targets) {
|
|
504
|
+
const r = await startScan(c.id, { reason: 'Scheduled', deadlineMs: each }).catch((e) => ({ error: e.message }));
|
|
505
|
+
out.scans.push({ collectionId: c.id, ...r });
|
|
506
|
+
}
|
|
521
507
|
}
|
|
522
508
|
|
|
523
|
-
// A rotation that has been started finishes on its own
|
|
524
|
-
//
|
|
509
|
+
// A rotation that has been started finishes on its own, and NOT gated on `scan`.
|
|
510
|
+
// Scanning is opt-in because it costs money on a bucket nobody asked us to walk; a
|
|
511
|
+
// rotation only touches collections someone explicitly put into rotation, and leaving
|
|
512
|
+
// one half-moved is worse than the work of finishing it. Tying the two together is how
|
|
513
|
+
// `POST /rotate` came to mint a key, report "running", and move nothing forever.
|
|
525
514
|
//
|
|
526
515
|
// Last, and out of what the scans left, because a rotation is elective and a scan is
|
|
527
516
|
// how the drive notices files that changed underneath it.
|
|
528
517
|
out.rotated = await stepRotations(targets, each);
|
|
518
|
+
for (const m of out.rotated) {
|
|
519
|
+
if (m.moved) console.log(`[trove] rotation ${m.collectionId}: ${m.moved} moved, ${m.status}`);
|
|
520
|
+
}
|
|
529
521
|
return out;
|
|
530
522
|
}
|
|
531
523
|
|
|
@@ -537,7 +529,7 @@ export async function createServer(config = {}) {
|
|
|
537
529
|
// inside a Durable Object want. `begin*` goes wherever `config.background` says,
|
|
538
530
|
// which for a front-line Worker isolate is the object rather than itself.
|
|
539
531
|
startScan, startReindex, beginScan: routeBeginScan, beginReindex: routeBeginReindex,
|
|
540
|
-
runMaintenance, checkStorage, rotation, mcp, auth, close };
|
|
532
|
+
runMaintenance, checkStorage: (opts) => storageCheck.run(opts), rotation, mcp, auth, close };
|
|
541
533
|
}
|
|
542
534
|
|
|
543
535
|
/**
|
|
@@ -588,12 +580,42 @@ export const SAMPLE_CSP = [
|
|
|
588
580
|
"connect-src 'self'",
|
|
589
581
|
].join('; ');
|
|
590
582
|
|
|
583
|
+
/**
|
|
584
|
+
* Say so when the limits cannot actually be enforced.
|
|
585
|
+
*
|
|
586
|
+
* In-memory counters are exact on a runtime where one process serves every request, and on
|
|
587
|
+
* Workers they count PER ISOLATE — so "60 a minute" becomes 60 times however many isolates
|
|
588
|
+
* the platform decided to run, which is not a limit. `startFlusher: false` is how the
|
|
589
|
+
* Worker adapter already says "timers do not survive a request here", which is the same
|
|
590
|
+
* fact about the same absence of a long-lived process.
|
|
591
|
+
*
|
|
592
|
+
* A warning rather than a refusal, because there is a correct answer at that layer —
|
|
593
|
+
* Cloudflare's own rate limiting, configured outside the app — and refusing to boot over a
|
|
594
|
+
* limit an operator may have handled elsewhere would be worse than saying it.
|
|
595
|
+
*/
|
|
596
|
+
function warnOnUnenforceableLimits(config = {}) {
|
|
597
|
+
const rl = config.rateLimit;
|
|
598
|
+
if (!rl?.enabled || rl.store === 'kv') return;
|
|
599
|
+
if (config.startFlusher !== false) return; // a long-lived process; memory counters are exact
|
|
600
|
+
console.warn(
|
|
601
|
+
'[trove] Rate limits are counted IN MEMORY on a runtime with no long-lived process, so '
|
|
602
|
+
+ 'they apply per isolate rather than per drive. Set TROVE_RATE_LIMIT_STORE=kv for one '
|
|
603
|
+
+ 'shared budget, or configure rate limiting at the edge.',
|
|
604
|
+
);
|
|
605
|
+
}
|
|
606
|
+
|
|
591
607
|
/**
|
|
592
608
|
* Warn (once, to the console) when a configuration is world-open — anonymous auth
|
|
593
609
|
* plus the default collection granting everyone every capability. Safe on
|
|
594
|
-
* localhost, dangerous when exposed.
|
|
610
|
+
* localhost, dangerous when exposed.
|
|
611
|
+
*
|
|
612
|
+
* Called from `createServer`, so it covers every adapter that exists and every one that
|
|
613
|
+
* will. It was called by the adapters, and the Worker adapter — the one whose default
|
|
614
|
+
* deploy is world-open on a public URL — was the one that did not call it. Not exported
|
|
615
|
+
* for the same reason: an export is a second way to reach this, and a second way is how
|
|
616
|
+
* one caller came to skip it.
|
|
595
617
|
*/
|
|
596
|
-
|
|
618
|
+
function warnOnOpenAccess(config = {}) {
|
|
597
619
|
const anon = !config.identity || config.identity.driver === 'anonymous' || config.identity === 'anonymous';
|
|
598
620
|
const open = config.collections !== false && config.defaultOpen !== false;
|
|
599
621
|
if (anon && open) {
|
|
@@ -838,6 +860,39 @@ export function configFromEnv(env = (typeof process !== 'undefined' ? process.en
|
|
|
838
860
|
// Per-file upload quota (bytes). Unbounded unless set.
|
|
839
861
|
if (env.TROVE_MAX_UPLOAD_BYTES) config.maxUploadBytes = Number(env.TROVE_MAX_UPLOAD_BYTES);
|
|
840
862
|
|
|
863
|
+
// What one caller may cost, per class of work. On by default with the generous defaults
|
|
864
|
+
// in core/rateLimit.js — a limit nobody turns on is not a limit, and these are chosen so
|
|
865
|
+
// that using the drive normally never reaches one.
|
|
866
|
+
//
|
|
867
|
+
// TROVE_RATE_LIMIT=off no limiting at all (a single-user drive on a laptop)
|
|
868
|
+
// TROVE_RATE_LIMIT_STORE=kv counters in the shared store, so every instance sees one
|
|
869
|
+
// budget. Costs a read and a write per limited request, and
|
|
870
|
+
// it is what a multi-instance or Workers deployment needs —
|
|
871
|
+
// see describeRateLimits for why memory is not enough there.
|
|
872
|
+
// TROVE_RATE_LIMITS='{"search":{"limit":20,"windowMs":60000}}' per-class overrides,
|
|
873
|
+
// merged over the defaults so naming one class keeps the rest.
|
|
874
|
+
config.rateLimit = { enabled: true, store: 'memory', limits: { ...DEFAULT_RATE_LIMITS } };
|
|
875
|
+
if (env.TROVE_RATE_LIMIT != null && /^(0|off|false|no)$/i.test(String(env.TROVE_RATE_LIMIT))) {
|
|
876
|
+
config.rateLimit.enabled = false;
|
|
877
|
+
}
|
|
878
|
+
if (env.TROVE_RATE_LIMIT_STORE) config.rateLimit.store = String(env.TROVE_RATE_LIMIT_STORE);
|
|
879
|
+
if (env.TROVE_RATE_LIMITS) {
|
|
880
|
+
// Refused loudly. A malformed limits document that fell back to the defaults would be
|
|
881
|
+
// an operator believing they had tightened something they had not.
|
|
882
|
+
let overrides;
|
|
883
|
+
try {
|
|
884
|
+
overrides = JSON.parse(env.TROVE_RATE_LIMITS);
|
|
885
|
+
} catch (err) {
|
|
886
|
+
throw TroveError.invalid(`TROVE_RATE_LIMITS is not valid JSON: ${err.message}`);
|
|
887
|
+
}
|
|
888
|
+
for (const [name, rule] of Object.entries(overrides)) {
|
|
889
|
+
if (!RATE_CLASSES.includes(name)) {
|
|
890
|
+
throw TroveError.invalid(`Unknown rate-limit class "${name}" — expected one of: ${RATE_CLASSES.join(', ')}`);
|
|
891
|
+
}
|
|
892
|
+
config.rateLimit.limits[name] = { ...config.rateLimit.limits[name], ...rule };
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
|
|
841
896
|
// Deny plugin API calls with no server install record (fully closes the "any client
|
|
842
897
|
// can name any pluginId" gap). Off by default for back-compat with pre-existing
|
|
843
898
|
// local-only installs; flip on once clients have re-uploaded their account plugins.
|
|
@@ -17,19 +17,11 @@ import { TroveError, challengeHeaders, protectedResourceMetadata } from '@3sln/t
|
|
|
17
17
|
import { McpServer, rpcError, JSONRPC_ERRORS } from './protocol.js';
|
|
18
18
|
import { registerTroveTools } from './tools.js';
|
|
19
19
|
import { mcpConfigFromEnv, mcpResourceUri } from './auth.js';
|
|
20
|
-
import { crossSiteRefusal } from '../router.js';
|
|
20
|
+
import { crossSiteRefusal, corsOriginFor } from '../router.js';
|
|
21
21
|
import { leaseScope } from '../scope.js';
|
|
22
22
|
|
|
23
23
|
const MAX_BODY_BYTES = 1024 * 1024;
|
|
24
24
|
|
|
25
|
-
/** The CORS origin to echo, or null for "no cross-origin access" — same rule as the API. */
|
|
26
|
-
function allowedOrigin(configured, reqOrigin) {
|
|
27
|
-
if (!configured) return null;
|
|
28
|
-
if (configured === '*') return '*';
|
|
29
|
-
const allowed = String(configured).split(',').map((s) => s.trim()).filter(Boolean);
|
|
30
|
-
return reqOrigin && allowed.includes(reqOrigin) ? reqOrigin : null;
|
|
31
|
-
}
|
|
32
|
-
|
|
33
25
|
export { McpServer, mcpConfigFromEnv, mcpResourceUri };
|
|
34
26
|
|
|
35
27
|
export function createMcpServer({ name = 'trove', version = '0.0.1' } = {}) {
|
|
@@ -69,7 +61,7 @@ function authRequired(cfg, identity) {
|
|
|
69
61
|
* @param {object} deps
|
|
70
62
|
* @param {object} deps.auth the drive's resolved auth discovery (see resolveAuthDiscovery)
|
|
71
63
|
*/
|
|
72
|
-
export function createMcpHandler({ vfs, collections, container, identity, config = {}, auth = {}, version } = {}) {
|
|
64
|
+
export function createMcpHandler({ vfs, collections, container, identity, config = {}, auth = {}, version, rateLimiter = null } = {}) {
|
|
73
65
|
const cfg = { path: '/mcp', enabled: true, ...mcpConfigFromEnv(config.env || {}), ...(config.mcp || {}) };
|
|
74
66
|
if (cfg.enabled === false) return null;
|
|
75
67
|
const server = createMcpServer({ version });
|
|
@@ -106,7 +98,7 @@ export function createMcpHandler({ vfs, collections, container, identity, config
|
|
|
106
98
|
// unreadable to the attacker, but the deletions still happen. Agents are not
|
|
107
99
|
// browsers and do not need CORS at all, so this follows the same
|
|
108
100
|
// TROVE_CORS_ORIGIN allowlist the JSON API does, and stays off by default.
|
|
109
|
-
const origin =
|
|
101
|
+
const origin = corsOriginFor(config?.corsOrigin, req.headers.get('origin'));
|
|
110
102
|
if (!origin) return new Response(null, { status: 204 });
|
|
111
103
|
return new Response(null, {
|
|
112
104
|
status: 204,
|
|
@@ -180,7 +172,10 @@ export function createMcpHandler({ vfs, collections, container, identity, config
|
|
|
180
172
|
// node or collection HANDLE and operates through it, so there is no MCP-shaped path
|
|
181
173
|
// around the collection ACL and no unrestricted `vfs` sitting in a tool body.
|
|
182
174
|
const scope = leaseScope(container, principal);
|
|
183
|
-
|
|
175
|
+
// The limiter reaches this surface too. An agent holding somebody's token can search
|
|
176
|
+
// as fast as it likes otherwise, and a search is the paid one — "exactly as privileged
|
|
177
|
+
// as the person whose token it holds" has to include how much they can spend.
|
|
178
|
+
const ctx = { vfs, collections, principal, config, access: scope.access, rateLimiter };
|
|
184
179
|
|
|
185
180
|
try {
|
|
186
181
|
// A batch is an array. Notifications inside it contribute nothing to the reply,
|