@3sln/trove 0.0.12 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/package.json +2 -1
- package/packages/core/src/collections/index.js +71 -10
- package/packages/core/src/encryption/keys.js +4 -3
- package/packages/core/src/encryption/policy.js +14 -7
- package/packages/core/src/encryption/rotation.js +46 -12
- package/packages/core/src/errors.js +8 -0
- package/packages/core/src/index.js +8 -2
- package/packages/core/src/indexers/contribution.js +18 -17
- package/packages/core/src/indexing.js +8 -4
- package/packages/core/src/issues.js +8 -0
- package/packages/core/src/links.js +21 -9
- package/packages/core/src/metadata/interface.js +22 -0
- package/packages/core/src/metadata/memory.js +15 -0
- package/packages/core/src/metadata/sqlite.js +23 -1
- package/packages/core/src/plugins/contributions.js +13 -0
- package/packages/core/src/rateLimit.js +253 -0
- package/packages/core/src/scan.js +44 -16
- package/packages/core/src/sidecar/document.js +18 -14
- package/packages/core/src/sidecar/index.js +9 -10
- package/packages/core/src/sidecar/manager.js +10 -15
- package/packages/core/src/uploads.js +40 -15
- package/packages/core/src/vfs.js +59 -34
- package/packages/plugin-sdk/src/browser.js +155 -0
- package/packages/plugin-sdk/src/protocol.js +13 -1
- package/packages/server/src/adapters/bun.js +1 -2
- package/packages/server/src/adapters/node.js +1 -2
- package/packages/server/src/engine/index.js +1 -1
- package/packages/server/src/engine/providers/access.js +17 -34
- package/packages/server/src/engine/providers/core.js +112 -14
- package/packages/server/src/index.js +145 -103
- package/packages/server/src/mcp/index.js +7 -12
- package/packages/server/src/mcp/tools.js +24 -29
- package/packages/server/src/router.js +31 -8
- package/packages/server/src/routes.js +74 -113
- package/packages/server/src/scope.js +78 -0
- package/packages/web/dist/assets/main-828yzsr7.js +511 -0
- package/packages/web/dist/assets/main-828yzsr7.js.map +120 -0
- package/packages/web/dist/index.html +1 -1
- package/packages/web/dist/sw.js +1 -1
- package/packages/web/src/bl/actions.js +226 -104
- package/packages/web/src/bl/activity.js +32 -82
- package/packages/web/src/bl/index.js +23 -11
- package/packages/web/src/bl/launcher.js +2 -9
- package/packages/web/src/bl/links.js +31 -18
- package/packages/web/src/bl/offline.js +24 -10
- package/packages/web/src/bl/pluginInstall.js +29 -26
- package/packages/web/src/bl/queries.js +76 -68
- package/packages/web/src/bl/services.js +22 -14
- package/packages/web/src/bl/social.js +39 -25
- package/packages/web/src/bl/state.js +22 -4
- package/packages/web/src/bl/viewState.js +9 -0
- package/packages/web/src/bl/watchQuery.js +1 -8
- package/packages/web/src/dispatch.js +26 -0
- package/packages/web/src/platform/api.js +32 -0
- package/packages/web/src/platform/commands.js +14 -15
- package/packages/web/src/platform/context.js +3 -58
- package/packages/web/src/platform/contributions.js +1 -35
- package/packages/web/src/platform/fileChunks.js +281 -0
- package/packages/web/src/platform/index.js +8 -9
- package/packages/web/src/platform/mediaUrls.js +10 -0
- package/packages/web/src/platform/navigation.js +35 -21
- package/packages/web/src/platform/pickers.js +39 -19
- package/packages/web/src/platform/pluginHost.js +4 -1
- package/packages/web/src/platform/pluginRpc.js +47 -2
- package/packages/web/src/platform/spatialNav.js +1 -1
- package/packages/web/src/platform/viewport.js +14 -6
- package/packages/web/src/platform/voiceSearch.js +17 -9
- package/packages/web/src/platform/whenclause.js +9 -19
- package/packages/web/src/runtime.js +2 -37
- package/packages/web/src/ui/activate.js +18 -2
- package/packages/web/src/ui/components/activityPanel.js +1 -1
- package/packages/web/src/ui/components/adminView.js +4 -7
- package/packages/web/src/ui/components/editorArea.js +10 -1
- package/packages/web/src/ui/components/launcher.js +29 -13
- package/packages/web/src/ui/components/openers/index.js +17 -22
- package/packages/web/src/ui/components/openers/markdown.js +2 -2
- package/packages/web/src/ui/components/overlays.js +31 -5
- package/packages/web/src/ui/components/pluginReview.js +7 -4
- package/packages/web/src/ui/components/pluginsView.js +1 -1
- package/packages/web/src/ui/components/settingsView.js +30 -27
- package/packages/web/src/ui/components/statusBar.js +8 -6
- package/packages/web/src/ui/components/views/grid.js +2 -2
- package/packages/web/src/ui/components/views/parts.js +1 -1
- package/packages/web/src/ui/compositions/workbench.js +20 -6
- package/packages/web/src/ui/sanitize.js +0 -5
- package/packages/web/src/workbench.js +37 -19
- package/packages/web/dist/assets/main-c9dnnnc6.js +0 -356
- package/packages/web/dist/assets/main-c9dnnnc6.js.map +0 -118
package/README.md
CHANGED
|
@@ -503,8 +503,11 @@ the same values as config fields instead.
|
|
|
503
503
|
| `TROVE_JWT_ISSUER` / `_AUDIENCE` / `_ALGS` | — | claim checks after the signature passes |
|
|
504
504
|
| `TROVE_AUTH_SERVER` | the JWT issuer | where refused clients are sent to sign in |
|
|
505
505
|
| `TROVE_ADMINS` | — | comma-separated ids with whole-drive rights |
|
|
506
|
+
| **rate limits** | | |
|
|
507
|
+
| `TROVE_RATE_LIMIT` | on | `off` to meter nothing |
|
|
508
|
+
| `TROVE_RATE_LIMIT_STORE` | `memory` | `kv` for one budget across instances (needed on Workers) |
|
|
509
|
+
| `TROVE_RATE_LIMITS` | — | JSON per class, e.g. `{"search":{"limit":20,"windowMs":60000}}` |
|
|
506
510
|
| **collections** | | |
|
|
507
|
-
| `TROVE_COLLECTIONS` | on | `false` for one open store with no ACLs |
|
|
508
511
|
| `TROVE_DEFAULT_OPEN` | `true` | **set `false`** before exposing it |
|
|
509
512
|
| `TROVE_COLLECTION_CREATOR_ROLES` | — | roles allowed to create collections |
|
|
510
513
|
| **agents** | | |
|
|
@@ -1193,8 +1196,17 @@ packages/
|
|
|
1193
1196
|
plugin-sdk/ @3sln/trove/plugin-sdk — the iframe-side plugin API + RPC
|
|
1194
1197
|
create-trove/ @3sln/create-trove — the scaffolder; the one directory that is
|
|
1195
1198
|
its own published package rather than part of @3sln/trove
|
|
1199
|
+
plugins/
|
|
1200
|
+
audiobook/ a plugin we ship, built into a signed zip
|
|
1196
1201
|
```
|
|
1197
1202
|
|
|
1203
|
+
`plugins/` is a BUILD location rather than a workspace, and the difference is the point: a
|
|
1204
|
+
plugin's artifact is a zip the server independently re-parses and installs, never an npm
|
|
1205
|
+
dependency and never imported from `packages/`. `bun run build:plugins` produces the same
|
|
1206
|
+
bytes a user would drag into the install dialog — there is no privileged path for our own.
|
|
1207
|
+
Nothing there is preinstalled: a drive with a plugin its owner did not choose is a drive
|
|
1208
|
+
with a capability grant its owner did not make. See `plugins/README.md`.
|
|
1209
|
+
|
|
1198
1210
|
## Releasing
|
|
1199
1211
|
|
|
1200
1212
|
Two packages, one version:
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@3sln/trove",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.13",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Trove — a self-hostable, plugin-extensible Google Drive. Semantic search, pluggable storage (S3 / filesystem / NAS), and a VS Code-style contribution system with sandboxed plugins.",
|
|
6
6
|
"repository": {
|
|
@@ -41,6 +41,7 @@
|
|
|
41
41
|
"scripts": {
|
|
42
42
|
"dev": "web-dev-server --config packages/web/web-dev-server.config.mjs",
|
|
43
43
|
"build:web": "cd packages/web && bun build.mjs",
|
|
44
|
+
"build:plugins": "bun plugins/build.mjs",
|
|
44
45
|
"serve": "bun packages/server/src/adapters/bun.js",
|
|
45
46
|
"serve:node": "node packages/server/src/adapters/node.js",
|
|
46
47
|
"test": "bun test",
|
|
@@ -80,12 +80,59 @@ export class CollectionService {
|
|
|
80
80
|
.sort((a, b) => (a.name || '').localeCompare(b.name || ''));
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
+
// --- the same questions, asked of an API key ---------------------------------
|
|
84
|
+
//
|
|
85
|
+
// A key is authority that ARRIVED WITH THE REQUEST. There is nobody to look up in the
|
|
86
|
+
// ACL, and `principal` is null on a key request — so asking the principal side answers
|
|
87
|
+
// for the ANONYMOUS caller, which is wrong in both directions. On a locked drive a
|
|
88
|
+
// correctly-scoped key is told it can read nothing; on a `defaultOpen` drive the
|
|
89
|
+
// `anyone` grant lets a key scoped to one collection read and write every one of them.
|
|
90
|
+
//
|
|
91
|
+
// Decided from the key ALONE, never unioned with whatever session happens to be
|
|
92
|
+
// attached — the same rule engine/providers/access.js states for node and collection
|
|
93
|
+
// handles, and the reason a weak key cannot borrow a strong session.
|
|
94
|
+
|
|
95
|
+
/** The collections an API key may read. `list`'s twin. */
|
|
96
|
+
async listForGrant(grant) {
|
|
97
|
+
const rows = await this.kv.list(NS);
|
|
98
|
+
return rows
|
|
99
|
+
.map((r) => r.value)
|
|
100
|
+
.filter((c) => c && grant.can(c.id, 'read'))
|
|
101
|
+
.map((c) => this.describeForGrant(c, grant))
|
|
102
|
+
.sort((a, b) => (a.name || '').localeCompare(b.name || ''));
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** `assert`'s twin. Refuses with what the KEY lacks, not with what a user lacks. */
|
|
106
|
+
async assertForGrant(grant, collectionId, capability) {
|
|
107
|
+
const c = await this.get(collectionId);
|
|
108
|
+
if (!grant.can(collectionId, capability)) {
|
|
109
|
+
throw TroveError.forbidden(`This API key does not hold "${capability}" on collection "${c.name}"`);
|
|
110
|
+
}
|
|
111
|
+
return c;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* `hasWholeDrive`'s twin: a key that can read and write every collection that exists.
|
|
116
|
+
*
|
|
117
|
+
* Same definition, so the answer does not depend on which credential asked — and it
|
|
118
|
+
* shrinks the moment a collection the key does not name is created, which is the safe
|
|
119
|
+
* direction. A `*`-scoped key satisfies it by construction. There is no admin-list
|
|
120
|
+
* shortcut, because a key is never a named admin: the drive-wide verbs that grant new
|
|
121
|
+
* power (minting keys, installing a server indexer) are gated by `requireHumanAdmin`
|
|
122
|
+
* and `isAdmin` and stay out of a key's reach entirely.
|
|
123
|
+
*/
|
|
124
|
+
async grantHasWholeDrive(grant) {
|
|
125
|
+
const all = (await this.kv.list(NS)).map((r) => r.value).filter(Boolean);
|
|
126
|
+
if (!all.length) return false;
|
|
127
|
+
return all.every((c) => grant.can(c.id, 'read') && grant.can(c.id, 'write'));
|
|
128
|
+
}
|
|
129
|
+
|
|
83
130
|
/**
|
|
84
131
|
* The encryption settings for a collection, generating a key the first time.
|
|
85
132
|
*
|
|
86
133
|
* The key is generated, not derived from anything a user types. A passphrase would buy
|
|
87
|
-
* nothing: the server knows the key regardless — it
|
|
88
|
-
*
|
|
134
|
+
* nothing: the server knows the key regardless — it seals what arrives and decrypts what
|
|
135
|
+
* is read — so there is no protection to gain from the user holding it, and every cost
|
|
89
136
|
* would still apply. A random 256-bit key cannot be forgotten, guessed, or shoulder-read,
|
|
90
137
|
* and needs no prompt in front of the collection.
|
|
91
138
|
*
|
|
@@ -117,9 +164,14 @@ export class CollectionService {
|
|
|
117
164
|
* whole reason every object carries a fingerprint. Retiring a key is only safe once
|
|
118
165
|
* nothing names it any more.
|
|
119
166
|
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
122
|
-
*
|
|
167
|
+
* SERVER-SIDE ONLY, AND ITS RESULT MUST NEVER REACH A RESPONSE. That is the actual
|
|
168
|
+
* invariant and it was previously unstated — worse, contradicted, by comments describing
|
|
169
|
+
* a transfer plan that handed the key to a client. That trade came off (see
|
|
170
|
+
* `#planEncryption`, which answers `sealedBy: 'server'`): encrypted collections pass
|
|
171
|
+
* through the drive in both directions, and every call site of this is inside the server.
|
|
172
|
+
*
|
|
173
|
+
* Callers are sealing on the way in, decryption on the way out, indexing, and rotation.
|
|
174
|
+
* Not reachable through `describe`, which returns a fingerprint.
|
|
123
175
|
*
|
|
124
176
|
* @param {string} collectionId
|
|
125
177
|
* @param {string} [fingerprint] which key; omitted means the current one
|
|
@@ -218,15 +270,24 @@ export class CollectionService {
|
|
|
218
270
|
|
|
219
271
|
/** A safe, principal-scoped view (no secrets). */
|
|
220
272
|
describe(c, principal) {
|
|
221
|
-
|
|
273
|
+
return this.#describe(c, this.capabilities(principal, c));
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/** The same record, with the capabilities an API KEY holds rather than a principal's. */
|
|
277
|
+
describeForGrant(c, grant) {
|
|
278
|
+
return this.#describe(c, grant.capabilitiesFor(c.id));
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
#describe(c, held) {
|
|
282
|
+
const caps = [...held];
|
|
222
283
|
return {
|
|
223
284
|
id: c.id, name: c.name, description: c.description || '',
|
|
224
285
|
driver: c.store?.driver, system: !!c.system,
|
|
225
286
|
capabilities: caps, createdAt: c.createdAt,
|
|
226
|
-
//
|
|
227
|
-
//
|
|
228
|
-
// without
|
|
229
|
-
//
|
|
287
|
+
// `{ enabled, fingerprint, rules }` and nothing more — see describeEncryption. Safe
|
|
288
|
+
// to hand to anyone who can see the collection, because a fingerprint names a key
|
|
289
|
+
// without being it. Null when the collection is not encrypted, so a client never has
|
|
290
|
+
// to ask a second question to find out.
|
|
230
291
|
encryption: describeEncryption(c.encryption),
|
|
231
292
|
};
|
|
232
293
|
}
|
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
// The threat model is the STORAGE HOST, and only that: the bucket holds ciphertext, so a
|
|
4
4
|
// leaked bucket credential, a public-bucket misconfiguration, or a storage vendor who is
|
|
5
5
|
// not the compute vendor learns sizes and timestamps and no content. The server holds the
|
|
6
|
-
// key — it has to, in order to
|
|
7
|
-
// is explicitly not end-to-end and does not pretend to be.
|
|
6
|
+
// key — it has to, in order to seal what arrives, decrypt what is read, and decrypt for an
|
|
7
|
+
// indexer — so this is explicitly not end-to-end and does not pretend to be. The key does
|
|
8
|
+
// not leave the server: a client sends and receives plaintext through the drive.
|
|
8
9
|
//
|
|
9
10
|
// Which is why the key is GENERATED rather than derived from something a user types.
|
|
10
11
|
// A passphrase would buy nothing here: the server knows the key either way, so there is no
|
|
@@ -15,7 +16,7 @@
|
|
|
15
16
|
// passphrase a person would choose.
|
|
16
17
|
//
|
|
17
18
|
// Access to the key is therefore an ACCESS-CONTROL question, not a knowledge one: whoever
|
|
18
|
-
// may read the collection may
|
|
19
|
+
// may read the collection may read its contents, and the drive decrypts on their behalf.
|
|
19
20
|
//
|
|
20
21
|
// If this ever becomes end-to-end, the change is confined to where the key comes from and
|
|
21
22
|
// who is allowed it. The envelope, the fingerprint, and every stored object are unchanged.
|
|
@@ -18,8 +18,8 @@ import { TroveError } from '../errors.js';
|
|
|
18
18
|
* What a collection stores about its encryption.
|
|
19
19
|
*
|
|
20
20
|
* The fingerprint is safe to show anyone who may see the collection: it names the key
|
|
21
|
-
* without being it. The key itself is never part of this
|
|
22
|
-
*
|
|
21
|
+
* without being it. The key itself is never part of this, and never reaches a client at
|
|
22
|
+
* all — see `#planEncryption`, which answers `sealedBy: 'server'`.
|
|
23
23
|
*
|
|
24
24
|
* @typedef {object} EncryptionConfig
|
|
25
25
|
* @property {boolean} enabled
|
|
@@ -95,12 +95,19 @@ export function shouldEncrypt(encryption, { name = '', contentType = '' } = {})
|
|
|
95
95
|
}
|
|
96
96
|
|
|
97
97
|
/**
|
|
98
|
-
* What a client is told about a collection's encryption
|
|
98
|
+
* What a client is told about a collection's encryption: `{ enabled, fingerprint, rules }`.
|
|
99
99
|
*
|
|
100
|
-
* Enough to know that objects here are sealed and which key seals them
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
100
|
+
* Enough to know that objects here are sealed and which key seals them. There is no salt
|
|
101
|
+
* and no KDF material, because there is no passphrase — the key is generated, and this
|
|
102
|
+
* comment used to explain why handing out the KDF parameters was safe, describing a design
|
|
103
|
+
* that no longer exists in the subsystem where being wrong is most expensive.
|
|
104
|
+
*
|
|
105
|
+
* A fingerprint names a key without being it, which is the whole reason it is safe to
|
|
106
|
+
* publish: it is what lets a stored object say which key opens it, and a client say which
|
|
107
|
+
* key a collection is currently on, without either of them holding one.
|
|
108
|
+
*
|
|
109
|
+
* There is no "locked" state and nothing to prompt for. The drive seals and unseals; a
|
|
110
|
+
* client never decrypts, so there is nothing it needs beyond knowing that it happens.
|
|
104
111
|
*/
|
|
105
112
|
export function describeEncryption(encryption) {
|
|
106
113
|
if (!encryption?.enabled) return null;
|
|
@@ -23,9 +23,15 @@
|
|
|
23
23
|
// on it — not when a counter says the job is done. A count can be wrong; a pass that finds
|
|
24
24
|
// nothing cannot be. That also handles the awkward case of an object uploaded onto the old
|
|
25
25
|
// key by a request that was in flight when the rotation started.
|
|
26
|
+
//
|
|
27
|
+
// WHICH IS ONLY SOUND IF THE PASS SEES EVERYTHING THE KEY OPENS. The trash keeps its
|
|
28
|
+
// bytes, so a soft-deleted object is still sealed and still needs its key on the day
|
|
29
|
+
// someone restores it. The walk therefore sources from `listSealed`, which spans live and
|
|
30
|
+
// trashed, and not from `listItems`, which is live-only by design. Retiring on a pass over
|
|
31
|
+
// a subset is not observation, it is a guess — and the guess destroys data silently.
|
|
26
32
|
|
|
27
33
|
import { TroveError } from '../errors.js';
|
|
28
|
-
import { encryptStream, decodeHeader } from './envelope.js';
|
|
34
|
+
import { encryptStream, decodeHeader, HEADER_BYTES } from './envelope.js';
|
|
29
35
|
import { fromHex, fingerprint, toHex } from './keys.js';
|
|
30
36
|
|
|
31
37
|
const NS = 'rotations';
|
|
@@ -197,12 +203,16 @@ export class RotationService {
|
|
|
197
203
|
}
|
|
198
204
|
|
|
199
205
|
for (;;) {
|
|
200
|
-
|
|
206
|
+
// `listSealed`, NOT `listItems`: the trash keeps its bytes, so a trashed object is
|
|
207
|
+
// still sealed with the old key, and a walk that cannot see it would retire that key
|
|
208
|
+
// out from under every future restore. The store applies the sealed predicate too,
|
|
209
|
+
// so a mostly-plaintext collection is one page rather than hundreds to skip.
|
|
210
|
+
const page = await this.vfs.metadata.listSealed(collectionId, { cursor, limit: 50 });
|
|
201
211
|
const items = page.items || [];
|
|
202
212
|
for (const node of items) {
|
|
203
|
-
// Only
|
|
204
|
-
//
|
|
205
|
-
if (
|
|
213
|
+
// Only items not already on the current key. This is what makes re-running a slice
|
|
214
|
+
// free rather than destructive.
|
|
215
|
+
if (node.encryption.fingerprint === state.to) continue;
|
|
206
216
|
sawStragglers = true;
|
|
207
217
|
try {
|
|
208
218
|
const r = await this.#moveSlice(node, key, fp, null, { deadline, now });
|
|
@@ -275,14 +285,13 @@ export class RotationService {
|
|
|
275
285
|
*/
|
|
276
286
|
async #moveSlice(node, newKey, newFingerprint, inflight, { deadline, now }) {
|
|
277
287
|
const storage = await this.vfs.storageFor(node.collectionId);
|
|
278
|
-
const chunkSize = node
|
|
279
|
-
const plaintextSize = node.size;
|
|
288
|
+
const { plaintextSize, chunkSize } = await this.#geometryOf(node, storage);
|
|
280
289
|
|
|
281
290
|
// A store without multipart is a local one — filesystem, memory — where the whole
|
|
282
291
|
// object is already within reach. There is nothing to check point against, and nothing
|
|
283
292
|
// that needs it: the budget that makes this necessary is a network one.
|
|
284
293
|
if (!storage.capabilities?.multipart) {
|
|
285
|
-
await this.#moveWhole(node, newKey, newFingerprint, storage, chunkSize);
|
|
294
|
+
await this.#moveWhole(node, newKey, newFingerprint, storage, { plaintextSize, chunkSize });
|
|
286
295
|
return { done: true, inflight: null };
|
|
287
296
|
}
|
|
288
297
|
|
|
@@ -307,7 +316,7 @@ export class RotationService {
|
|
|
307
316
|
const end = Math.min(start + chunksPerPart * chunkSize, plaintextSize) - 1;
|
|
308
317
|
const last = end + 1 >= plaintextSize;
|
|
309
318
|
|
|
310
|
-
const read = await this.vfs.
|
|
319
|
+
const read = await this.vfs.readNode(node, { range: { start, end } });
|
|
311
320
|
const sealed = await encryptStream(newKey, read.stream, {
|
|
312
321
|
fingerprint: newFingerprint,
|
|
313
322
|
plaintextSize,
|
|
@@ -368,13 +377,38 @@ export class RotationService {
|
|
|
368
377
|
return { done: true, inflight: null };
|
|
369
378
|
}
|
|
370
379
|
|
|
380
|
+
/**
|
|
381
|
+
* The geometry of the object as the OBJECT states it.
|
|
382
|
+
*
|
|
383
|
+
* The Vfs already answers the authority question and answers it this way: the item's
|
|
384
|
+
* record says which key sealed it and at what chunk size, and that is what makes a
|
|
385
|
+
* listing cheap — but it is a copy, and a copy can be stale (restored from a backup,
|
|
386
|
+
* adopted by a scan, written by another version).
|
|
387
|
+
*
|
|
388
|
+
* The two arms here disagreed. The multipart arm took `node.size`; the whole-object arm
|
|
389
|
+
* took the envelope's, so which answer a drive got depended on whether its store supports
|
|
390
|
+
* multipart. A stale-LARGE `node.size` makes every slice raise "Expected N bytes…
|
|
391
|
+
* received M", so the rotation can never reach `done` and the old key is never retired; a
|
|
392
|
+
* stale-SMALL one seals a header that decrypts to the wrong length forever.
|
|
393
|
+
*
|
|
394
|
+
* One 44-byte read against a multi-megabyte transfer.
|
|
395
|
+
*/
|
|
396
|
+
async #geometryOf(node, storage) {
|
|
397
|
+
const head = await storage.get(node.storageKey, { range: { start: 0, end: HEADER_BYTES - 1 } });
|
|
398
|
+
const header = decodeHeader(new Uint8Array(await new Response(head.stream).arrayBuffer()));
|
|
399
|
+
return {
|
|
400
|
+
plaintextSize: header.plaintextSize ?? node.size,
|
|
401
|
+
chunkSize: header.chunkSize ?? node.encryption.chunkSize,
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
|
|
371
405
|
/** The whole object in one put, for a store that cannot do multipart. */
|
|
372
|
-
async #moveWhole(node, newKey, newFingerprint, storage, chunkSize) {
|
|
406
|
+
async #moveWhole(node, newKey, newFingerprint, storage, { plaintextSize, chunkSize }) {
|
|
373
407
|
const nextKey = rotatedKey(node.storageKey);
|
|
374
|
-
const read = await this.vfs.
|
|
408
|
+
const read = await this.vfs.readNode(node);
|
|
375
409
|
const sealed = await encryptStream(newKey, read.stream, {
|
|
376
410
|
fingerprint: newFingerprint,
|
|
377
|
-
plaintextSize
|
|
411
|
+
plaintextSize,
|
|
378
412
|
chunkSize,
|
|
379
413
|
});
|
|
380
414
|
await storage.put(nextKey, new Uint8Array(await new Response(sealed).arrayBuffer()), {
|
|
@@ -118,6 +118,14 @@ export class TroveError extends Error {
|
|
|
118
118
|
static forbidden(message = 'Forbidden', opts) {
|
|
119
119
|
return new TroveError(ErrorCode.FORBIDDEN, message, opts);
|
|
120
120
|
}
|
|
121
|
+
/**
|
|
122
|
+
* Rate limited — 429, and retryable, which is what tells it apart from being out of
|
|
123
|
+
* disk. See the constructor: QUOTA is 507 when it is NOT retryable, because telling a
|
|
124
|
+
* client to retry against a full volume sends it into a loop only a human can end.
|
|
125
|
+
*/
|
|
126
|
+
static rateLimited(message, opts) {
|
|
127
|
+
return new TroveError(ErrorCode.QUOTA, message, { retryable: true, ...opts });
|
|
128
|
+
}
|
|
121
129
|
static internal(message = 'Internal error', opts) {
|
|
122
130
|
return new TroveError(ErrorCode.INTERNAL, message, opts);
|
|
123
131
|
}
|
|
@@ -60,7 +60,7 @@ export { TaskRegistry } from './tasks.js';
|
|
|
60
60
|
export { IssueRegistry, issueId } from './issues.js';
|
|
61
61
|
// Reconcile a collection against what its store actually holds (changes made outside Trove).
|
|
62
62
|
export { CollectionScanner } from './scan.js';
|
|
63
|
-
export {
|
|
63
|
+
export { clampContribution, clampTagValue, DEFAULT_CAPS } from './indexers/contribution.js';
|
|
64
64
|
|
|
65
65
|
// Cloudflare Vectorize — first-class pluggable vector DB.
|
|
66
66
|
export { VectorizeVectorStore } from './search/vectorize.js';
|
|
@@ -68,6 +68,12 @@ export { VectorizeVectorStore } from './search/vectorize.js';
|
|
|
68
68
|
// Server-side key/value store (subscriptions, inboxes, profiles).
|
|
69
69
|
export { KeyValueStore, MemoryKV, SqliteKV } from './kv.js';
|
|
70
70
|
export { SignedUrls, resolveUrlSecret, URL_PURPOSES } from './signedUrls.js';
|
|
71
|
+
// What one caller may cost, per class of work — see rateLimit.js for why classes rather
|
|
72
|
+
// than one number, and for which runtimes can enforce a limit at all.
|
|
73
|
+
export {
|
|
74
|
+
RateLimiter, MemoryRateStore, KvRateStore, rateSubject, describeRateLimits,
|
|
75
|
+
DEFAULT_RATE_LIMITS, RATE_CLASSES,
|
|
76
|
+
} from './rateLimit.js';
|
|
71
77
|
// API keys: capability without identity. CapabilityProvider is the injection point —
|
|
72
78
|
// the counterpart to IdentityProvider, for credentials that say what rather than who.
|
|
73
79
|
export {
|
|
@@ -91,7 +97,7 @@ export {
|
|
|
91
97
|
normalizeServers, resolveAuthDiscovery, usableAuthServer, headerSafe,
|
|
92
98
|
} from './identity/discovery.js';
|
|
93
99
|
|
|
94
|
-
// Sidecar documents: conversations, tags,
|
|
100
|
+
// Sidecar documents: conversations, tags, subscriptions (CRDT, cold-in-S3).
|
|
95
101
|
export { SidecarService, SidecarStore, SidecarManager } from './sidecar/index.js';
|
|
96
102
|
export * as sidecarOps from './sidecar/document.js';
|
|
97
103
|
|
|
@@ -8,11 +8,24 @@
|
|
|
8
8
|
// tags filterable key/values, merged into the item's queryable tags
|
|
9
9
|
// metadata arbitrary structured data (an audiobook's chapter index, …)
|
|
10
10
|
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
11
|
+
// TWO SENSES OF "CONTRIBUTION", and this is the other one. Here it is per-node ENRICHMENT
|
|
12
|
+
// produced at runtime and addressed by `contributorId`. The first sense lives in
|
|
13
|
+
// core/src/plugins/contributions.js: an EXTENSION POINT declared in a manifest and
|
|
14
|
+
// addressed by URI. A plugin declares an `indexer` contribution (that sense) which at
|
|
15
|
+
// runtime produces contributions (this one).
|
|
16
|
+
//
|
|
17
|
+
// The rule: that sense is DECLARED, static, and named by URI; this one is PRODUCED, per
|
|
18
|
+
// node, and named by contributor.
|
|
19
|
+
//
|
|
20
|
+
// `clampContribution` does the whole contract in one pass: it accepts the legacy
|
|
21
|
+
// `{ documents, facet }` spelling, decides the canonical shape, and applies the caps. It
|
|
22
|
+
// was two functions, and the second — `normalizeContribution` — had grown its own copy of
|
|
23
|
+
// the legacy mapping, so both halves of a contract that exists here BECAUSE it must not
|
|
24
|
+
// drift had drifted into both. At its only call site it could never see a legacy key,
|
|
25
|
+
// because clamp had already mapped it; all it contributed was three defaults.
|
|
26
|
+
//
|
|
27
|
+
// The caps used to live in the plugin isolate runtime, which meant they applied to one of
|
|
28
|
+
// the three producers. They are a property of the contribution, not of whoever made it.
|
|
16
29
|
|
|
17
30
|
/** Output caps applied to every contribution, whatever produced it. */
|
|
18
31
|
export const DEFAULT_CAPS = {
|
|
@@ -25,18 +38,6 @@ export const DEFAULT_CAPS = {
|
|
|
25
38
|
maxMetadataBytes: 256 * 1024, // JSON-serialized metadata
|
|
26
39
|
};
|
|
27
40
|
|
|
28
|
-
/**
|
|
29
|
-
* Normalize to the three canonical scopes, accepting the legacy `{ documents, facet }`
|
|
30
|
-
* shape (documents→semanticTexts, facet→metadata).
|
|
31
|
-
*/
|
|
32
|
-
export function normalizeContribution(c = {}) {
|
|
33
|
-
return {
|
|
34
|
-
semanticTexts: c.semanticTexts || c.documents || [],
|
|
35
|
-
tags: c.tags || null,
|
|
36
|
-
metadata: c.metadata || c.facet || null,
|
|
37
|
-
};
|
|
38
|
-
}
|
|
39
|
-
|
|
40
41
|
/**
|
|
41
42
|
* Clamp a contribution to `caps`, dropping (not erroring on) whatever exceeds them.
|
|
42
43
|
*
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
import { TroveError } from './errors.js';
|
|
13
13
|
import { readAll } from './util.js';
|
|
14
|
-
import {
|
|
14
|
+
import { clampContribution, DEFAULT_CAPS } from './indexers/contribution.js';
|
|
15
15
|
|
|
16
16
|
export class IndexingCoordinator {
|
|
17
17
|
/**
|
|
@@ -61,7 +61,11 @@ export class IndexingCoordinator {
|
|
|
61
61
|
* the one most exposed to untrusted code.
|
|
62
62
|
*/
|
|
63
63
|
async #applyContribution(nodeId, contributorId, contribution) {
|
|
64
|
-
|
|
64
|
+
// `clamp` alone. It already maps the legacy `documents`/`facet` keys itself, so
|
|
65
|
+
// `normalize` at this call site can never see one — all it contributed was three
|
|
66
|
+
// defaults, while contribution.js's header says both halves live there because they
|
|
67
|
+
// must not drift. They had already drifted into both functions.
|
|
68
|
+
const { semanticTexts = [], tags = null, metadata = null } = clampContribution(contribution, this.caps);
|
|
65
69
|
// ALWAYS write the search half, including when it is empty. `indexDocuments` clears
|
|
66
70
|
// this (node, contributor)'s prior docs before it writes, so passing [] is how you
|
|
67
71
|
// say "this contributor has nothing to say about this node any more". Skipping the
|
|
@@ -75,7 +79,7 @@ export class IndexingCoordinator {
|
|
|
75
79
|
// with no scopes at all is that contributor withdrawing, so its facets go too,
|
|
76
80
|
// rather than leaving an excerpt describing content that no longer exists.
|
|
77
81
|
if (tags || metadata) await this.metadata.setContribution(nodeId, contributorId, { tags, metadata });
|
|
78
|
-
else if (!semanticTexts.length) await this.metadata.clearContribution
|
|
82
|
+
else if (!semanticTexts.length) await this.metadata.clearContribution(nodeId, contributorId);
|
|
79
83
|
}
|
|
80
84
|
|
|
81
85
|
/** Clear this contributor's search entries for a node (an empty re-index). */
|
|
@@ -257,7 +261,7 @@ export class IndexingCoordinator {
|
|
|
257
261
|
// can't count leaves this null, and the caller shows an indeterminate indicator —
|
|
258
262
|
// which is the right answer. Inventing a total would produce a progress bar that
|
|
259
263
|
// lies, and that is worse than a spinner.
|
|
260
|
-
const total = await this.metadata.countItems
|
|
264
|
+
const total = await this.metadata.countItems().catch(() => null) ?? null;
|
|
261
265
|
let indexed = 0;
|
|
262
266
|
let failed = 0;
|
|
263
267
|
let afterId = null;
|
|
@@ -76,6 +76,14 @@ export class IssueRegistry {
|
|
|
76
76
|
*/
|
|
77
77
|
async raise(spec) {
|
|
78
78
|
if (!spec?.kind || !spec?.title) throw TroveError.invalid('An issue needs a kind and a title');
|
|
79
|
+
// A `retry` that is not `{ op }` is silently un-retryable: `canRetry` reads
|
|
80
|
+
// `issue.retry.op`, so a bare string answers false, the route reports
|
|
81
|
+
// `retryable: false`, and no button ever renders. That happened — the sidecar-flush
|
|
82
|
+
// issue passed `retry: 'sidecar-flush'` and its own comment two lines above asserted a
|
|
83
|
+
// handler was registered for it. Caught at the raise, where the mistake is.
|
|
84
|
+
if (spec.retry != null && typeof spec.retry?.op !== 'string') {
|
|
85
|
+
throw TroveError.invalid('An issue\u2019s `retry` must be { op } — a bare string is not retryable');
|
|
86
|
+
}
|
|
79
87
|
const id = issueId(spec.kind, spec.subject);
|
|
80
88
|
const existing = await this.kv.get(NS, id);
|
|
81
89
|
const at = this.now();
|
|
@@ -32,8 +32,10 @@ export const TROVE_SCHEME = 'trove:';
|
|
|
32
32
|
// rather than storing kilobytes that will never match.
|
|
33
33
|
const MAX_SELECTOR = 255;
|
|
34
34
|
// How many distinct links one item may record. A document is a grouping, not a database
|
|
35
|
-
// dump; past this the value is in the search index, not the link graph.
|
|
36
|
-
|
|
35
|
+
// dump; past this the value is in the search index, not the link graph. Not exported: it
|
|
36
|
+
// is the default for `extractTroveLinks`'s `limit`, and nothing outside chose a different
|
|
37
|
+
// one — an exported cap nobody passes reads as a knob when it is a constant.
|
|
38
|
+
const MAX_LINKS_PER_ITEM = 500;
|
|
37
39
|
|
|
38
40
|
// Collection ids are slugs (see CollectionService); a name is anything but a slash,
|
|
39
41
|
// since the pathname shorthand has to stop somewhere.
|
|
@@ -81,6 +83,18 @@ export function isTroveUri(uri) {
|
|
|
81
83
|
return parseTroveUri(uri) != null;
|
|
82
84
|
}
|
|
83
85
|
|
|
86
|
+
/**
|
|
87
|
+
* The canonical string for one address. The ONE place the template lives.
|
|
88
|
+
*
|
|
89
|
+
* There were five hand-written copies of it, which is five chances to get the encoding or
|
|
90
|
+
* the separator subtly different — and `canonicalTroveUri`, whose whole docstring is
|
|
91
|
+
* "two links that address the same item by the same selector must produce the same
|
|
92
|
+
* string", was one of the copies rather than the source.
|
|
93
|
+
*/
|
|
94
|
+
export function formatTroveUri({ collection, by, value }) {
|
|
95
|
+
return `${TROVE_SCHEME}${collection}?${by}=${encodeURIComponent(value)}`;
|
|
96
|
+
}
|
|
97
|
+
|
|
84
98
|
/**
|
|
85
99
|
* Build a `trove:` URI. `by` defaults to 'name' — the legible form, which is what
|
|
86
100
|
* gets written into documents; pass 'id' for a link that must survive a rename.
|
|
@@ -89,10 +103,10 @@ export function troveUri(node, by = 'name') {
|
|
|
89
103
|
const collection = node?.collectionId || 'default';
|
|
90
104
|
if (by === 'id') {
|
|
91
105
|
if (!node?.id) throw TroveError.invalid('trove: link by id needs a node id');
|
|
92
|
-
return
|
|
106
|
+
return formatTroveUri({ collection, by: 'id', value: node.id });
|
|
93
107
|
}
|
|
94
108
|
if (!node?.name) throw TroveError.invalid('trove: link by name needs a node name');
|
|
95
|
-
return
|
|
109
|
+
return formatTroveUri({ collection, by: 'name', value: node.name });
|
|
96
110
|
}
|
|
97
111
|
|
|
98
112
|
/**
|
|
@@ -114,7 +128,7 @@ export function extractTroveLinks(text, { limit = MAX_LINKS_PER_ITEM } = {}) {
|
|
|
114
128
|
const raw = m[0].replace(/[.,;:!?]+$/, '');
|
|
115
129
|
const parsed = parseTroveUri(raw);
|
|
116
130
|
if (!parsed) continue;
|
|
117
|
-
const canonical =
|
|
131
|
+
const canonical = formatTroveUri(parsed);
|
|
118
132
|
if (seen.has(canonical)) continue;
|
|
119
133
|
seen.add(canonical);
|
|
120
134
|
out.push({ uri: canonical, raw, ...parsed });
|
|
@@ -129,7 +143,7 @@ export function extractTroveLinks(text, { limit = MAX_LINKS_PER_ITEM } = {}) {
|
|
|
129
143
|
*/
|
|
130
144
|
export function canonicalTroveUri(uri) {
|
|
131
145
|
const p = parseTroveUri(uri);
|
|
132
|
-
return p ?
|
|
146
|
+
return p ? formatTroveUri(p) : null;
|
|
133
147
|
}
|
|
134
148
|
|
|
135
149
|
/** Both canonical forms an item can be addressed by — what a backlink query looks for. */
|
|
@@ -220,7 +234,5 @@ export function parseShareUrl(url) {
|
|
|
220
234
|
export function troveUriFromShareUrl(url) {
|
|
221
235
|
const parsed = parseShareUrl(url);
|
|
222
236
|
if (!parsed) return null;
|
|
223
|
-
return parsed
|
|
224
|
-
? `${TROVE_SCHEME}${parsed.collection}?id=${encodeURIComponent(parsed.value)}`
|
|
225
|
-
: `${TROVE_SCHEME}${parsed.collection}?name=${encodeURIComponent(parsed.value)}`;
|
|
237
|
+
return formatTroveUri(parsed);
|
|
226
238
|
}
|
|
@@ -165,6 +165,28 @@ export class MetadataStore {
|
|
|
165
165
|
async trashedStorageKeys(collectionId) {
|
|
166
166
|
throw TroveError.unsupported('trashedStorageKeys not implemented');
|
|
167
167
|
}
|
|
168
|
+
/**
|
|
169
|
+
* Every SEALED item in a collection, live AND trashed, paged by (name, id).
|
|
170
|
+
*
|
|
171
|
+
* Key rotation is the caller, and it cannot use `listItems` for the same reason the
|
|
172
|
+
* scanner cannot: that is live-only by design, while the Vfs deliberately keeps the
|
|
173
|
+
* BYTES of a soft-deleted item so a restore can bring it back. A rotation that walks
|
|
174
|
+
* only live items completes a pass having never looked at the trash, retires the old
|
|
175
|
+
* key on the strength of it, and every trashed object still sealed with that key
|
|
176
|
+
* becomes permanently unopenable — the restore fails, forever. Retiring by observation
|
|
177
|
+
* is only sound if the observation covers everything the key still opens.
|
|
178
|
+
*
|
|
179
|
+
* Sealed-only because that is the entire set a rotation can act on, and applying the
|
|
180
|
+
* predicate in the store means a mostly-plaintext collection is one page rather than
|
|
181
|
+
* hundreds of pages of items to skip.
|
|
182
|
+
*
|
|
183
|
+
* @param {string} collectionId
|
|
184
|
+
* @param {{limit?: number, cursor?: string}} [opts]
|
|
185
|
+
* @returns {Promise<{items: object[], nextCursor: string|null}>}
|
|
186
|
+
*/
|
|
187
|
+
async listSealed(collectionId, opts) {
|
|
188
|
+
throw TroveError.unsupported('listSealed not implemented');
|
|
189
|
+
}
|
|
168
190
|
/** Rename an item. Rejects ALREADY_EXISTS if the name is taken in its collection. */
|
|
169
191
|
async rename(id, newName) {
|
|
170
192
|
throw TroveError.unsupported('rename not implemented');
|
|
@@ -153,6 +153,21 @@ export class MemoryStore extends MetadataStore {
|
|
|
153
153
|
return keys;
|
|
154
154
|
}
|
|
155
155
|
|
|
156
|
+
async listSealed(collectionId = 'default', opts = {}) {
|
|
157
|
+
// `this.nodes` rather than `#live()`, deliberately — see the interface docblock. The
|
|
158
|
+
// trash is the half of the key's working set that `listItems` cannot see.
|
|
159
|
+
const items = [...this.nodes.values()].filter((n) => n.collectionId === collectionId && n.encryption);
|
|
160
|
+
items.sort((a, b) => compareForSort(a, b, 'name', 1));
|
|
161
|
+
const limit = opts.limit ?? 500;
|
|
162
|
+
const at = decodeCursor('name', opts.cursor);
|
|
163
|
+
const rest = at ? items.filter((n) => afterCursor(n, 'name', at, false)) : items;
|
|
164
|
+
const page = rest.slice(0, limit);
|
|
165
|
+
return {
|
|
166
|
+
items: page.map(clone),
|
|
167
|
+
nextCursor: rest.length > limit ? encodeCursor('name', page[page.length - 1]) : null,
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
|
|
156
171
|
async trashedBefore(cutoff, limit = 500) {
|
|
157
172
|
return [...this.nodes.values()]
|
|
158
173
|
.filter((n) => n.deletedAt && n.deletedAt < cutoff)
|
|
@@ -208,7 +208,6 @@ export class SqliteStore extends MetadataStore {
|
|
|
208
208
|
);
|
|
209
209
|
return rows.map(row);
|
|
210
210
|
}
|
|
211
|
-
/** Items trashed before `cutoff` — what the purge sweep collects. */
|
|
212
211
|
async trashedStorageKeys(collectionId) {
|
|
213
212
|
const rows = await this.db.all(
|
|
214
213
|
'SELECT storageKey FROM nodes WHERE collectionId = ? AND deletedAt IS NOT NULL AND storageKey IS NOT NULL',
|
|
@@ -217,6 +216,29 @@ export class SqliteStore extends MetadataStore {
|
|
|
217
216
|
return new Set(rows.map((r) => r.storageKey));
|
|
218
217
|
}
|
|
219
218
|
|
|
219
|
+
async listSealed(collectionId = 'default', opts = {}) {
|
|
220
|
+
// No `deletedAt IS NULL`, deliberately — see the interface docblock. The trash is the
|
|
221
|
+
// half of the key's working set that `listItems` cannot see.
|
|
222
|
+
const limit = opts.limit ?? 500;
|
|
223
|
+
const at = decodeCursor('name', opts.cursor);
|
|
224
|
+
// The same keyset shape and the same NOCASE expression `listItems` uses, so a cursor
|
|
225
|
+
// means the same thing here and the page boundary lines up with the ordering.
|
|
226
|
+
const where = at ? 'AND (name COLLATE NOCASE > ? OR (name COLLATE NOCASE = ? AND id > ?))' : '';
|
|
227
|
+
const args = at ? [at.value, at.value, at.id] : [];
|
|
228
|
+
const rows = await this.db.all(
|
|
229
|
+
`SELECT * FROM nodes WHERE collectionId = ? AND encryption IS NOT NULL ${where}
|
|
230
|
+
ORDER BY name COLLATE NOCASE ASC, id ASC
|
|
231
|
+
LIMIT ?`,
|
|
232
|
+
collectionId, ...args, limit + 1,
|
|
233
|
+
);
|
|
234
|
+
const hasMore = rows.length > limit;
|
|
235
|
+
const page = rows.slice(0, limit).map(row);
|
|
236
|
+
return {
|
|
237
|
+
items: page,
|
|
238
|
+
nextCursor: hasMore ? encodeCursor('name', page[page.length - 1]) : null,
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
|
|
220
242
|
async trashedBefore(cutoff, limit = 500) {
|
|
221
243
|
const rows = await this.db.all(
|
|
222
244
|
'SELECT * FROM nodes WHERE deletedAt IS NOT NULL AND deletedAt < ? ORDER BY deletedAt ASC LIMIT ?',
|