@3sln/trove 0.0.8 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/core/src/encryption/envelope.js +46 -7
- package/packages/core/src/encryption/rotation.js +187 -68
- package/packages/core/src/uploads.js +130 -29
- package/packages/core/src/vfs.js +91 -5
- package/packages/server/src/adapters/staticAssets.js +14 -5
- package/packages/server/src/engine/index.js +1 -1
- package/packages/server/src/engine/providers/core.js +12 -0
- package/packages/server/src/index.js +45 -2
- package/packages/server/src/routes.js +59 -26
- package/packages/web/dist/assets/main-jg5vmp8f.js +356 -0
- package/packages/web/dist/assets/main-jg5vmp8f.js.map +118 -0
- package/packages/web/dist/assets/styles-e5gk19rn.css +1 -0
- package/packages/web/dist/index.html +2 -2
- package/packages/web/dist/sw.js +1 -1
- package/packages/web/src/bl/actions.js +1454 -137
- package/packages/web/src/bl/commands.js +103 -284
- package/packages/web/src/bl/context.js +69 -0
- package/packages/web/src/bl/index.js +136 -40
- package/packages/web/src/bl/intern.js +129 -0
- package/packages/web/src/bl/launcher.js +251 -0
- package/packages/web/src/bl/links.js +1 -1
- package/packages/web/src/bl/match.js +71 -0
- package/packages/web/src/bl/mentions.js +39 -0
- package/packages/web/src/bl/offline.js +0 -4
- package/packages/web/src/bl/openers.js +65 -33
- package/packages/web/src/bl/pluginInstall.js +42 -21
- package/packages/web/src/bl/queries.js +669 -0
- package/packages/web/src/bl/services.js +77 -141
- package/packages/web/src/bl/social.js +3 -1
- package/packages/web/src/bl/state.js +144 -0
- package/packages/web/src/bl/status.js +96 -0
- package/packages/web/src/bl/trust.js +109 -0
- package/packages/web/src/bl/viewState.js +60 -0
- package/packages/web/src/bl/views.js +64 -0
- package/packages/web/src/bl/watchQuery.js +73 -0
- package/packages/web/src/platform/api.js +31 -102
- package/packages/web/src/platform/commands.js +49 -9
- package/packages/web/src/platform/context.js +146 -51
- package/packages/web/src/platform/index.js +27 -20
- package/packages/web/src/platform/navigation.js +34 -4
- package/packages/web/src/platform/pickers.js +46 -0
- package/packages/web/src/platform/pluginHost.js +43 -10
- package/packages/web/src/platform/pluginRpc.js +6 -2
- package/packages/web/src/platform/spatialNav.js +0 -2
- package/packages/web/src/platform/viewport.js +1 -6
- package/packages/web/src/platform/voiceSearch.js +1 -3
- package/packages/web/src/platform/whenclause.js +22 -3
- package/packages/web/src/styles.css +74 -4
- package/packages/web/src/ui/activate.js +22 -0
- package/packages/web/src/ui/components/activityBar.js +7 -3
- package/packages/web/src/ui/components/activityPanel.js +10 -11
- package/packages/web/src/ui/components/adminView.js +191 -0
- package/packages/web/src/ui/components/collectionGate.js +4 -3
- package/packages/web/src/ui/components/commandPalette.js +24 -55
- package/packages/web/src/ui/components/editorArea.js +16 -13
- package/packages/web/src/ui/components/launcher.js +39 -225
- package/packages/web/src/ui/components/openers/index.js +7 -4
- package/packages/web/src/ui/components/openers/markdown.js +5 -3
- package/packages/web/src/ui/components/overlays.js +158 -45
- package/packages/web/src/ui/components/phoneChrome.js +35 -33
- package/packages/web/src/ui/components/pluginReview.js +62 -23
- package/packages/web/src/ui/components/pluginsView.js +30 -28
- package/packages/web/src/ui/components/settingsView.js +151 -58
- package/packages/web/src/ui/components/social.js +42 -62
- package/packages/web/src/ui/components/statusBar.js +30 -74
- package/packages/web/src/ui/components/views/grid.js +3 -2
- package/packages/web/src/ui/components/views/index.js +7 -62
- package/packages/web/src/ui/components/views/list.js +3 -2
- package/packages/web/src/ui/components/views/parts.js +15 -4
- package/packages/web/src/ui/compositions/workbench.js +166 -58
- package/packages/web/src/ui/region.js +53 -0
- package/packages/web/src/workbench.js +17 -15
- package/packages/web/dist/assets/main-y778bpte.js +0 -356
- package/packages/web/dist/assets/main-y778bpte.js.map +0 -106
- package/packages/web/dist/assets/styles-nfy8t3n1.css +0 -1
- package/packages/web/src/platform/overlay.js +0 -81
- package/packages/web/src/platform/workbench.js +0 -156
|
@@ -14,8 +14,11 @@
|
|
|
14
14
|
|
|
15
15
|
import { TroveError, ErrorCode } from './errors.js';
|
|
16
16
|
import { newId, isValidItemName } from './util.js';
|
|
17
|
-
import {
|
|
18
|
-
|
|
17
|
+
import {
|
|
18
|
+
cipherSize, DEFAULT_CHUNK_SIZE, isEnvelope, decodeHeader, encodeHeader, encryptStream,
|
|
19
|
+
HEADER_BYTES as ENVELOPE_HEAD,
|
|
20
|
+
} from './encryption/envelope.js';
|
|
21
|
+
import { fromHex, toHex } from './encryption/keys.js';
|
|
19
22
|
import { shouldEncrypt } from './encryption/policy.js';
|
|
20
23
|
|
|
21
24
|
export const DEFAULT_PART_SIZE = 8 * 1024 * 1024; // 8 MiB
|
|
@@ -103,6 +106,40 @@ export class KvSessionStore {
|
|
|
103
106
|
}
|
|
104
107
|
}
|
|
105
108
|
|
|
109
|
+
/**
|
|
110
|
+
* Whatever a caller handed us, as a stream.
|
|
111
|
+
*
|
|
112
|
+
* A part arrives as a stream over HTTP and as bytes from a test or an in-process caller;
|
|
113
|
+
* `encryptStream` takes only the former, and the sealing path has to accept both.
|
|
114
|
+
*/
|
|
115
|
+
function asStream(body) {
|
|
116
|
+
if (body && typeof body.getReader === 'function') return body;
|
|
117
|
+
if (body && typeof body.stream === 'function') return body.stream(); // Blob
|
|
118
|
+
const bytes = typeof body === 'string'
|
|
119
|
+
? new TextEncoder().encode(body)
|
|
120
|
+
: (body instanceof Uint8Array ? body : new Uint8Array(body || 0));
|
|
121
|
+
return new ReadableStream({
|
|
122
|
+
start(c) {
|
|
123
|
+
if (bytes.length) c.enqueue(bytes);
|
|
124
|
+
c.close();
|
|
125
|
+
},
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** A stream that emits `head`, then everything `rest` produces. */
|
|
130
|
+
function prependBytes(head, rest) {
|
|
131
|
+
const reader = rest.getReader();
|
|
132
|
+
return new ReadableStream({
|
|
133
|
+
start(c) { c.enqueue(head); },
|
|
134
|
+
async pull(c) {
|
|
135
|
+
const { value, done } = await reader.read();
|
|
136
|
+
if (done) c.close();
|
|
137
|
+
else c.enqueue(value);
|
|
138
|
+
},
|
|
139
|
+
cancel(reason) { return reader.cancel(reason); },
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
|
|
106
143
|
export class UploadManager {
|
|
107
144
|
/**
|
|
108
145
|
* @param {object} deps
|
|
@@ -204,6 +241,10 @@ export class UploadManager {
|
|
|
204
241
|
storedSize,
|
|
205
242
|
encrypted: encrypting,
|
|
206
243
|
chunkSize: encrypting ? chunkSize : null,
|
|
244
|
+
// Fixed for the object, at negotiation, because parts are sealed INDEPENDENTLY and
|
|
245
|
+
// may arrive in any order — see `#sealPart`. Generating it per part would reuse
|
|
246
|
+
// nonces across the object, which AES-GCM does not survive.
|
|
247
|
+
noncePrefix: encrypting ? toHex(crypto.getRandomValues(new Uint8Array(8))) : null,
|
|
207
248
|
keyFingerprint: encrypting ? policy.encryption.fingerprint : null,
|
|
208
249
|
contentType,
|
|
209
250
|
createdAt: Date.now(),
|
|
@@ -215,18 +256,35 @@ export class UploadManager {
|
|
|
215
256
|
|
|
216
257
|
const limits = this.#limits();
|
|
217
258
|
|
|
259
|
+
// Presigned strategies hand the client a URL to the bucket, which is only safe when
|
|
260
|
+
// the client has something the bucket may hold. It seals nothing now, so for an
|
|
261
|
+
// encrypted collection a presigned PUT would put PLAINTEXT in the store — exactly what
|
|
262
|
+
// the encryption exists to prevent. Those go through us, and we seal on the way past.
|
|
263
|
+
const canPresign = caps.presignUpload && !encrypting;
|
|
264
|
+
|
|
218
265
|
// Small file + presign → single PUT straight to storage (never through us).
|
|
219
|
-
if (storedSize <= SINGLE_PUT_LIMIT &&
|
|
266
|
+
if (storedSize <= SINGLE_PUT_LIMIT && canPresign) {
|
|
220
267
|
session.strategy = 'single';
|
|
221
268
|
await this.sessions.put(session);
|
|
222
269
|
const url = await storage.presignPut(storageKey, { contentType });
|
|
223
|
-
return { ...planSummary(session), strategy: 'single', multipart: false, presigned: true, url, limits, encryption: this.#planEncryption(session
|
|
270
|
+
return { ...planSummary(session), strategy: 'single', multipart: false, presigned: true, url, limits, encryption: this.#planEncryption(session) };
|
|
224
271
|
}
|
|
225
272
|
|
|
226
273
|
// Multipart (presigned parts straight to storage, or streamed through us).
|
|
227
274
|
if (caps.multipart) {
|
|
228
|
-
session.strategy =
|
|
229
|
-
|
|
275
|
+
session.strategy = canPresign ? 'presign' : 'direct';
|
|
276
|
+
// Measured in PLAINTEXT for an encrypted upload: the client sends us the file as it
|
|
277
|
+
// is, and the envelope it becomes is bigger by a tag per chunk. Sizing parts against
|
|
278
|
+
// the stored size would ask the client for bytes it does not have.
|
|
279
|
+
const wireSize = encrypting ? req.size : storedSize;
|
|
280
|
+
const partCount = Math.max(1, Math.ceil(wireSize / this.partSize));
|
|
281
|
+
// A part must hold a whole number of chunks, or the chunk a part boundary lands
|
|
282
|
+
// inside would be sealed twice under two different indices.
|
|
283
|
+
if (encrypting && this.partSize % chunkSize !== 0) {
|
|
284
|
+
throw TroveError.internal(
|
|
285
|
+
`Part size ${this.partSize} is not a multiple of the ${chunkSize}-byte chunk size`,
|
|
286
|
+
);
|
|
287
|
+
}
|
|
230
288
|
// `#limits()` advertises maxParts in the very same response, and nothing enforced
|
|
231
289
|
// it: a 150 GiB file planned 19,200 parts against a ceiling of 10,000, which S3
|
|
232
290
|
// rejects at part 10,001 — after the client has transferred 80 GiB. On a presign
|
|
@@ -249,38 +307,31 @@ export class UploadManager {
|
|
|
249
307
|
parts.push({ partNumber: n, url: await storage.presignPart(storageKey, session.uploadId, n) });
|
|
250
308
|
}
|
|
251
309
|
}
|
|
252
|
-
return { ...planSummary(session), strategy: session.strategy, multipart: true, presigned: session.strategy === 'presign', partCount, parts, limits, encryption: this.#planEncryption(session
|
|
310
|
+
return { ...planSummary(session), strategy: session.strategy, multipart: true, presigned: session.strategy === 'presign', partCount, parts, limits, encryption: this.#planEncryption(session) };
|
|
253
311
|
}
|
|
254
312
|
|
|
255
313
|
// Fallback: whole-object PUT streamed through us (tiny/simple backends).
|
|
256
314
|
session.strategy = 'direct-single';
|
|
257
315
|
await this.sessions.put(session);
|
|
258
|
-
return { ...planSummary(session), strategy: 'direct-single', multipart: false, presigned: false, limits, encryption: this.#planEncryption(session
|
|
316
|
+
return { ...planSummary(session), strategy: 'direct-single', multipart: false, presigned: false, limits, encryption: this.#planEncryption(session) };
|
|
259
317
|
}
|
|
260
318
|
|
|
261
319
|
/**
|
|
262
|
-
* What
|
|
320
|
+
* What a client is told about encryption: that it is happening, and nothing else.
|
|
263
321
|
*
|
|
264
|
-
*
|
|
265
|
-
*
|
|
266
|
-
*
|
|
267
|
-
*
|
|
268
|
-
*
|
|
322
|
+
* This used to hand over the collection's data key so the browser could seal before the
|
|
323
|
+
* bytes left it — which is what made a presigned direct-to-bucket PUT possible while the
|
|
324
|
+
* bucket only ever held ciphertext. That trade is off. Encrypted collections now pass
|
|
325
|
+
* through the drive in BOTH directions, so the key never leaves the server and a client
|
|
326
|
+
* has nothing it could leak.
|
|
269
327
|
*
|
|
270
|
-
*
|
|
328
|
+
* The cost is deliberate and worth naming: bytes for an encrypted collection are paid
|
|
329
|
+
* for twice, once to us and once to the store, and a presigned upload is unavailable for
|
|
330
|
+
* them. Unencrypted collections are untouched and still go straight to the bucket.
|
|
271
331
|
*/
|
|
272
|
-
#planEncryption(session
|
|
332
|
+
#planEncryption(session) {
|
|
273
333
|
if (!session.encrypted) return null;
|
|
274
|
-
return {
|
|
275
|
-
algorithm: 'AES-256-GCM',
|
|
276
|
-
chunkSize: session.chunkSize,
|
|
277
|
-
fingerprint: policy.encryption.fingerprint,
|
|
278
|
-
// Hex rather than raw bytes: this rides in a JSON plan.
|
|
279
|
-
key: policy.dataKeyHex,
|
|
280
|
-
// What the client should end up PUTting, so it can check its own work before
|
|
281
|
-
// spending the bytes.
|
|
282
|
-
storedSize: session.storedSize,
|
|
283
|
-
};
|
|
334
|
+
return { algorithm: 'AES-256-GCM', sealedBy: 'server' };
|
|
284
335
|
}
|
|
285
336
|
|
|
286
337
|
|
|
@@ -351,7 +402,8 @@ export class UploadManager {
|
|
|
351
402
|
// deliberately doesn't apply — so this is the only place a client can be stopped
|
|
352
403
|
// from writing an unbounded object through us.
|
|
353
404
|
const capped = capStream(body, this.maxBytes);
|
|
354
|
-
const
|
|
405
|
+
const toStore = s.encrypted ? await this.#sealPart(s, 1, capped, true) : capped;
|
|
406
|
+
const info = await storage.put(s.storageKey, toStore, { contentType: s.contentType, ...opts });
|
|
355
407
|
s.parts[1] = { etag: info.etag || 'single' };
|
|
356
408
|
await this.sessions.put(s);
|
|
357
409
|
return { partNumber: 1, etag: s.parts[1].etag };
|
|
@@ -366,13 +418,62 @@ export class UploadManager {
|
|
|
366
418
|
if (!Number.isInteger(partNumber) || partNumber < 1 || (s.partCount && partNumber > s.partCount)) {
|
|
367
419
|
throw TroveError.invalid(`Part ${partNumber} is outside this upload's ${s.partCount} part(s)`);
|
|
368
420
|
}
|
|
369
|
-
// A part is exactly `partSize` bytes, except the last, which is smaller.
|
|
370
|
-
|
|
421
|
+
// A part is exactly `partSize` bytes, except the last, which is smaller. Capped on the
|
|
422
|
+
// way IN — that is the plaintext the client sends; what we write is bigger by a tag per
|
|
423
|
+
// chunk.
|
|
424
|
+
const capped = capStream(body, s.partSize);
|
|
425
|
+
const toStore = s.encrypted
|
|
426
|
+
? await this.#sealPart(s, partNumber, capped, partNumber === (s.partCount ?? 1))
|
|
427
|
+
: capped;
|
|
428
|
+
const res = await storage.putPart(s.storageKey, s.uploadId, partNumber, toStore, opts);
|
|
371
429
|
s.parts[partNumber] = { etag: res.etag };
|
|
372
430
|
await this.sessions.put(s);
|
|
373
431
|
return res;
|
|
374
432
|
}
|
|
375
433
|
|
|
434
|
+
/**
|
|
435
|
+
* Seal one part on its way to the store.
|
|
436
|
+
*
|
|
437
|
+
* Parts are sealed INDEPENDENTLY, which is what lets a client keep sending them
|
|
438
|
+
* concurrently now that it no longer seals them itself. That is safe only because a
|
|
439
|
+
* part's position in the envelope is a function of its NUMBER: part n carries plaintext
|
|
440
|
+
* bytes [(n-1)·partSize, n·partSize), so it begins at chunk (n-1)·partSize/chunkSize, and
|
|
441
|
+
* the nonce prefix was fixed for the whole object at negotiation. Nothing depends on the
|
|
442
|
+
* order parts arrive in, and no two chunks can ever be sealed under the same nonce.
|
|
443
|
+
*
|
|
444
|
+
* The header goes on the front of part 1 and nowhere else — it is the first 44 bytes of
|
|
445
|
+
* the object. Every part but the last is `partial`, which is what makes `encryptStream`
|
|
446
|
+
* refuse to finish an envelope early and insist on stopping at a chunk boundary.
|
|
447
|
+
*/
|
|
448
|
+
async #sealPart(s, partNumber, body, isLast) {
|
|
449
|
+
const chunksPerPart = s.partSize / s.chunkSize;
|
|
450
|
+
const startChunk = (partNumber - 1) * chunksPerPart;
|
|
451
|
+
const key = await this.#dataKeyFor(s);
|
|
452
|
+
const sealed = await encryptStream(key, asStream(body), {
|
|
453
|
+
fingerprint: fromHex(s.keyFingerprint),
|
|
454
|
+
plaintextSize: s.size,
|
|
455
|
+
chunkSize: s.chunkSize,
|
|
456
|
+
resume: { noncePrefix: fromHex(s.noncePrefix), index: startChunk },
|
|
457
|
+
partial: !isLast,
|
|
458
|
+
});
|
|
459
|
+
if (partNumber !== 1) return sealed;
|
|
460
|
+
const header = encodeHeader({
|
|
461
|
+
chunkSize: s.chunkSize,
|
|
462
|
+
plaintextSize: s.size,
|
|
463
|
+
noncePrefix: fromHex(s.noncePrefix),
|
|
464
|
+
fingerprint: fromHex(s.keyFingerprint),
|
|
465
|
+
});
|
|
466
|
+
return prependBytes(header, sealed);
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/** The collection's key, for sealing. Never leaves this process. */
|
|
470
|
+
async #dataKeyFor(s) {
|
|
471
|
+
const policy = await this.encryptionFor(s.collectionId);
|
|
472
|
+
const key = policy?.dataKeyHex ? fromHex(policy.dataKeyHex) : null;
|
|
473
|
+
if (!key) throw TroveError.internal('This collection is encrypted but its key is unavailable');
|
|
474
|
+
return key;
|
|
475
|
+
}
|
|
476
|
+
|
|
376
477
|
/** Which parts are still outstanding (resume support). */
|
|
377
478
|
async status(uploadId) {
|
|
378
479
|
const s = await this.#session(uploadId);
|
package/packages/core/src/vfs.js
CHANGED
|
@@ -12,8 +12,12 @@
|
|
|
12
12
|
|
|
13
13
|
import { TroveError, isOutOfSpace } from './errors.js';
|
|
14
14
|
import { UploadManager } from './uploads.js';
|
|
15
|
-
import { toHex } from './encryption/keys.js';
|
|
16
|
-
import {
|
|
15
|
+
import { fromHex, toHex } from './encryption/keys.js';
|
|
16
|
+
import {
|
|
17
|
+
encrypt, decryptStream, decodeHeader, cipherRangeFor, cipherSize, HEADER_BYTES,
|
|
18
|
+
DEFAULT_CHUNK_SIZE,
|
|
19
|
+
} from './encryption/envelope.js';
|
|
20
|
+
import { shouldEncrypt } from './encryption/policy.js';
|
|
17
21
|
import { IndexerRegistry } from './indexers/registry.js';
|
|
18
22
|
import { ParsingSearchTransformer, matchTagFilters } from './search/transformer.js';
|
|
19
23
|
import { extname } from './util.js';
|
|
@@ -56,6 +60,24 @@ async function bytesOf(stream) {
|
|
|
56
60
|
return out;
|
|
57
61
|
}
|
|
58
62
|
|
|
63
|
+
/**
|
|
64
|
+
* Whatever `writeFile` was handed, as bytes.
|
|
65
|
+
*
|
|
66
|
+
* Sealing needs the plaintext size before it can write the envelope header, and this path
|
|
67
|
+
* is the convenience write for things already resident, so buffering costs nothing that was
|
|
68
|
+
* not already paid.
|
|
69
|
+
*/
|
|
70
|
+
async function bytesOfBody(body) {
|
|
71
|
+
if (body == null) return new Uint8Array(0);
|
|
72
|
+
if (typeof body === 'string') return new TextEncoder().encode(body);
|
|
73
|
+
if (body instanceof Uint8Array) return body;
|
|
74
|
+
if (ArrayBuffer.isView(body)) return new Uint8Array(body.buffer, body.byteOffset, body.byteLength);
|
|
75
|
+
if (body instanceof ArrayBuffer) return new Uint8Array(body);
|
|
76
|
+
if (typeof body.arrayBuffer === 'function') return new Uint8Array(await body.arrayBuffer());
|
|
77
|
+
if (typeof body.getReader === 'function') return bytesOf(body);
|
|
78
|
+
return new Uint8Array(body);
|
|
79
|
+
}
|
|
80
|
+
|
|
59
81
|
/**
|
|
60
82
|
* Emit only the bytes between `start` and `end` of a stream.
|
|
61
83
|
*
|
|
@@ -132,6 +154,32 @@ export class Vfs {
|
|
|
132
154
|
if (this.collections) await this.collections.init();
|
|
133
155
|
}
|
|
134
156
|
|
|
157
|
+
/**
|
|
158
|
+
* What this collection wants sealed, and the key for it — or null.
|
|
159
|
+
*
|
|
160
|
+
* The same question `UploadManager` asks through its injected `encryptionFor`, including
|
|
161
|
+
* `shouldEncrypt`, so a collection with per-item rules answers identically whichever way
|
|
162
|
+
* the bytes arrive.
|
|
163
|
+
*/
|
|
164
|
+
async #sealingFor(collectionId, name, contentType) {
|
|
165
|
+
if (!this.collections?.encryptionFor) return null;
|
|
166
|
+
// A collection record that is not there cannot ask for anything. `writeFile` defaults
|
|
167
|
+
// to 'default', which on a zero-config drive is a storage backend rather than a
|
|
168
|
+
// collection anyone created, and asking about it throws rather than answering "no".
|
|
169
|
+
let encryption;
|
|
170
|
+
try {
|
|
171
|
+
encryption = await this.collections.encryptionFor(collectionId);
|
|
172
|
+
} catch (err) {
|
|
173
|
+
if (err?.code === 'not_found') return null;
|
|
174
|
+
throw err;
|
|
175
|
+
}
|
|
176
|
+
if (!encryption?.enabled) return null;
|
|
177
|
+
if (!shouldEncrypt(encryption, { name, contentType })) return null;
|
|
178
|
+
const key = await this.collections.dataKeyFor(collectionId);
|
|
179
|
+
if (!key) throw TroveError.internal('This collection is encrypted but its key is unavailable');
|
|
180
|
+
return { key, fingerprint: fromHex(encryption.fingerprint), chunkSize: encryption.chunkSize || DEFAULT_CHUNK_SIZE };
|
|
181
|
+
}
|
|
182
|
+
|
|
135
183
|
/** Resolve the storage backend for a collection. */
|
|
136
184
|
async storageFor(collectionId = 'default') {
|
|
137
185
|
if (this.collections) return this.collections.storageFor(collectionId);
|
|
@@ -256,9 +304,31 @@ export class Vfs {
|
|
|
256
304
|
const storageKey = `obj_${cryptoId()}`;
|
|
257
305
|
const ct = contentType || this.guessContentType(name);
|
|
258
306
|
const storage = await this.storageFor(collectionId);
|
|
307
|
+
// Sealed here too, if the collection says so.
|
|
308
|
+
//
|
|
309
|
+
// This path wrote straight to the bucket and recorded the item with no `encryption`,
|
|
310
|
+
// never asking — so a server-side write put a READABLE file in a collection someone had
|
|
311
|
+
// set up to be encrypted, and stamped it as unencrypted so the read path served it back
|
|
312
|
+
// happily and nothing ever said otherwise. Once the drive started sealing uploads, this
|
|
313
|
+
// was the only remaining way to get plaintext into an encrypted bucket.
|
|
314
|
+
const sealing = await this.#sealingFor(collectionId, name, ct);
|
|
315
|
+
let toStore = body;
|
|
316
|
+
let encryption = null;
|
|
317
|
+
let plaintextSize = null;
|
|
318
|
+
if (sealing) {
|
|
319
|
+
// Buffered, unlike the upload path, which streams. This is the convenience write for
|
|
320
|
+
// things already resident — a sidecar, a test fixture, an in-process import — and the
|
|
321
|
+
// envelope needs the plaintext size before it can write its header.
|
|
322
|
+
const plain = await bytesOfBody(body);
|
|
323
|
+
plaintextSize = plain.length;
|
|
324
|
+
toStore = await encrypt(sealing.key, plain, {
|
|
325
|
+
fingerprint: sealing.fingerprint, chunkSize: sealing.chunkSize,
|
|
326
|
+
});
|
|
327
|
+
encryption = { fingerprint: toHex(sealing.fingerprint), chunkSize: sealing.chunkSize };
|
|
328
|
+
}
|
|
259
329
|
let info;
|
|
260
330
|
try {
|
|
261
|
-
info = await storage.put(storageKey,
|
|
331
|
+
info = await storage.put(storageKey, toStore, { contentType: ct, signal });
|
|
262
332
|
} catch (err) {
|
|
263
333
|
// A write that failed for lack of room is a standing condition, not one bad
|
|
264
334
|
// request: the next upload will fail the same way. Record it so it is visible
|
|
@@ -266,7 +336,11 @@ export class Vfs {
|
|
|
266
336
|
if (isOutOfSpace(err)) await this.storageUsage(collectionId).catch(() => {});
|
|
267
337
|
throw err;
|
|
268
338
|
}
|
|
269
|
-
|
|
339
|
+
// The size the user sees is the FILE's, not the envelope's.
|
|
340
|
+
const node = await this.#upsertItem({
|
|
341
|
+
collectionId, name, storageKey, size: plaintextSize ?? info.size, contentType: ct,
|
|
342
|
+
etag: info.etag, encryption,
|
|
343
|
+
});
|
|
270
344
|
// Small server-side writes index synchronously (search is ready on return);
|
|
271
345
|
// large client uploads (completeUpload) index in the background instead.
|
|
272
346
|
await this.indexing.indexNode(node).catch((e) => console.error('index error', e));
|
|
@@ -502,7 +576,19 @@ export class Vfs {
|
|
|
502
576
|
const node = await this.resolve(id);
|
|
503
577
|
if (!node.storageKey) throw TroveError.notFound('File content');
|
|
504
578
|
const storage = await this.storageFor(node.collectionId);
|
|
505
|
-
|
|
579
|
+
// An ENCRYPTED object is never presigned to the store — the same rule `getDownload`
|
|
580
|
+
// already follows, and the one place it was missing. A bucket URL serves CIPHERTEXT,
|
|
581
|
+
// and everything a minted URL exists for (an <img src>, a <video src>, cache.add(), a
|
|
582
|
+
// URL handed to an external service) fetches bytes with nowhere to run decryption. So
|
|
583
|
+
// every thumbnail and preview in an encrypted collection on a presigning store pointed
|
|
584
|
+
// at ciphertext and rendered nothing.
|
|
585
|
+
//
|
|
586
|
+
// Worse than proxying would have been: a bucket URL is a DIFFERENT ORIGIN, so the
|
|
587
|
+
// service worker never saw it either — the one place that could have decrypted was
|
|
588
|
+
// routed around. Sending these through our own address is what lets the worker read
|
|
589
|
+
// straight from the store and decrypt (platform/directRead.js), and lets the server
|
|
590
|
+
// decrypt for anything without a worker.
|
|
591
|
+
if (storage.capabilities.presignDownload && !node.encryption) {
|
|
506
592
|
const seconds = clampAge(op, expiresIn);
|
|
507
593
|
const url = await storage.presignGet(node.storageKey, {
|
|
508
594
|
expiresIn: seconds, responseContentType: node.contentType,
|
|
@@ -33,6 +33,7 @@ import path from 'node:path';
|
|
|
33
33
|
// The policy lives in its own module because index.js needs it too and has to stay
|
|
34
34
|
// loadable on Workers, where `node:path` is not there to be imported.
|
|
35
35
|
import { IMMUTABLE_PREFIX, cacheControlFor } from '../cachePolicy.js';
|
|
36
|
+
import { SHARE_PATH } from '@3sln/trove/core/links.js';
|
|
36
37
|
|
|
37
38
|
export { IMMUTABLE_PREFIX, cacheControlFor };
|
|
38
39
|
|
|
@@ -46,10 +47,16 @@ export const MIME = {
|
|
|
46
47
|
/**
|
|
47
48
|
* Should a miss here fall back to the SPA entry point?
|
|
48
49
|
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
50
|
+
* Most of the app lives at `/` — `navigation.js` pushes state with no URL — so for those
|
|
51
|
+
* views the fallback is only robustness, a refresh landing somewhere unexpected.
|
|
52
|
+
*
|
|
53
|
+
* Share links are the exception, and they break the rule this used to rest on. A share
|
|
54
|
+
* link is `/c/<collection>/i/<item>`, and an item name ends in a filename, so
|
|
55
|
+
* "anything with an extension is asking for a file" refused exactly the deep link the
|
|
56
|
+
* scheme exists to provide: the server 404'd before the app could load, and the
|
|
57
|
+
* client-side routing that would have resolved it never ran. A share path is a route
|
|
58
|
+
* whatever it ends with, so it is recognised before the extension rule rather than
|
|
59
|
+
* fighting it.
|
|
53
60
|
*
|
|
54
61
|
* @param {string} pathname
|
|
55
62
|
*/
|
|
@@ -58,7 +65,9 @@ export function shouldFallBack(pathname) {
|
|
|
58
65
|
// Content-addressed: a miss is a stale reference to a build that no longer exists.
|
|
59
66
|
// Answering it with HTML is what poisons a service worker cache.
|
|
60
67
|
if (pathname.startsWith(IMMUTABLE_PREFIX)) return false;
|
|
61
|
-
//
|
|
68
|
+
// A route the app owns, named by the same module that produces the links.
|
|
69
|
+
if (pathname === SHARE_PATH || pathname.startsWith(`${SHARE_PATH}/`)) return true;
|
|
70
|
+
// Anything else with an extension is asking for a file, not a view.
|
|
62
71
|
if (path.extname(pathname)) return false;
|
|
63
72
|
return true;
|
|
64
73
|
}
|
|
@@ -49,7 +49,7 @@ export function createDriveEngine(config = {}, lifecycleState = { closing: false
|
|
|
49
49
|
export const BACKBONE = [
|
|
50
50
|
'storage', 'sqlite', 'metadata', 'kv', 'tasks', 'issues', 'notifications',
|
|
51
51
|
'sidecar', 'collections', 'identity', 'auth', 'search', 'vfs', 'plugins',
|
|
52
|
-
'apiKeys', 'capabilities', 'lifecycle',
|
|
52
|
+
'apiKeys', 'capabilities', 'rotation', 'lifecycle',
|
|
53
53
|
];
|
|
54
54
|
|
|
55
55
|
/** The shape `beginScan` has always returned, so no caller has to change. */
|
|
@@ -33,6 +33,7 @@ import {
|
|
|
33
33
|
cloudflareAccess,
|
|
34
34
|
KeyValueStore, MemoryKV, SqliteKV,
|
|
35
35
|
KvSessionStore,
|
|
36
|
+
RotationService,
|
|
36
37
|
SqliteProvider, LocalSqliteProvider,
|
|
37
38
|
SidecarService, NotificationCenter, WebPushService, WebPushChannel, NotificationChannel,
|
|
38
39
|
ApiKeyService, CapabilityProvider, ApiKeyCapabilityProvider,
|
|
@@ -330,6 +331,17 @@ export function coreProviders(config, lifecycleState) {
|
|
|
330
331
|
// and the task succeeding clears the issue.
|
|
331
332
|
tasks: Provider.fromLazySingleton(() => resolve(config.tasks, TaskRegistry, () => new TaskRegistry())),
|
|
332
333
|
|
|
334
|
+
// Moving a collection onto a new key, incrementally. Depends on vfs rather than the
|
|
335
|
+
// other way round, so it sits after it in the graph and the container orders itself.
|
|
336
|
+
rotation: Provider.fromLazySingleton(
|
|
337
|
+
async (deps) => {
|
|
338
|
+
const { kv, vfs, collections } = await need(deps, ['kv', 'vfs', 'collections']);
|
|
339
|
+
return new RotationService({ kv, vfs, collections });
|
|
340
|
+
},
|
|
341
|
+
null,
|
|
342
|
+
{ deps: ['kv', 'vfs', 'collections'] },
|
|
343
|
+
),
|
|
344
|
+
|
|
333
345
|
issues: Provider.fromLazySingleton(
|
|
334
346
|
async (deps) => {
|
|
335
347
|
const { kv } = await need(deps, ['kv']);
|
|
@@ -66,7 +66,7 @@ export async function createServer(config = {}) {
|
|
|
66
66
|
const backbone = await engine.container.lease(BACKBONE);
|
|
67
67
|
const {
|
|
68
68
|
storage, sqlite: sqliteProvider, metadata, kv, tasks, issues, notifications,
|
|
69
|
-
sidecar, collections, identity, auth, search, vfs, plugins, apiKeys, capabilities,
|
|
69
|
+
sidecar, collections, identity, auth, search, vfs, plugins, apiKeys, capabilities, rotation,
|
|
70
70
|
} = backbone.resources;
|
|
71
71
|
|
|
72
72
|
// Aliased so the rest of this function reads as it did; the container's
|
|
@@ -232,6 +232,36 @@ export async function createServer(config = {}) {
|
|
|
232
232
|
? null
|
|
233
233
|
: rebuildIndexIfLost(vfs, search, startReindex);
|
|
234
234
|
|
|
235
|
+
/**
|
|
236
|
+
* Advance every rotation that is part-way through, and say what moved.
|
|
237
|
+
*
|
|
238
|
+
* Extracted because there are TWO schedulers — the interval below, for a process that
|
|
239
|
+
* lives, and `runMaintenance`, for a Worker whose timers do not survive the request —
|
|
240
|
+
* and only the second one used to step rotations. On Bun and Node, which is where this
|
|
241
|
+
* is self-hosted, `POST /rotate` therefore minted the new key, reported "running", and
|
|
242
|
+
* then moved nothing, ever: `rotation.step` had no caller, and there is no route that
|
|
243
|
+
* steps one either. The old key stayed in the ring indefinitely, which is the exact
|
|
244
|
+
* failure a rotation exists to end.
|
|
245
|
+
*
|
|
246
|
+
* One function, both callers, so the next thing added to periodic work cannot land in
|
|
247
|
+
* one scheduler and not the other.
|
|
248
|
+
*/
|
|
249
|
+
async function stepRotations(targets, budgetEach) {
|
|
250
|
+
const out = [];
|
|
251
|
+
for (const c of targets) {
|
|
252
|
+
const state = await rotation.state(c.id).catch(() => null);
|
|
253
|
+
if (!state || state.status !== 'running') continue;
|
|
254
|
+
// The slice claims the collection, so a firing that overlaps a manual run does
|
|
255
|
+
// nothing rather than racing it.
|
|
256
|
+
const next = await rotation.step(c.id, { budgetMs: budgetEach }).catch((e) => {
|
|
257
|
+
console.error(`[trove] rotation slice for ${c.id} failed`, e);
|
|
258
|
+
return null;
|
|
259
|
+
});
|
|
260
|
+
if (next) out.push({ collectionId: c.id, moved: next.moved, failed: next.failed, status: next.status });
|
|
261
|
+
}
|
|
262
|
+
return out;
|
|
263
|
+
}
|
|
264
|
+
|
|
235
265
|
// Periodic maintenance. Both of these caches are otherwise unbounded: abandoned
|
|
236
266
|
// upload sessions (a client that starts an upload and never finishes) accumulate in
|
|
237
267
|
// the session store forever, and sidecar documents stay resident after their last
|
|
@@ -251,6 +281,12 @@ export async function createServer(config = {}) {
|
|
|
251
281
|
.then(() => sidecar.sweep())
|
|
252
282
|
.then(() => (trashMs > 0 ? vfs.purgeTrash({ before: Date.now() - trashMs }) : null))
|
|
253
283
|
.then((r) => { if (r?.purged) console.log(`[trove] purged ${r.purged} item(s) from the trash after ${config.trashRetentionDays ?? 30} days`); })
|
|
284
|
+
// A rotation started through the API finishes on its own here. Unlike a scan this
|
|
285
|
+
// is NOT opt-in: it only touches collections someone has explicitly put into
|
|
286
|
+
// rotation, and leaving one half-moved is worse than the work of finishing it.
|
|
287
|
+
.then(() => (collections ? collections.all().catch(() => []) : []))
|
|
288
|
+
.then((targets) => stepRotations(targets, Math.max(1000, Math.floor(everyMs / 4))))
|
|
289
|
+
.then((moved) => { for (const m of moved) if (m.moved) console.log(`[trove] rotation ${m.collectionId}: ${m.moved} moved, ${m.status}`); })
|
|
254
290
|
.catch((e) => console.error('maintenance sweep failed', e));
|
|
255
291
|
}, everyMs);
|
|
256
292
|
maintenance.unref?.();
|
|
@@ -471,6 +507,13 @@ export async function createServer(config = {}) {
|
|
|
471
507
|
const r = await startScan(c.id, { reason: 'Scheduled', deadlineMs: each }).catch((e) => ({ error: e.message }));
|
|
472
508
|
out.scans.push({ collectionId: c.id, ...r });
|
|
473
509
|
}
|
|
510
|
+
|
|
511
|
+
// A rotation that has been started finishes on its own — see `stepRotations`, which
|
|
512
|
+
// the interval scheduler shares so the two cannot drift apart.
|
|
513
|
+
//
|
|
514
|
+
// Last, and out of what the scans left, because a rotation is elective and a scan is
|
|
515
|
+
// how the drive notices files that changed underneath it.
|
|
516
|
+
out.rotated = await stepRotations(targets, each);
|
|
474
517
|
return out;
|
|
475
518
|
}
|
|
476
519
|
|
|
@@ -482,7 +525,7 @@ export async function createServer(config = {}) {
|
|
|
482
525
|
// inside a Durable Object want. `begin*` goes wherever `config.background` says,
|
|
483
526
|
// which for a front-line Worker isolate is the object rather than itself.
|
|
484
527
|
startScan, startReindex, beginScan: routeBeginScan, beginReindex: routeBeginReindex,
|
|
485
|
-
runMaintenance, checkStorage, mcp, auth, close };
|
|
528
|
+
runMaintenance, checkStorage, rotation, mcp, auth, close };
|
|
486
529
|
}
|
|
487
530
|
|
|
488
531
|
/**
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import { Router, json, parseRange } from './router.js';
|
|
6
6
|
import {
|
|
7
7
|
TroveError, assertSafePluginSql, concatBytes, metadataUrl, publicOrigin,
|
|
8
|
-
shouldEncrypt,
|
|
8
|
+
shouldEncrypt, estimateRotationCost,
|
|
9
9
|
} from '@3sln/trove/core';
|
|
10
10
|
import { parseContribUri, CORE_DOMAIN } from '@3sln/trove/core/plugins/identity.js';
|
|
11
11
|
|
|
@@ -449,16 +449,16 @@ export function createRouter() {
|
|
|
449
449
|
const b = await body(ctx.req);
|
|
450
450
|
if (!b.name) throw TroveError.invalid('name is required');
|
|
451
451
|
const collection = await ctx.access.collection(scopedCollection(ctx), 'write');
|
|
452
|
-
//
|
|
453
|
-
// can seal the bytes before they reach the bucket. That key decrypts EVERYTHING in the
|
|
454
|
-
// collection, which makes it a read capability however it arrives — and `write` does
|
|
455
|
-
// not imply `read` here (only `admin` expands). Without this, a write-only API key,
|
|
456
|
-
// which the key model explicitly supports, could ask for a plan for a one-byte file and
|
|
457
|
-
// receive the means to decrypt the whole collection.
|
|
452
|
+
// `write` is enough again, including on an encrypted collection.
|
|
458
453
|
//
|
|
459
|
-
//
|
|
460
|
-
//
|
|
461
|
-
|
|
454
|
+
// This used to demand `read` as well, for a good reason that has since gone away: the
|
|
455
|
+
// plan handed the client the collection's data key so it could seal locally, and that
|
|
456
|
+
// key decrypts EVERYTHING in the collection — a read capability however it arrives. So
|
|
457
|
+
// a write-only ingest credential, which the key model explicitly supports, could not
|
|
458
|
+
// upload to an encrypted collection at all.
|
|
459
|
+
//
|
|
460
|
+
// The drive seals now and the key never leaves it, so a plan carries nothing readable
|
|
461
|
+
// and there is nothing left to protect. A write-only credential works everywhere.
|
|
462
462
|
return uploadDescriptor(await collection.createUpload({
|
|
463
463
|
name: b.name, size: Number(b.size ?? 0), contentType: b.contentType,
|
|
464
464
|
overwrite: b.overwrite === true,
|
|
@@ -684,6 +684,55 @@ export function createRouter() {
|
|
|
684
684
|
return ctx.storageCheck.run({ origin: publicOrigin(ctx.req, ctx.config) });
|
|
685
685
|
});
|
|
686
686
|
|
|
687
|
+
// --- key rotation ----------------------------------------------------------
|
|
688
|
+
//
|
|
689
|
+
// Moving a collection onto a new key is admin work in the strict sense: it rewrites every
|
|
690
|
+
// object in the collection and costs real money on a metered store. `requireHumanAdmin`
|
|
691
|
+
// refuses a request arriving on an API key — a credential that could re-key a collection
|
|
692
|
+
// could also make its contents unreadable to everyone else holding the old one.
|
|
693
|
+
|
|
694
|
+
/** What a rotation would cost, before anyone starts one. */
|
|
695
|
+
r.get('/api/collections/:collection/rotate/estimate', ['collections', 'vfs'], async (ctx) => {
|
|
696
|
+
const collectionId = scopedCollection(ctx);
|
|
697
|
+
await ctx.access.collection(collectionId, 'admin');
|
|
698
|
+
requireHumanAdmin(ctx, 'estimate a key rotation');
|
|
699
|
+
const record = await ctx.collections.get(collectionId);
|
|
700
|
+
const stats = await ctx.vfs.metadata.collectionStats?.(collectionId).catch(() => null);
|
|
701
|
+
return estimateRotationCost(
|
|
702
|
+
{ driver: record.store?.driver, endpoint: record.store?.endpoint || record.store?.s3?.endpoint },
|
|
703
|
+
{ objects: stats?.items ?? 0, bytes: stats?.bytes ?? 0 },
|
|
704
|
+
);
|
|
705
|
+
});
|
|
706
|
+
|
|
707
|
+
r.get('/api/collections/:collection/rotate', ['collections', 'rotation'], async (ctx) => {
|
|
708
|
+
const collectionId = scopedCollection(ctx);
|
|
709
|
+
await ctx.access.collection(collectionId, 'admin');
|
|
710
|
+
requireHumanAdmin(ctx, 'read key rotation state');
|
|
711
|
+
// Null rather than 404: "this collection has never been rotated" is an answer, and a
|
|
712
|
+
// client polling for progress should not have to treat it as an error.
|
|
713
|
+
return { rotation: (await ctx.rotation.state(collectionId)) || null };
|
|
714
|
+
});
|
|
715
|
+
|
|
716
|
+
r.post('/api/collections/:collection/rotate', ['collections', 'rotation'], async (ctx) => {
|
|
717
|
+
const collectionId = scopedCollection(ctx);
|
|
718
|
+
await ctx.access.collection(collectionId, 'admin');
|
|
719
|
+
requireHumanAdmin(ctx, 'rotate a key');
|
|
720
|
+
// Begin only mints the key and makes it current; the objects move in slices, from the
|
|
721
|
+
// cron or from further calls. Returning immediately is the point — the walk can take
|
|
722
|
+
// hours and holding the request open for it would just time out.
|
|
723
|
+
const state = await ctx.rotation.begin(collectionId, ctx.principal);
|
|
724
|
+
return { rotation: state };
|
|
725
|
+
});
|
|
726
|
+
|
|
727
|
+
r.delete('/api/collections/:collection/rotate', ['collections', 'rotation'], async (ctx) => {
|
|
728
|
+
const collectionId = scopedCollection(ctx);
|
|
729
|
+
await ctx.access.collection(collectionId, 'admin');
|
|
730
|
+
requireHumanAdmin(ctx, 'cancel a key rotation');
|
|
731
|
+
// Stops the walk. What has already moved stays moved, and both keys stay in the ring,
|
|
732
|
+
// so nothing becomes unreadable — an abandoned rotation is untidy, not destructive.
|
|
733
|
+
return { rotation: await ctx.rotation.cancel(collectionId) };
|
|
734
|
+
});
|
|
735
|
+
|
|
687
736
|
// Rebuild the search index on demand. Admin-only: it re-reads every object in the
|
|
688
737
|
// drive, so it is a real load, and it is drive-wide rather than scoped to anything
|
|
689
738
|
// the caller owns. Returns the task, which is how the caller watches it.
|
|
@@ -1162,22 +1211,6 @@ async function assertTaskAccess(ctx, task, what) {
|
|
|
1162
1211
|
* outlive its own revocation. Then the ordinary admin check on the principal.
|
|
1163
1212
|
*/
|
|
1164
1213
|
|
|
1165
|
-
/**
|
|
1166
|
-
* An upload plan that will carry the collection key needs `read`, not merely `write`.
|
|
1167
|
-
*
|
|
1168
|
-
* Checked before the session is created, so a refusal costs nothing and leaves no orphan.
|
|
1169
|
-
*/
|
|
1170
|
-
async function assertReadIfKeyed(ctx, body) {
|
|
1171
|
-
if (!ctx.collections?.encryptionFor) return;
|
|
1172
|
-
const collectionId = scopedCollection(ctx);
|
|
1173
|
-
const encryption = await ctx.collections.encryptionFor(collectionId);
|
|
1174
|
-
if (!encryption?.enabled) return;
|
|
1175
|
-
const contentType = body.contentType || ctx.vfs?.guessContentType?.(body.name) || '';
|
|
1176
|
-
if (!shouldEncrypt(encryption, { name: body.name, contentType })) return;
|
|
1177
|
-
// Throws if the caller does not hold read on this collection.
|
|
1178
|
-
await ctx.access.collection(collectionId, 'read');
|
|
1179
|
-
}
|
|
1180
|
-
|
|
1181
1214
|
function requireHumanAdmin(ctx, action) {
|
|
1182
1215
|
if (ctx.grant) {
|
|
1183
1216
|
throw TroveError.forbidden(`An API key cannot ${action} — sign in as an administrator`);
|