@3sln/trove 0.0.11 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/package.json +2 -1
- package/packages/core/src/collections/index.js +71 -10
- package/packages/core/src/encryption/keys.js +4 -3
- package/packages/core/src/encryption/policy.js +14 -7
- package/packages/core/src/encryption/rotation.js +46 -12
- package/packages/core/src/errors.js +8 -0
- package/packages/core/src/index.js +8 -2
- package/packages/core/src/indexers/contribution.js +18 -17
- package/packages/core/src/indexing.js +8 -4
- package/packages/core/src/issues.js +8 -0
- package/packages/core/src/links.js +21 -9
- package/packages/core/src/metadata/interface.js +22 -0
- package/packages/core/src/metadata/memory.js +15 -0
- package/packages/core/src/metadata/sqlite.js +23 -1
- package/packages/core/src/plugins/contributions.js +13 -0
- package/packages/core/src/rateLimit.js +253 -0
- package/packages/core/src/scan.js +44 -16
- package/packages/core/src/sidecar/document.js +18 -14
- package/packages/core/src/sidecar/index.js +9 -10
- package/packages/core/src/sidecar/manager.js +10 -15
- package/packages/core/src/uploads.js +40 -15
- package/packages/core/src/vfs.js +59 -34
- package/packages/plugin-sdk/src/browser.js +155 -0
- package/packages/plugin-sdk/src/protocol.js +13 -1
- package/packages/server/src/access/externalEvaluation.js +17 -0
- package/packages/server/src/adapters/bun.js +1 -2
- package/packages/server/src/adapters/node.js +1 -2
- package/packages/server/src/engine/index.js +1 -1
- package/packages/server/src/engine/providers/access.js +17 -34
- package/packages/server/src/engine/providers/core.js +112 -14
- package/packages/server/src/index.js +160 -105
- package/packages/server/src/mcp/index.js +7 -12
- package/packages/server/src/mcp/tools.js +24 -29
- package/packages/server/src/router.js +31 -8
- package/packages/server/src/routes.js +74 -113
- package/packages/server/src/scope.js +78 -0
- package/packages/web/dist/assets/main-828yzsr7.js +511 -0
- package/packages/web/dist/assets/main-828yzsr7.js.map +120 -0
- package/packages/web/dist/index.html +1 -1
- package/packages/web/dist/sw.js +1 -1
- package/packages/web/src/bl/actions.js +226 -104
- package/packages/web/src/bl/activity.js +32 -82
- package/packages/web/src/bl/index.js +23 -11
- package/packages/web/src/bl/launcher.js +2 -9
- package/packages/web/src/bl/links.js +31 -18
- package/packages/web/src/bl/offline.js +24 -10
- package/packages/web/src/bl/pluginInstall.js +29 -26
- package/packages/web/src/bl/queries.js +76 -68
- package/packages/web/src/bl/services.js +22 -14
- package/packages/web/src/bl/social.js +39 -25
- package/packages/web/src/bl/state.js +22 -4
- package/packages/web/src/bl/viewState.js +9 -0
- package/packages/web/src/bl/watchQuery.js +1 -8
- package/packages/web/src/dispatch.js +26 -0
- package/packages/web/src/platform/api.js +32 -0
- package/packages/web/src/platform/commands.js +14 -15
- package/packages/web/src/platform/context.js +3 -58
- package/packages/web/src/platform/contributions.js +1 -35
- package/packages/web/src/platform/fileChunks.js +281 -0
- package/packages/web/src/platform/index.js +8 -9
- package/packages/web/src/platform/mediaUrls.js +10 -0
- package/packages/web/src/platform/navigation.js +35 -21
- package/packages/web/src/platform/pickers.js +39 -19
- package/packages/web/src/platform/pluginHost.js +4 -1
- package/packages/web/src/platform/pluginRpc.js +47 -2
- package/packages/web/src/platform/spatialNav.js +1 -1
- package/packages/web/src/platform/viewport.js +14 -6
- package/packages/web/src/platform/voiceSearch.js +17 -9
- package/packages/web/src/platform/whenclause.js +9 -19
- package/packages/web/src/runtime.js +2 -37
- package/packages/web/src/ui/activate.js +18 -2
- package/packages/web/src/ui/components/activityPanel.js +1 -1
- package/packages/web/src/ui/components/adminView.js +4 -7
- package/packages/web/src/ui/components/editorArea.js +10 -1
- package/packages/web/src/ui/components/launcher.js +29 -13
- package/packages/web/src/ui/components/openers/index.js +17 -22
- package/packages/web/src/ui/components/openers/markdown.js +2 -2
- package/packages/web/src/ui/components/overlays.js +31 -5
- package/packages/web/src/ui/components/pluginReview.js +7 -4
- package/packages/web/src/ui/components/pluginsView.js +1 -1
- package/packages/web/src/ui/components/settingsView.js +30 -27
- package/packages/web/src/ui/components/statusBar.js +8 -6
- package/packages/web/src/ui/components/views/grid.js +2 -2
- package/packages/web/src/ui/components/views/parts.js +1 -1
- package/packages/web/src/ui/compositions/workbench.js +20 -6
- package/packages/web/src/ui/sanitize.js +0 -5
- package/packages/web/src/workbench.js +37 -19
- package/packages/web/dist/assets/main-c9dnnnc6.js +0 -356
- package/packages/web/dist/assets/main-c9dnnnc6.js.map +0 -118
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// Waiting for a dispatched action to actually finish.
|
|
2
|
+
//
|
|
3
|
+
// `dispatch(action)` answers an event FEED — `DispatchFeed extends EventTarget`, with no
|
|
4
|
+
// `then` — and ngin schedules the body on a `setTimeout`. So `await engine.dispatch(x)`
|
|
5
|
+
// resolves immediately and is *guaranteed* to return before the action has started. It
|
|
6
|
+
// reads as sequencing and is the opposite of it.
|
|
7
|
+
//
|
|
8
|
+
// Six sites did it, one of them eighty lines below the comment in the same class explaining
|
|
9
|
+
// why it cannot work, and at least one was visible: creating a collection navigated to it
|
|
10
|
+
// before the collections list had loaded, so the status bar showed a raw `col_…` id where
|
|
11
|
+
// the name belongs.
|
|
12
|
+
//
|
|
13
|
+
// The completion signal is the terminal event. It RESOLVES with whichever of
|
|
14
|
+
// complete/error/abort fired rather than rejecting, so a failing action does not throw at
|
|
15
|
+
// the call site — the caller decides whether the next step still makes sense.
|
|
16
|
+
//
|
|
17
|
+
// Takes anything with a `dispatch`, so the engine and CommandService (which dispatches
|
|
18
|
+
// through its own seam) share one implementation instead of two spellings of it.
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* @param {{dispatch: (action: object) => {next: (types: string[]) => Promise<object>}}} dispatcher
|
|
22
|
+
* @param {object} action
|
|
23
|
+
* @returns {Promise<{type: 'complete'|'error'|'abort'}>}
|
|
24
|
+
*/
|
|
25
|
+
export const runAction = (dispatcher, action) =>
|
|
26
|
+
dispatcher.dispatch(action).next(['complete', 'error', 'abort']);
|
|
@@ -371,6 +371,38 @@ export class TroveApiClient {
|
|
|
371
371
|
return new TextDecoder().decode(await this.readBytes(id, opts));
|
|
372
372
|
}
|
|
373
373
|
|
|
374
|
+
/**
|
|
375
|
+
* Read `[start, end)` of a file.
|
|
376
|
+
*
|
|
377
|
+
* HALF-OPEN, like every other range in JavaScript — `slice`, `subarray`, `TypedArray`.
|
|
378
|
+
* HTTP's is inclusive, and the conversion happens at this one boundary rather than in
|
|
379
|
+
* every caller, because an off-by-one that only shows up as a missing final byte of a
|
|
380
|
+
* container header is the kind of bug that reads as a corrupt file.
|
|
381
|
+
*
|
|
382
|
+
* `etag` comes back with the bytes, and callers are expected to care: a file overwritten
|
|
383
|
+
* in place keeps its id, so anything cached off these bytes — a chunk store, a parsed
|
|
384
|
+
* atom layout — has to notice that the head of the old file and the tail of the new one
|
|
385
|
+
* are not the same file. `total` is parsed from `content-range`, which is the only place
|
|
386
|
+
* a ranged read learns how big the whole thing is.
|
|
387
|
+
*
|
|
388
|
+
* @returns {Promise<{bytes: Uint8Array, etag: string|null, total: number|null}>}
|
|
389
|
+
*/
|
|
390
|
+
async readRange(id, { start = 0, end, signal } = {}) {
|
|
391
|
+
const range = end == null ? `bytes=${start}-` : `bytes=${start}-${end - 1}`;
|
|
392
|
+
const res = await this._fetch(this.downloadUrl(id), {
|
|
393
|
+
signal,
|
|
394
|
+
headers: { ...this.authHeaders(), range },
|
|
395
|
+
});
|
|
396
|
+
// A zero-length file has no satisfiable range, and asking for one is not an error the
|
|
397
|
+
// caller can do anything with — it is an empty file, which is a legitimate answer.
|
|
398
|
+
if (res.status === 416) return { bytes: new Uint8Array(0), etag: res.headers.get('etag'), total: 0 };
|
|
399
|
+
if (!res.ok && res.status !== 206) throw new TroveError('internal', `Read failed (${res.status})`);
|
|
400
|
+
const bytes = new Uint8Array(await res.arrayBuffer());
|
|
401
|
+
const total = Number(/\/(\d+)$/.exec(res.headers.get('content-range') || '')?.[1])
|
|
402
|
+
|| (res.status === 200 ? bytes.length : null);
|
|
403
|
+
return { bytes, etag: res.headers.get('etag'), total };
|
|
404
|
+
}
|
|
405
|
+
|
|
374
406
|
/**
|
|
375
407
|
* Read at most `maxBytes` of a file as text.
|
|
376
408
|
*
|
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
// plain functions, plugin handlers proxy over RPC). Errors surface as a
|
|
6
6
|
// notification instead of a silent console log — failure is always visible.
|
|
7
7
|
|
|
8
|
+
import { runAction } from '../dispatch.js';
|
|
9
|
+
|
|
8
10
|
export class CommandService {
|
|
9
11
|
/**
|
|
10
12
|
* @param {import('./contributions.js').ContributionRegistry} contributions
|
|
@@ -56,10 +58,6 @@ export class CommandService {
|
|
|
56
58
|
};
|
|
57
59
|
}
|
|
58
60
|
|
|
59
|
-
has(id) {
|
|
60
|
-
return this.handlers.has(id);
|
|
61
|
-
}
|
|
62
|
-
|
|
63
61
|
isEnabled(id) {
|
|
64
62
|
const cmd = this.contributions.get(id);
|
|
65
63
|
if (cmd?.when && !this.context.evaluate(cmd.when)) return false;
|
|
@@ -91,25 +89,26 @@ export class CommandService {
|
|
|
91
89
|
}
|
|
92
90
|
const cmd = this.contributions.get(id);
|
|
93
91
|
if (!this.isAvailable(cmd)) {
|
|
94
|
-
|
|
92
|
+
// Not "…isn't available OFFLINE": that word was printed whenever an availability
|
|
93
|
+
// hook was installed, which platform/index.js makes unconditionally true, and
|
|
94
|
+
// `#availableSpec` answers false for a plugin frame that is inactive or has stopped
|
|
95
|
+
// answering its heartbeat BEFORE it ever consults `this.online`. So a crashed iframe
|
|
96
|
+
// on a perfectly connected machine told the user to check their network.
|
|
97
|
+
this.notifications.warn(`“${cmd?.title || id}” isn’t available right now.`);
|
|
95
98
|
return;
|
|
96
99
|
}
|
|
97
100
|
if (!this.isEnabled(id)) return; // gated by when-clause
|
|
98
101
|
try {
|
|
99
102
|
const actions = actionsFor(...args);
|
|
100
103
|
if (!actions) return;
|
|
101
|
-
// One at a time, each genuinely finished before the next begins
|
|
102
|
-
//
|
|
103
|
-
//
|
|
104
|
-
// immediately and the action runs afterwards, so a bare `await dispatch(a)` in a loop
|
|
105
|
-
// would start all of them at once while looking like it sequenced them. `next` on the
|
|
106
|
-
// terminal events is the actual completion signal.
|
|
104
|
+
// One at a time, each genuinely finished before the next begins — see
|
|
105
|
+
// src/dispatch.js for why a bare `await dispatch(a)` in this loop would start them
|
|
106
|
+
// all at once while looking like it sequenced them.
|
|
107
107
|
//
|
|
108
|
-
//
|
|
109
|
-
//
|
|
110
|
-
// whose first step failed rarely wants its second.
|
|
108
|
+
// A failing action does not throw here, but it does stop the rest: a command whose
|
|
109
|
+
// first step failed rarely wants its second.
|
|
111
110
|
for (const action of [].concat(actions)) {
|
|
112
|
-
const settled = await this
|
|
111
|
+
const settled = await runAction(this, action);
|
|
113
112
|
if (settled?.type !== 'complete') break;
|
|
114
113
|
}
|
|
115
114
|
} catch (err) {
|
|
@@ -21,8 +21,8 @@
|
|
|
21
21
|
//
|
|
22
22
|
// Nothing sets a built-in key now. They are derived — see bl/context.js.
|
|
23
23
|
|
|
24
|
-
import { cell
|
|
25
|
-
import { evaluateWhen
|
|
24
|
+
import { cell } from '../runtime.js';
|
|
25
|
+
import { evaluateWhen } from './whenclause.js';
|
|
26
26
|
|
|
27
27
|
/** A cell whose value never changes — for facts about the machine rather than the drive. */
|
|
28
28
|
const constant = (value) => ({ onDirty: () => () => {}, getValue: () => value });
|
|
@@ -30,7 +30,6 @@ const constant = (value) => ({ onDirty: () => () => {}, getValue: () => value })
|
|
|
30
30
|
export class ContextRegistry {
|
|
31
31
|
#cells = new Map(); // key -> a Cell its owner holds
|
|
32
32
|
#offs = new Map(); // key -> unsubscribe
|
|
33
|
-
#slots = new Map(); // key -> a stable cell that outlives any one owner
|
|
34
33
|
#snapshot = cell({});
|
|
35
34
|
|
|
36
35
|
constructor(initial = {}) {
|
|
@@ -73,59 +72,8 @@ export class ContextRegistry {
|
|
|
73
72
|
this.#changed(key); // and so is losing one
|
|
74
73
|
}
|
|
75
74
|
|
|
76
|
-
|
|
77
|
-
* A cell for one key, whether or not anything owns it yet.
|
|
78
|
-
*
|
|
79
|
-
* The indirection is the point. A keymap naming a plugin's register is parsed long before
|
|
80
|
-
* that plugin installs, and the plugin may later be uninstalled — so a watcher cannot
|
|
81
|
-
* hold the OWNER's cell, which does not exist at either end. It holds this slot, which
|
|
82
|
-
* reads through to whoever currently owns the key and goes dirty when ownership changes
|
|
83
|
-
* as well as when the value does. Unowned reads as `undefined`, which every clause
|
|
84
|
-
* already treats as falsy.
|
|
85
|
-
*
|
|
86
|
-
* Slots are kept once created. They are bounded by the number of distinct keys any
|
|
87
|
-
* when-clause has ever named, which is small and does not grow with use.
|
|
88
|
-
*/
|
|
89
|
-
cellFor(key) {
|
|
90
|
-
let slot = this.#slots.get(key);
|
|
91
|
-
if (!slot) {
|
|
92
|
-
const listeners = new Set();
|
|
93
|
-
slot = {
|
|
94
|
-
listeners,
|
|
95
|
-
onDirty: (fn) => { listeners.add(fn); return () => listeners.delete(fn); },
|
|
96
|
-
getValue: () => this.#cells.get(key)?.getValue(),
|
|
97
|
-
};
|
|
98
|
-
this.#slots.set(key, slot);
|
|
99
|
-
}
|
|
100
|
-
return slot;
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
/**
|
|
104
|
-
* A when-clause as a live boolean.
|
|
105
|
-
*
|
|
106
|
-
* Derived over exactly the keys the expression NAMES (the parser collects them — see
|
|
107
|
-
* whenclause.js), so a clause about `view.active` does not recompute when the selection
|
|
108
|
-
* changes. That is the difference from `evaluate`, which reads the whole snapshot: a
|
|
109
|
-
* consumer holding many clauses — the palette holds one per registered command — stops
|
|
110
|
-
* re-running all of them on every unrelated change.
|
|
111
|
-
*
|
|
112
|
-
* A clause naming nothing derives over nothing and is computed once, which is right: a
|
|
113
|
-
* constant cannot become false.
|
|
114
|
-
*/
|
|
115
|
-
watch(expr) {
|
|
116
|
-
const predicate = compileWhen(expr);
|
|
117
|
-
const keys = predicate.keys ?? [];
|
|
118
|
-
return derive(keys.map((key) => this.cellFor(key)), (...values) => {
|
|
119
|
-
const ctx = {};
|
|
120
|
-
for (let i = 0; i < keys.length; i++) ctx[keys[i]] = values[i];
|
|
121
|
-
return predicate(ctx);
|
|
122
|
-
});
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
#changed(key) {
|
|
75
|
+
#changed() {
|
|
126
76
|
this.#recompute();
|
|
127
|
-
const slot = this.#slots.get(key);
|
|
128
|
-
if (slot) for (const fn of [...slot.listeners]) fn();
|
|
129
77
|
}
|
|
130
78
|
|
|
131
79
|
/**
|
|
@@ -154,9 +102,6 @@ export class ContextRegistry {
|
|
|
154
102
|
get(key) {
|
|
155
103
|
return this.#cells.get(key)?.getValue();
|
|
156
104
|
}
|
|
157
|
-
has(key) {
|
|
158
|
-
return this.#cells.has(key);
|
|
159
|
-
}
|
|
160
105
|
observe() {
|
|
161
106
|
return this.#snapshot;
|
|
162
107
|
}
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
// keymap { bindings:[{key, command, when?, args?}] }
|
|
24
24
|
// view { title, icon?, match?, priority?, render, move? }
|
|
25
25
|
|
|
26
|
-
import { cell
|
|
26
|
+
import { cell } from '../runtime.js';
|
|
27
27
|
import { selectorMatches } from '@3sln/trove/core/util.js';
|
|
28
28
|
import { parseContribUri, coreUri, CONTRIB_SCHEME } from '@3sln/trove/core/plugins/identity.js';
|
|
29
29
|
import { CONTRIBUTION_TYPES as PACKAGE_TYPES } from '@3sln/trove/core/plugins/contributions.js';
|
|
@@ -47,7 +47,6 @@ export class ContributionRegistry {
|
|
|
47
47
|
constructor() {
|
|
48
48
|
this.items = new Map(); // uri -> { uri, type, id, name, pluginId, ...options }
|
|
49
49
|
this.cell = cell([]);
|
|
50
|
-
this.byType = new Map(); // type -> derived cell (lazily created)
|
|
51
50
|
}
|
|
52
51
|
|
|
53
52
|
#emit() {
|
|
@@ -94,13 +93,6 @@ export class ContributionRegistry {
|
|
|
94
93
|
unregister(nameOrUri) {
|
|
95
94
|
if (this.items.delete(toUri(nameOrUri))) this.#emit();
|
|
96
95
|
}
|
|
97
|
-
/** Drop everything a plugin contributed (uninstall / reload). */
|
|
98
|
-
unregisterPlugin(pluginId) {
|
|
99
|
-
let changed = false;
|
|
100
|
-
for (const [uri, c] of this.items) if (c.pluginId === pluginId) { this.items.delete(uri); changed = true; }
|
|
101
|
-
if (changed) this.#emit();
|
|
102
|
-
}
|
|
103
|
-
|
|
104
96
|
get(nameOrUri) {
|
|
105
97
|
return nameOrUri ? this.items.get(toUri(nameOrUri)) || null : null;
|
|
106
98
|
}
|
|
@@ -110,29 +102,9 @@ export class ContributionRegistry {
|
|
|
110
102
|
ofType(type) {
|
|
111
103
|
return this.all().filter((c) => c.type === type);
|
|
112
104
|
}
|
|
113
|
-
/** One plugin's contributions, optionally of a single type. */
|
|
114
|
-
ofPlugin(pluginId, type) {
|
|
115
|
-
return this.all().filter((c) => c.pluginId === pluginId && (!type || c.type === type));
|
|
116
|
-
}
|
|
117
105
|
observe() {
|
|
118
106
|
return this.cell;
|
|
119
107
|
}
|
|
120
|
-
/**
|
|
121
|
-
* A reactive view of one type (status items, openers, …).
|
|
122
|
-
*
|
|
123
|
-
* Derived rather than a second subject fanned out to by hand: `derive` recomputes
|
|
124
|
-
* from the one list, and a watcher of the result is only re-rendered when its own
|
|
125
|
-
* slice actually differs — so registering an opener no longer redraws the status bar.
|
|
126
|
-
*/
|
|
127
|
-
observeType(type) {
|
|
128
|
-
let view = this.byType.get(type);
|
|
129
|
-
if (!view) {
|
|
130
|
-
view = derive([this.cell], (all) => all.filter((c) => c.type === type));
|
|
131
|
-
this.byType.set(type, view);
|
|
132
|
-
}
|
|
133
|
-
return view;
|
|
134
|
-
}
|
|
135
|
-
|
|
136
108
|
// --- typed lookups ---------------------------------------------------------
|
|
137
109
|
|
|
138
110
|
/** Every opener whose selector matches `node`, best (highest priority) first. */
|
|
@@ -142,12 +114,6 @@ export class ContributionRegistry {
|
|
|
142
114
|
.sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0));
|
|
143
115
|
}
|
|
144
116
|
|
|
145
|
-
/** Pick the best opener for a node, honouring `when` and availability. */
|
|
146
|
-
openerFor(node, evaluate, isAvailable) {
|
|
147
|
-
return this.openersFor(node)
|
|
148
|
-
.find((o) => (!o.when || evaluate(o.when)) && (!isAvailable || isAvailable(o))) || null;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
117
|
/** Every keybinding from every registered keymap, in registration order. */
|
|
152
118
|
keybindings() {
|
|
153
119
|
return this.ofType('keymap').flatMap((k) =>
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
// Where a file's bytes come from, and which of them are worth keeping.
|
|
2
|
+
//
|
|
3
|
+
// A viewer reading a range asks here rather than asking the network, and the answer comes
|
|
4
|
+
// from the first of three places that has it:
|
|
5
|
+
//
|
|
6
|
+
// 1. A PINNED whole-file copy. bl/offline.js already puts the whole Response into Cache
|
|
7
|
+
// Storage under `mediaUrls.cacheKey(id)` — a deliberately stable key, so `unpin` can
|
|
8
|
+
// find what `pin` stored. Slicing that cached blob is disk-backed and costs nothing,
|
|
9
|
+
// so a pinned book plays with the network off and this tier comes almost free.
|
|
10
|
+
// 2. CHUNKS of a download in progress. Playing from the middle fetches the middle, and
|
|
11
|
+
// if that file is also being taken offline those bytes are worth keeping — so the
|
|
12
|
+
// read contributes them and the background filler skips them later.
|
|
13
|
+
// 3. The network, keeping NOTHING.
|
|
14
|
+
//
|
|
15
|
+
// THE THIRD CASE IS THE DEFAULT, and it is the rule the whole design hangs off: a plugin
|
|
16
|
+
// ranging over a file nobody asked to keep must not quietly fill the disk with it. Bytes
|
|
17
|
+
// are retained only for an item someone has actually asked to have offline, and `start(id)`
|
|
18
|
+
// is that asking. Until it is called this is a plain ranged reader with a cache lookup in
|
|
19
|
+
// front of it.
|
|
20
|
+
//
|
|
21
|
+
// THE ETAG IS IN THE KEY. A file overwritten in place keeps its id, so an id-keyed chunk
|
|
22
|
+
// cache would hand a reader the head of the old file and the tail of the new one. For a
|
|
23
|
+
// container format that is a parse failure, and a confusing one — the bytes are all valid,
|
|
24
|
+
// they are just from two different files. A changed etag makes every old chunk stop
|
|
25
|
+
// matching, and the sweep reclaims them.
|
|
26
|
+
|
|
27
|
+
const CHUNKS_CACHE = 'trove-chunks-v1';
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* 4 MiB.
|
|
31
|
+
*
|
|
32
|
+
* A long audiobook is a few hundred entries at this size rather than a few thousand, and
|
|
33
|
+
* Cache Storage is a real database — the per-entry overhead is what makes a small chunk
|
|
34
|
+
* size expensive, not the bytes. Big enough that one chunk usually covers a seek, small
|
|
35
|
+
* enough that the first one arrives quickly.
|
|
36
|
+
*/
|
|
37
|
+
export const CHUNK_SIZE = 4 * 1024 * 1024;
|
|
38
|
+
|
|
39
|
+
const chunkOf = (offset) => Math.floor(offset / CHUNK_SIZE);
|
|
40
|
+
const concat = (parts, total) => {
|
|
41
|
+
const out = new Uint8Array(total);
|
|
42
|
+
let at = 0;
|
|
43
|
+
for (const p of parts) { out.set(p, at); at += p.length; }
|
|
44
|
+
return out;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export class FileChunks {
|
|
48
|
+
/** @param {{api: object, mediaUrls: object}} deps */
|
|
49
|
+
constructor({ api, mediaUrls }) {
|
|
50
|
+
this.api = api;
|
|
51
|
+
this.mediaUrls = mediaUrls;
|
|
52
|
+
// What someone asked to keep: id -> { etag, total, cancel, filling, done, failed }.
|
|
53
|
+
// In memory because it is about THIS session's background work; what survives a reload
|
|
54
|
+
// is the chunks themselves, which is the durable half.
|
|
55
|
+
this.kept = new Map();
|
|
56
|
+
this.listeners = new Set();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Progress, for a viewer drawing a bar. `off()` to stop listening. */
|
|
60
|
+
onProgress(fn) {
|
|
61
|
+
this.listeners.add(fn);
|
|
62
|
+
return () => this.listeners.delete(fn);
|
|
63
|
+
}
|
|
64
|
+
#emit(id) {
|
|
65
|
+
const s = this.status(id);
|
|
66
|
+
for (const fn of [...this.listeners]) { try { fn(id, s); } catch { /* a listener is not our problem */ } }
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
#key(id, etag, index) {
|
|
70
|
+
// The stable download URL, plus a QUERY parameter. Not a fragment: the Cache API keys
|
|
71
|
+
// on a Request, and a Request's url excludes its fragment — so `…#chunk=0` and
|
|
72
|
+
// `…#chunk=1` are the same key, and every chunk of every file would collide on the
|
|
73
|
+
// first one. It reads as "part of this file" in devtools either way.
|
|
74
|
+
return `${this.mediaUrls.cacheKey(id)}&chunk=${etag || 'none'}:${CHUNK_SIZE}:${index}`;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async #cache(name) {
|
|
78
|
+
if (!('caches' in globalThis)) return null;
|
|
79
|
+
return caches.open(name).catch(() => null);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The pinned whole-file copy, if there is one.
|
|
84
|
+
*
|
|
85
|
+
* Tier 1, and the reason it is first: it needs no chunking, no etag bookkeeping and no
|
|
86
|
+
* network, because `bl/offline.js` already put the entire Response there.
|
|
87
|
+
*/
|
|
88
|
+
async #pinned(id) {
|
|
89
|
+
const cache = await this.#cache('trove-files-v1');
|
|
90
|
+
if (!cache) return null;
|
|
91
|
+
return cache.match(this.mediaUrls.cacheKey(id), { ignoreVary: true }).catch(() => null);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Read `[start, end)`.
|
|
96
|
+
*
|
|
97
|
+
* @param {string} id
|
|
98
|
+
* @param {{start?: number, end?: number, signal?: AbortSignal}} range
|
|
99
|
+
* @returns {Promise<{bytes: Uint8Array, etag: string|null, total: number|null}>}
|
|
100
|
+
*/
|
|
101
|
+
async read(id, { start = 0, end, signal } = {}) {
|
|
102
|
+
const whole = await this.#pinned(id);
|
|
103
|
+
if (whole) {
|
|
104
|
+
const blob = await whole.blob();
|
|
105
|
+
const stop = end == null ? blob.size : Math.min(end, blob.size);
|
|
106
|
+
const bytes = new Uint8Array(await blob.slice(start, stop).arrayBuffer());
|
|
107
|
+
return { bytes, etag: whole.headers.get('etag'), total: blob.size };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const keep = this.kept.get(id);
|
|
111
|
+
// Not kept: a plain ranged read, and nothing is written anywhere. This is the branch
|
|
112
|
+
// that runs for every file nobody asked to keep, which is almost all of them.
|
|
113
|
+
if (!keep) return this.api.readRange(id, { start, end, signal });
|
|
114
|
+
|
|
115
|
+
return this.#readChunked(id, keep, start, end, signal);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** The kept path: whole chunks, from the cache where possible, contributed on a miss. */
|
|
119
|
+
async #readChunked(id, keep, start, end, signal) {
|
|
120
|
+
const cache = await this.#cache(CHUNKS_CACHE);
|
|
121
|
+
if (!cache) return this.api.readRange(id, { start, end, signal });
|
|
122
|
+
|
|
123
|
+
// The total has to be known before the last chunk can be sized, and the cheapest way
|
|
124
|
+
// to learn it is the first read — which we were doing anyway.
|
|
125
|
+
if (keep.total == null) {
|
|
126
|
+
const head = await this.api.readRange(id, { start: 0, end: 1, signal });
|
|
127
|
+
keep.total = head.total;
|
|
128
|
+
keep.etag = head.etag;
|
|
129
|
+
}
|
|
130
|
+
const stop = Math.min(end == null ? keep.total : end, keep.total);
|
|
131
|
+
if (stop <= start) return { bytes: new Uint8Array(0), etag: keep.etag, total: keep.total };
|
|
132
|
+
|
|
133
|
+
const parts = [];
|
|
134
|
+
let got = 0;
|
|
135
|
+
for (let i = chunkOf(start); i <= chunkOf(stop - 1); i++) {
|
|
136
|
+
const chunk = await this.#chunk(id, keep, i, cache, signal);
|
|
137
|
+
// The window this chunk contributes, in the chunk's own coordinates.
|
|
138
|
+
const from = Math.max(0, start - i * CHUNK_SIZE);
|
|
139
|
+
const to = Math.min(chunk.length, stop - i * CHUNK_SIZE);
|
|
140
|
+
const piece = chunk.subarray(from, to);
|
|
141
|
+
parts.push(piece);
|
|
142
|
+
got += piece.length;
|
|
143
|
+
}
|
|
144
|
+
return { bytes: concat(parts, got), etag: keep.etag, total: keep.total };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** One chunk: cached if it is there, fetched and KEPT if it is not. */
|
|
148
|
+
async #chunk(id, keep, index, cache, signal) {
|
|
149
|
+
const key = this.#key(id, keep.etag, index);
|
|
150
|
+
const hit = await cache.match(key).catch(() => null);
|
|
151
|
+
if (hit) {
|
|
152
|
+
keep.have.add(index);
|
|
153
|
+
return new Uint8Array(await hit.arrayBuffer());
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const start = index * CHUNK_SIZE;
|
|
157
|
+
const end = Math.min(start + CHUNK_SIZE, keep.total);
|
|
158
|
+
const res = await this.api.readRange(id, { start, end, signal });
|
|
159
|
+
// The etag can change under us mid-download — someone replaced the file. Everything
|
|
160
|
+
// already stored is from a file that no longer exists, so it is dropped rather than
|
|
161
|
+
// mixed with what is arriving.
|
|
162
|
+
if (res.etag && keep.etag && res.etag !== keep.etag) {
|
|
163
|
+
await this.#dropChunks(id);
|
|
164
|
+
keep.have.clear();
|
|
165
|
+
keep.etag = res.etag;
|
|
166
|
+
keep.total = res.total ?? keep.total;
|
|
167
|
+
}
|
|
168
|
+
await cache.put(key, new Response(res.bytes)).catch(() => {});
|
|
169
|
+
keep.have.add(index);
|
|
170
|
+
this.#emit(id);
|
|
171
|
+
return res.bytes;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Keep this item offline, and fill in what is missing in the background.
|
|
176
|
+
*
|
|
177
|
+
* IDEMPOTENT: starting a download that is already running returns its status rather than
|
|
178
|
+
* racing a second filler over the same chunks — two fillers would each fetch every chunk
|
|
179
|
+
* the other had not written yet, which is the whole file twice.
|
|
180
|
+
*/
|
|
181
|
+
async start(id) {
|
|
182
|
+
const running = this.kept.get(id);
|
|
183
|
+
if (running) return this.status(id);
|
|
184
|
+
|
|
185
|
+
const head = await this.api.readRange(id, { start: 0, end: 1 });
|
|
186
|
+
const controller = new AbortController();
|
|
187
|
+
const keep = {
|
|
188
|
+
etag: head.etag, total: head.total, filling: true, done: false, failed: null,
|
|
189
|
+
// Which chunk indices are stored. `loaded` is DERIVED from this rather than counted
|
|
190
|
+
// up as bytes arrive, so a chunk written by a viewer's seek and a chunk written by
|
|
191
|
+
// the background filler are the same fact and neither double-counts the other.
|
|
192
|
+
have: new Set(),
|
|
193
|
+
controller,
|
|
194
|
+
};
|
|
195
|
+
this.kept.set(id, keep);
|
|
196
|
+
this.#emit(id);
|
|
197
|
+
// Not awaited: `start` answers immediately so a viewer can begin playing while the
|
|
198
|
+
// rest arrives. Failures land on the status, which is what a progress bar reads.
|
|
199
|
+
this.#fill(id, keep, controller.signal).catch((err) => {
|
|
200
|
+
keep.filling = false;
|
|
201
|
+
keep.failed = err?.message || String(err);
|
|
202
|
+
this.#emit(id);
|
|
203
|
+
});
|
|
204
|
+
return this.status(id);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** In order, skipping what is already there — which is what makes a seek pay for itself. */
|
|
208
|
+
async #fill(id, keep, signal) {
|
|
209
|
+
const cache = await this.#cache(CHUNKS_CACHE);
|
|
210
|
+
if (!cache) throw new Error('This browser has no Cache Storage, so nothing can be kept offline');
|
|
211
|
+
const count = Math.max(1, Math.ceil((keep.total || 0) / CHUNK_SIZE));
|
|
212
|
+
for (let i = 0; i < count; i++) {
|
|
213
|
+
if (signal.aborted) return;
|
|
214
|
+
await this.#chunk(id, keep, i, cache, signal);
|
|
215
|
+
}
|
|
216
|
+
keep.filling = false;
|
|
217
|
+
keep.done = true;
|
|
218
|
+
this.#emit(id);
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* How much of it is here.
|
|
223
|
+
*
|
|
224
|
+
* `loaded` counts what has been WRITTEN, not what has been asked for, so a bar built on
|
|
225
|
+
* it never runs ahead of the bytes.
|
|
226
|
+
*/
|
|
227
|
+
status(id) {
|
|
228
|
+
const keep = this.kept.get(id);
|
|
229
|
+
if (!keep) return { kept: false, loaded: 0, total: null, ratio: 0, filling: false, done: false, error: null };
|
|
230
|
+
const total = keep.total ?? null;
|
|
231
|
+
// The last chunk is short, so counting every stored chunk as a whole one would report
|
|
232
|
+
// more bytes than the file has.
|
|
233
|
+
const last = total ? Math.ceil(total / CHUNK_SIZE) - 1 : -1;
|
|
234
|
+
let loaded = 0;
|
|
235
|
+
for (const i of keep.have) loaded += i === last ? total - last * CHUNK_SIZE : CHUNK_SIZE;
|
|
236
|
+
return {
|
|
237
|
+
kept: true,
|
|
238
|
+
total,
|
|
239
|
+
loaded,
|
|
240
|
+
ratio: total ? Math.min(1, loaded / total) : 0,
|
|
241
|
+
filling: keep.filling,
|
|
242
|
+
done: keep.done,
|
|
243
|
+
error: keep.failed,
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/** Stop filling. What is already stored stays — a resumed download starts from it. */
|
|
248
|
+
cancel(id) {
|
|
249
|
+
const keep = this.kept.get(id);
|
|
250
|
+
if (!keep) return;
|
|
251
|
+
keep.controller?.abort();
|
|
252
|
+
keep.filling = false;
|
|
253
|
+
this.#emit(id);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/** Stop keeping it, and reclaim the bytes. */
|
|
257
|
+
async remove(id) {
|
|
258
|
+
const keep = this.kept.get(id);
|
|
259
|
+
this.cancel(id);
|
|
260
|
+
this.kept.delete(id);
|
|
261
|
+
keep?.have?.clear();
|
|
262
|
+
await this.#dropChunks(id);
|
|
263
|
+
this.#emit(id);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Drop every chunk for this file, whatever etag they were stored under.
|
|
268
|
+
*
|
|
269
|
+
* Prefix-matched rather than index-counted: after an etag change we no longer know how
|
|
270
|
+
* many chunks the old file had, and leaving them would be a leak nothing ever reclaims.
|
|
271
|
+
*/
|
|
272
|
+
async #dropChunks(id) {
|
|
273
|
+
const cache = await this.#cache(CHUNKS_CACHE);
|
|
274
|
+
if (!cache) return;
|
|
275
|
+
const prefix = `${this.mediaUrls.cacheKey(id)}&chunk=`;
|
|
276
|
+
const keys = await cache.keys().catch(() => []);
|
|
277
|
+
for (const req of keys) {
|
|
278
|
+
if (req.url.startsWith(prefix)) await cache.delete(req).catch(() => {});
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
@@ -20,6 +20,7 @@ import { ViewportService } from './viewport.js';
|
|
|
20
20
|
import { SpatialNavigationService } from './spatialNav.js';
|
|
21
21
|
import { VoiceSearchService } from './voiceSearch.js';
|
|
22
22
|
import { MediaUrlService } from './mediaUrls.js';
|
|
23
|
+
import { FileChunks } from './fileChunks.js';
|
|
23
24
|
|
|
24
25
|
/**
|
|
25
26
|
* The bearer token for this browser, if any.
|
|
@@ -33,14 +34,9 @@ import { MediaUrlService } from './mediaUrls.js';
|
|
|
33
34
|
export function readToken() {
|
|
34
35
|
try { return localStorage.getItem('trove.token') || null; } catch { return null; }
|
|
35
36
|
}
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
if (token) localStorage.setItem('trove.token', token);
|
|
40
|
-
else localStorage.removeItem('trove.token');
|
|
41
|
-
} catch { /* storage unavailable; the session is simply not persisted */ }
|
|
42
|
-
}
|
|
43
|
-
|
|
37
|
+
// Read only, deliberately: nothing in the workbench signs in. The key is written by the
|
|
38
|
+
// page that EMBEDS the workbench — see test/multiuser.e2e.mjs, which is the arrangement
|
|
39
|
+
// this is built for — so a writer here would be a second owner of somebody else's fact.
|
|
44
40
|
export function createPlatform({ baseUrl = '' } = {}) {
|
|
45
41
|
const contributions = new ContributionRegistry();
|
|
46
42
|
// Only the fixed facts are seeded. Everything else is registered by whoever owns it —
|
|
@@ -79,9 +75,12 @@ export function createPlatform({ baseUrl = '' } = {}) {
|
|
|
79
75
|
viewport, spatialNav, voice,
|
|
80
76
|
// The shell's own state is the engine's, not the platform's — createApp builds it and
|
|
81
77
|
// hands back what the few imperative edges below still need.
|
|
82
|
-
openPluginPanel: null,
|
|
83
78
|
};
|
|
84
79
|
platform.mediaUrls = new MediaUrlService({ api: platform.api, settings });
|
|
80
|
+
// Where a viewer's ranged reads go: a pinned copy, chunks of a download someone asked
|
|
81
|
+
// for, or the network keeping nothing. See fileChunks.js — the retention rule is the
|
|
82
|
+
// whole point of it having a name of its own.
|
|
83
|
+
platform.fileChunks = new FileChunks({ api: platform.api, mediaUrls: platform.mediaUrls });
|
|
85
84
|
platform.plugins = new PluginHost(platform);
|
|
86
85
|
// Commands consult the plugin host to hide/disable plugin commands that aren't
|
|
87
86
|
// available right now (offline, or the plugin isn't responding).
|
|
@@ -117,6 +117,16 @@ export class MediaUrlService {
|
|
|
117
117
|
try {
|
|
118
118
|
const res = await this.api.mintUrls(ids, op);
|
|
119
119
|
const now = Date.now();
|
|
120
|
+
// Sweep before inserting. An expired entry can never be served again — `#fresh`
|
|
121
|
+
// only decides whether a HIT is usable — and nothing else ever removed one: the
|
|
122
|
+
// sole `invalidate` caller is a media element's error handler. Browsing a large
|
|
123
|
+
// collection mints per pictorial tile, so the map grew an entry per image, none
|
|
124
|
+
// reachable and none collectable for the life of the page. Two lines, and unlike
|
|
125
|
+
// the interning table in bl/intern.js, evicting here costs a re-mint rather than a
|
|
126
|
+
// duplicate realization.
|
|
127
|
+
for (const [key, entry] of this.cache) {
|
|
128
|
+
if (entry.expiresAt && entry.expiresAt < now) this.cache.delete(key);
|
|
129
|
+
}
|
|
120
130
|
for (const [id, w] of waiting) {
|
|
121
131
|
const got = res.urls?.[id];
|
|
122
132
|
if (got) {
|