@3sln/trove 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1227 -0
- package/package.json +75 -0
- package/packages/core/src/collections/index.js +249 -0
- package/packages/core/src/errors.js +186 -0
- package/packages/core/src/identity/discovery.js +210 -0
- package/packages/core/src/identity/index.js +188 -0
- package/packages/core/src/identity/jwt.js +199 -0
- package/packages/core/src/index.js +104 -0
- package/packages/core/src/indexers/contribution.js +115 -0
- package/packages/core/src/indexers/registry.js +162 -0
- package/packages/core/src/indexing.js +340 -0
- package/packages/core/src/issues.js +150 -0
- package/packages/core/src/kv.js +0 -0
- package/packages/core/src/links.js +141 -0
- package/packages/core/src/metadata/cursor.js +73 -0
- package/packages/core/src/metadata/interface.js +244 -0
- package/packages/core/src/metadata/memory.js +270 -0
- package/packages/core/src/metadata/sqlite.js +412 -0
- package/packages/core/src/notifications/index.js +139 -0
- package/packages/core/src/notifications/webpush.js +217 -0
- package/packages/core/src/plugins/contributions.js +177 -0
- package/packages/core/src/plugins/identity.js +98 -0
- package/packages/core/src/plugins/index.js +225 -0
- package/packages/core/src/plugins/indexers.js +142 -0
- package/packages/core/src/plugins/installStore.js +134 -0
- package/packages/core/src/plugins/package.js +102 -0
- package/packages/core/src/plugins/packageStore.js +61 -0
- package/packages/core/src/plugins/runtime.js +101 -0
- package/packages/core/src/plugins/sql.js +52 -0
- package/packages/core/src/retry.js +74 -0
- package/packages/core/src/scan.js +302 -0
- package/packages/core/src/search/embeddings.js +128 -0
- package/packages/core/src/search/index.js +200 -0
- package/packages/core/src/search/keywordStore.js +107 -0
- package/packages/core/src/search/sqliteStores.js +455 -0
- package/packages/core/src/search/tagMatch.js +59 -0
- package/packages/core/src/search/transformer.js +195 -0
- package/packages/core/src/search/vectorStore.js +274 -0
- package/packages/core/src/search/vectorize.js +249 -0
- package/packages/core/src/sidecar/document.js +213 -0
- package/packages/core/src/sidecar/index.js +174 -0
- package/packages/core/src/sidecar/manager.js +239 -0
- package/packages/core/src/sidecar/store.js +46 -0
- package/packages/core/src/signedUrls.js +170 -0
- package/packages/core/src/sqlite-d1.js +162 -0
- package/packages/core/src/sqlite-driver.js +42 -0
- package/packages/core/src/sqlite.js +162 -0
- package/packages/core/src/storage/filesystem.js +283 -0
- package/packages/core/src/storage/interface.js +222 -0
- package/packages/core/src/storage/memory.js +113 -0
- package/packages/core/src/storage/prefixed.js +75 -0
- package/packages/core/src/storage/s3.js +316 -0
- package/packages/core/src/storage/s3sigv4.js +185 -0
- package/packages/core/src/tasks.js +228 -0
- package/packages/core/src/uploads.js +386 -0
- package/packages/core/src/util.js +125 -0
- package/packages/core/src/vfs.js +666 -0
- package/packages/plugin-sdk/src/browser.js +316 -0
- package/packages/plugin-sdk/src/index.js +32 -0
- package/packages/plugin-sdk/src/protocol.js +59 -0
- package/packages/plugin-sdk/src/rpc.js +95 -0
- package/packages/server/src/adapters/bun.js +78 -0
- package/packages/server/src/adapters/node.js +115 -0
- package/packages/server/src/adapters/staticAssets.js +123 -0
- package/packages/server/src/adapters/webDist.js +70 -0
- package/packages/server/src/adapters/worker-tasks.js +206 -0
- package/packages/server/src/adapters/worker.js +159 -0
- package/packages/server/src/cachePolicy.js +34 -0
- package/packages/server/src/engine/README.md +88 -0
- package/packages/server/src/engine/actions/scanCollection.js +114 -0
- package/packages/server/src/engine/index.js +95 -0
- package/packages/server/src/engine/lazy.js +25 -0
- package/packages/server/src/engine/providers/access.js +363 -0
- package/packages/server/src/engine/providers/core.js +405 -0
- package/packages/server/src/engine/providers/scan.js +67 -0
- package/packages/server/src/index.js +698 -0
- package/packages/server/src/manifest.js +98 -0
- package/packages/server/src/mcp/auth.js +40 -0
- package/packages/server/src/mcp/index.js +213 -0
- package/packages/server/src/mcp/protocol.js +181 -0
- package/packages/server/src/mcp/tools.js +351 -0
- package/packages/server/src/router.js +229 -0
- package/packages/server/src/routes.js +1066 -0
- package/packages/server/src/scope.js +43 -0
- package/packages/web/dist/assets/chunk-4xqbzebh.js +5 -0
- package/packages/web/dist/assets/chunk-4xqbzebh.js.map +9 -0
- package/packages/web/dist/assets/chunk-h05bxfbs.js +5 -0
- package/packages/web/dist/assets/chunk-h05bxfbs.js.map +10 -0
- package/packages/web/dist/assets/main-4cxs7prw.js +356 -0
- package/packages/web/dist/assets/main-4cxs7prw.js.map +103 -0
- package/packages/web/dist/assets/styles-kcx1x337.css +1 -0
- package/packages/web/dist/icon.svg +11 -0
- package/packages/web/dist/index.html +16 -0
- package/packages/web/dist/sql-wasm.wasm +0 -0
- package/packages/web/dist/sw.js +186 -0
- package/packages/web/src/bl/actions.js +410 -0
- package/packages/web/src/bl/activity.js +306 -0
- package/packages/web/src/bl/commands.js +274 -0
- package/packages/web/src/bl/fileType.js +49 -0
- package/packages/web/src/bl/index.js +70 -0
- package/packages/web/src/bl/links.js +54 -0
- package/packages/web/src/bl/offline.js +268 -0
- package/packages/web/src/bl/openers.js +71 -0
- package/packages/web/src/bl/pluginInstall.js +59 -0
- package/packages/web/src/bl/services.js +143 -0
- package/packages/web/src/bl/social.js +234 -0
- package/packages/web/src/bl/tagQuery.js +44 -0
- package/packages/web/src/main.js +10 -0
- package/packages/web/src/platform/api.js +529 -0
- package/packages/web/src/platform/commands.js +89 -0
- package/packages/web/src/platform/context.js +77 -0
- package/packages/web/src/platform/contributions.js +156 -0
- package/packages/web/src/platform/index.js +150 -0
- package/packages/web/src/platform/keybindings.js +199 -0
- package/packages/web/src/platform/mediaUrls.js +137 -0
- package/packages/web/src/platform/navigation.js +131 -0
- package/packages/web/src/platform/notifications.js +50 -0
- package/packages/web/src/platform/overlay.js +81 -0
- package/packages/web/src/platform/pluginClientDb.js +132 -0
- package/packages/web/src/platform/pluginDock.js +141 -0
- package/packages/web/src/platform/pluginFrames.js +194 -0
- package/packages/web/src/platform/pluginHost.js +648 -0
- package/packages/web/src/platform/pluginMedia.js +62 -0
- package/packages/web/src/platform/pluginModules.js +90 -0
- package/packages/web/src/platform/pluginNet.js +71 -0
- package/packages/web/src/platform/pluginPackage.js +247 -0
- package/packages/web/src/platform/pluginRpc.js +377 -0
- package/packages/web/src/platform/pluginSigning.js +168 -0
- package/packages/web/src/platform/pluginStore.js +67 -0
- package/packages/web/src/platform/settings.js +101 -0
- package/packages/web/src/platform/spatialNav.js +286 -0
- package/packages/web/src/platform/viewport.js +123 -0
- package/packages/web/src/platform/voice.js +133 -0
- package/packages/web/src/platform/voiceSearch.js +155 -0
- package/packages/web/src/platform/whenclause.js +162 -0
- package/packages/web/src/platform/workbench.js +156 -0
- package/packages/web/src/runtime.js +73 -0
- package/packages/web/src/styles.css +1382 -0
- package/packages/web/src/ui/components/activityBar.js +35 -0
- package/packages/web/src/ui/components/activityPanel.js +132 -0
- package/packages/web/src/ui/components/commandPalette.js +154 -0
- package/packages/web/src/ui/components/editorArea.js +75 -0
- package/packages/web/src/ui/components/launcher.js +392 -0
- package/packages/web/src/ui/components/openers/index.js +212 -0
- package/packages/web/src/ui/components/openers/markdown.js +222 -0
- package/packages/web/src/ui/components/overlays.js +255 -0
- package/packages/web/src/ui/components/phoneChrome.js +188 -0
- package/packages/web/src/ui/components/pluginReview.js +151 -0
- package/packages/web/src/ui/components/pluginsView.js +120 -0
- package/packages/web/src/ui/components/settingsView.js +258 -0
- package/packages/web/src/ui/components/social.js +290 -0
- package/packages/web/src/ui/components/statusBar.js +198 -0
- package/packages/web/src/ui/components/views/grid.js +115 -0
- package/packages/web/src/ui/components/views/index.js +155 -0
- package/packages/web/src/ui/components/views/list.js +50 -0
- package/packages/web/src/ui/components/views/parts.js +58 -0
- package/packages/web/src/ui/compositions/workbench.js +125 -0
- package/packages/web/src/ui/format.js +33 -0
- package/packages/web/src/ui/icon.js +81 -0
- package/packages/web/src/ui/media.js +114 -0
- package/packages/web/src/ui/sanitize.js +86 -0
- package/packages/web/src/workbench.js +205 -0
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
// TaskRegistry — work that is happening right now and takes long enough that someone
|
|
2
|
+
// should be able to watch it.
|
|
3
|
+
//
|
|
4
|
+
// Deliberately IN-MEMORY and per-process. A task describes work in flight, and work in
|
|
5
|
+
// flight does not survive the process doing it: if the server restarts mid-reindex,
|
|
6
|
+
// that reindex is not "still running", it is gone. Persisting tasks would mean
|
|
7
|
+
// resurrecting rows that claim to be running while nothing is, which is worse than
|
|
8
|
+
// forgetting them. What must outlive the process is the *consequence* of a failure —
|
|
9
|
+
// and that is an Issue (issues.js), which is durable precisely because it is a standing
|
|
10
|
+
// fact rather than an in-flight activity.
|
|
11
|
+
//
|
|
12
|
+
// Progress comes in two shapes, because honest progress does:
|
|
13
|
+
// • determinate — done / total, with a unit ("142 / 3,100 items")
|
|
14
|
+
// • indeterminate — total is null; we know it's working, not how far along
|
|
15
|
+
// A caller that doesn't know the total must NOT invent one. A progress bar that lies is
|
|
16
|
+
// worse than a spinner that doesn't.
|
|
17
|
+
|
|
18
|
+
import { newId } from './util.js';
|
|
19
|
+
|
|
20
|
+
/** How long a finished task stays listable, so a client polling every second sees it end. */
|
|
21
|
+
const DEFAULT_RETAIN_MS = 60_000;
|
|
22
|
+
/** Hard cap on retained finished tasks, so a chatty producer can't grow this forever. */
|
|
23
|
+
const MAX_FINISHED = 100;
|
|
24
|
+
|
|
25
|
+
export class TaskRegistry {
|
|
26
|
+
/**
|
|
27
|
+
* @param {{retainMs?: number, now?: () => number}} [opts]
|
|
28
|
+
*/
|
|
29
|
+
constructor({ retainMs = DEFAULT_RETAIN_MS, now = () => Date.now() } = {}) {
|
|
30
|
+
this.retainMs = retainMs;
|
|
31
|
+
this.now = now;
|
|
32
|
+
this.tasks = new Map(); // id -> task record
|
|
33
|
+
this._controllers = new Map(); // id -> AbortController
|
|
34
|
+
this._inFlight = new Set(); // promises for tasks still running — see pending()
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Begin a task. Prefer `run()` — it can't leak a task that never finishes.
|
|
39
|
+
*
|
|
40
|
+
* @param {object} spec
|
|
41
|
+
* @param {string} spec.title what the user sees ("Rebuilding the search index")
|
|
42
|
+
* @param {string} [spec.kind] coarse grouping ('index', 'transfer', …)
|
|
43
|
+
* @param {string} [spec.detail] a line under the title, updated as it goes
|
|
44
|
+
* @param {number|null} [spec.total] omit for indeterminate — do not guess
|
|
45
|
+
* @param {string} [spec.unit] 'items', 'files', 'bytes'
|
|
46
|
+
* @param {string} [spec.collectionId] scopes visibility to readers of that collection
|
|
47
|
+
* @param {boolean} [spec.cancellable]
|
|
48
|
+
*/
|
|
49
|
+
start(spec = {}) {
|
|
50
|
+
const id = spec.id || newId('tsk');
|
|
51
|
+
const controller = spec.cancellable ? new AbortController() : null;
|
|
52
|
+
if (controller) this._controllers.set(id, controller);
|
|
53
|
+
const task = {
|
|
54
|
+
id,
|
|
55
|
+
kind: spec.kind || 'general',
|
|
56
|
+
title: spec.title || 'Working',
|
|
57
|
+
detail: spec.detail || null,
|
|
58
|
+
status: 'running',
|
|
59
|
+
done: 0,
|
|
60
|
+
total: spec.total ?? null,
|
|
61
|
+
unit: spec.unit || null,
|
|
62
|
+
collectionId: spec.collectionId ?? null,
|
|
63
|
+
cancellable: !!controller,
|
|
64
|
+
startedAt: this.now(),
|
|
65
|
+
endedAt: null,
|
|
66
|
+
error: null,
|
|
67
|
+
};
|
|
68
|
+
this.tasks.set(id, task);
|
|
69
|
+
this.#prune();
|
|
70
|
+
return this.#handle(task, controller);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Run `fn` as a task, finishing it however `fn` ends. This is the shape callers
|
|
75
|
+
* should reach for: a `start()` whose owner throws before `succeed()`/`fail()` leaves
|
|
76
|
+
* a task spinning in the UI forever, describing work that stopped long ago.
|
|
77
|
+
*
|
|
78
|
+
* @param {object} spec as `start`
|
|
79
|
+
* @param {(handle: object) => Promise<any>} fn
|
|
80
|
+
*/
|
|
81
|
+
async run(spec, fn) {
|
|
82
|
+
return this.begin(spec, fn).done;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* As `run()`, but hands back the task record immediately alongside the promise.
|
|
87
|
+
*
|
|
88
|
+
* A route that starts long work has to answer with something the client can watch,
|
|
89
|
+
* and it cannot wait for the work to finish to find out what that something is. The
|
|
90
|
+
* record exists the moment the task starts; this is how a caller gets at it without
|
|
91
|
+
* awaiting the whole job.
|
|
92
|
+
*
|
|
93
|
+
* @returns {{task: object, done: Promise<any>}}
|
|
94
|
+
*/
|
|
95
|
+
begin(spec, fn) {
|
|
96
|
+
const handle = this.start(spec);
|
|
97
|
+
const promise = (async () => {
|
|
98
|
+
try {
|
|
99
|
+
const result = await fn(handle);
|
|
100
|
+
handle.succeed(result);
|
|
101
|
+
return result;
|
|
102
|
+
} catch (err) {
|
|
103
|
+
handle.fail(err);
|
|
104
|
+
throw err;
|
|
105
|
+
}
|
|
106
|
+
})();
|
|
107
|
+
// Track it so `pending()` can hand the whole set to a runtime that needs to be told
|
|
108
|
+
// the work exists — see below.
|
|
109
|
+
this._inFlight.add(promise);
|
|
110
|
+
promise.catch(() => {}).finally(() => this._inFlight.delete(promise));
|
|
111
|
+
return { task: this.get(handle.id), done: promise };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Every task still running, as one promise — or null when nothing is.
|
|
116
|
+
*
|
|
117
|
+
* A route starts a task and returns immediately, because the work takes minutes and
|
|
118
|
+
* holding the request open for it would just time out. On a long-lived process that
|
|
119
|
+
* is enough: the promise keeps running because the process does.
|
|
120
|
+
*
|
|
121
|
+
* On Cloudflare Workers it is not. The isolate may be discarded as soon as the
|
|
122
|
+
* response resolves, and a promise nobody declared is simply cancelled part-way — a
|
|
123
|
+
* scan that silently did a third of the bucket. `ctx.waitUntil` is how a Worker says
|
|
124
|
+
* "the response is done but I am not", and it needs something to wait on. This is it.
|
|
125
|
+
*/
|
|
126
|
+
pending() {
|
|
127
|
+
if (!this._inFlight.size) return null;
|
|
128
|
+
return Promise.allSettled([...this._inFlight]);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
#handle(task, controller) {
|
|
132
|
+
const set = (patch) => {
|
|
133
|
+
// A finished task is final: a late progress callback from work that has already
|
|
134
|
+
// been cancelled must not resurrect it as "running".
|
|
135
|
+
if (task.status !== 'running') return;
|
|
136
|
+
Object.assign(task, patch);
|
|
137
|
+
};
|
|
138
|
+
return {
|
|
139
|
+
id: task.id,
|
|
140
|
+
get signal() { return controller?.signal; },
|
|
141
|
+
get cancelled() { return !!controller?.signal.aborted; },
|
|
142
|
+
/** @param {{done?: number, total?: number|null, unit?: string, detail?: string}} p */
|
|
143
|
+
progress: (p = {}) => set({
|
|
144
|
+
done: p.done ?? task.done,
|
|
145
|
+
total: p.total === undefined ? task.total : p.total,
|
|
146
|
+
unit: p.unit ?? task.unit,
|
|
147
|
+
detail: p.detail ?? task.detail,
|
|
148
|
+
}),
|
|
149
|
+
succeed: (detail) => {
|
|
150
|
+
if (task.status !== 'running') return; // already cancelled/failed
|
|
151
|
+
Object.assign(task, {
|
|
152
|
+
status: 'done', endedAt: this.now(),
|
|
153
|
+
detail: typeof detail === 'string' ? detail : task.detail,
|
|
154
|
+
done: task.total ?? task.done,
|
|
155
|
+
});
|
|
156
|
+
this._controllers.delete(task.id);
|
|
157
|
+
},
|
|
158
|
+
fail: (err) => {
|
|
159
|
+
if (task.status !== 'running') return;
|
|
160
|
+
Object.assign(task, {
|
|
161
|
+
status: task.cancelled ? 'cancelled' : 'failed',
|
|
162
|
+
endedAt: this.now(),
|
|
163
|
+
error: err?.message || String(err || 'failed'),
|
|
164
|
+
});
|
|
165
|
+
this._controllers.delete(task.id);
|
|
166
|
+
},
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Ask a task to stop. Cancellation is COOPERATIVE — this aborts the signal and marks
|
|
172
|
+
* the task, but the work itself decides when to notice. Marking it here rather than
|
|
173
|
+
* waiting for the worker means the UI reflects the click immediately, which is the
|
|
174
|
+
* behaviour a user expects from a Cancel button.
|
|
175
|
+
*/
|
|
176
|
+
cancel(id) {
|
|
177
|
+
const task = this.tasks.get(id);
|
|
178
|
+
if (!task || task.status !== 'running') return false;
|
|
179
|
+
this._controllers.get(id)?.abort();
|
|
180
|
+
Object.assign(task, { status: 'cancelled', endedAt: this.now() });
|
|
181
|
+
this._controllers.delete(id);
|
|
182
|
+
return true;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
get(id) {
|
|
186
|
+
return this.tasks.get(id) ? { ...this.tasks.get(id) } : null;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Running tasks first (they're what the user is waiting on), then recently finished
|
|
191
|
+
* newest-first — so a client that polls at 1 Hz still sees a task that started and
|
|
192
|
+
* ended between two polls.
|
|
193
|
+
*
|
|
194
|
+
* @param {{collectionIds?: string[]}} [opts] restrict to tasks the caller may see;
|
|
195
|
+
* a task with no collectionId is drive-wide and only listed when `collectionIds`
|
|
196
|
+
* is undefined (collections disabled) or the caller is explicitly allowed it.
|
|
197
|
+
*/
|
|
198
|
+
list({ collectionIds, includeGlobal = true } = {}) {
|
|
199
|
+
this.#prune();
|
|
200
|
+
const visible = [...this.tasks.values()].filter((t) => {
|
|
201
|
+
if (t.collectionId == null) return includeGlobal;
|
|
202
|
+
return !collectionIds || collectionIds.includes(t.collectionId);
|
|
203
|
+
});
|
|
204
|
+
const running = visible.filter((t) => t.status === 'running').sort((a, b) => a.startedAt - b.startedAt);
|
|
205
|
+
const finished = visible.filter((t) => t.status !== 'running').sort((a, b) => b.endedAt - a.endedAt);
|
|
206
|
+
return [...running, ...finished].map((t) => ({ ...t }));
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** Drop a finished task from the list early (the user dismissed it). */
|
|
210
|
+
dismiss(id) {
|
|
211
|
+
const task = this.tasks.get(id);
|
|
212
|
+
if (task && task.status !== 'running') this.tasks.delete(id);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
#prune() {
|
|
216
|
+
const cutoff = this.now() - this.retainMs;
|
|
217
|
+
const finished = [];
|
|
218
|
+
for (const task of this.tasks.values()) {
|
|
219
|
+
if (task.status === 'running') continue;
|
|
220
|
+
if (task.endedAt < cutoff) this.tasks.delete(task.id);
|
|
221
|
+
else finished.push(task);
|
|
222
|
+
}
|
|
223
|
+
if (finished.length > MAX_FINISHED) {
|
|
224
|
+
finished.sort((a, b) => a.endedAt - b.endedAt);
|
|
225
|
+
for (const task of finished.slice(0, finished.length - MAX_FINISHED)) this.tasks.delete(task.id);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
@@ -0,0 +1,386 @@
|
|
|
1
|
+
// UploadManager — resumable, large-file uploads with a strategy chosen from the
|
|
2
|
+
// backend's capabilities:
|
|
3
|
+
// - 'single' : one presigned PUT (small files on a presign-capable backend).
|
|
4
|
+
// - 'presign' : multipart with a presigned URL per part — the client uploads
|
|
5
|
+
// parts straight to S3, we never see the bytes. The client
|
|
6
|
+
// reports each part's ETag back so we can complete.
|
|
7
|
+
// - 'direct' : multipart where each part is PUT to *our* server, which streams
|
|
8
|
+
// it to storage (filesystem/NAS/memory that can't presign).
|
|
9
|
+
//
|
|
10
|
+
// Sessions are held in a pluggable store (in-memory by default) so an upload can
|
|
11
|
+
// resume after a dropped connection: the client re-lists parts, re-requests any
|
|
12
|
+
// missing signed URL, and continues. `create` never allocates a node; the node
|
|
13
|
+
// appears only on `complete`, so a half-finished upload leaves no ghost files.
|
|
14
|
+
|
|
15
|
+
import { TroveError, ErrorCode } from './errors.js';
|
|
16
|
+
import { newId, isValidItemName } from './util.js';
|
|
17
|
+
|
|
18
|
+
export const DEFAULT_PART_SIZE = 8 * 1024 * 1024; // 8 MiB
|
|
19
|
+
const MIN_MULTIPART_PART = 5 * 1024 * 1024; // S3 floor (except final part)
|
|
20
|
+
const SINGLE_PUT_LIMIT = 5 * 1024 * 1024; // below this, one PUT beats multipart
|
|
21
|
+
const MAX_PARTS = 10_000; // S3 multipart ceiling
|
|
22
|
+
const SESSION_TTL_MS = 24 * 60 * 60 * 1000;
|
|
23
|
+
|
|
24
|
+
class MemorySessionStore {
|
|
25
|
+
constructor() {
|
|
26
|
+
this.map = new Map();
|
|
27
|
+
}
|
|
28
|
+
async get(id) {
|
|
29
|
+
return this.map.get(id) || null;
|
|
30
|
+
}
|
|
31
|
+
async put(session) {
|
|
32
|
+
this.map.set(session.id, session);
|
|
33
|
+
}
|
|
34
|
+
async delete(id) {
|
|
35
|
+
this.map.delete(id);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Which sessions have expired — WITHOUT deleting them.
|
|
39
|
+
*
|
|
40
|
+
* Deleting the record here is what leaked: the session holds the S3 `uploadId`, and
|
|
41
|
+
* that is the only handle that can abort the multipart. Drop it and the parts stay in
|
|
42
|
+
* the bucket, billed, with nothing left that could ever reclaim them (the collection
|
|
43
|
+
* scanner deliberately refuses to touch `obj_*` keys). Expiry has to go through
|
|
44
|
+
* UploadManager.sweepExpired, which aborts first and deletes after.
|
|
45
|
+
*/
|
|
46
|
+
async expired(now) {
|
|
47
|
+
return [...this.map.entries()].filter(([, s]) => now - s.createdAt > SESSION_TTL_MS).map(([id]) => id);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export class UploadManager {
|
|
52
|
+
/**
|
|
53
|
+
* @param {object} deps
|
|
54
|
+
* @param {import('./storage/interface.js').StorageBackend} deps.storage
|
|
55
|
+
* @param {object} [deps.sessions] session store (defaults in-memory)
|
|
56
|
+
* @param {number} [deps.partSize]
|
|
57
|
+
*/
|
|
58
|
+
constructor({ storage, storageFor, sessions, partSize = DEFAULT_PART_SIZE, maxBytes = null }) {
|
|
59
|
+
// Either a single backend, or a resolver keyed by collectionId (collections).
|
|
60
|
+
this.storageFor = storageFor ?? (async () => storage);
|
|
61
|
+
this.sessions = sessions ?? new MemorySessionStore();
|
|
62
|
+
this.partSize = partSize;
|
|
63
|
+
this.maxBytes = maxBytes || null; // per-file quota (null = unbounded)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// The self-describing limits the client gets back in every upload descriptor.
|
|
67
|
+
#limits() {
|
|
68
|
+
return {
|
|
69
|
+
maxBytes: this.maxBytes, // per-file quota (null = unbounded)
|
|
70
|
+
partSize: this.partSize,
|
|
71
|
+
minPartSize: MIN_MULTIPART_PART, // multipart floor, except the final part
|
|
72
|
+
singlePutLimit: SINGLE_PUT_LIMIT, // at/under this we use one PUT, not multipart
|
|
73
|
+
maxParts: MAX_PARTS,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
#storage(collectionId) {
|
|
78
|
+
return this.storageFor(collectionId);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Begin an upload. Returns a plan the client follows.
|
|
83
|
+
* @param {{collectionId?:string, name:string, size:number, contentType?:string}} req
|
|
84
|
+
*/
|
|
85
|
+
async create(req) {
|
|
86
|
+
if (!isValidItemName(req.name)) throw TroveError.invalid(`Invalid file name "${req.name}"`);
|
|
87
|
+
if (!(req.size >= 0)) throw TroveError.invalid('size must be a non-negative number');
|
|
88
|
+
if (this.maxBytes && req.size > this.maxBytes) {
|
|
89
|
+
// Deterministic per-file limit — retrying can't help, so it's non-retryable
|
|
90
|
+
// (capacity/rate quotas stay retryable via the default).
|
|
91
|
+
// TOO_LARGE (413), not QUOTA: the store has plenty of room, this file is simply
|
|
92
|
+
// bigger than this deployment permits. Reporting it as a capacity problem would
|
|
93
|
+
// send the user looking for space to free that would not help.
|
|
94
|
+
throw TroveError.tooLarge(`File exceeds the maximum upload size of ${this.maxBytes} bytes`, { details: { maxBytes: this.maxBytes, size: req.size } });
|
|
95
|
+
}
|
|
96
|
+
const collectionId = req.collectionId || 'default';
|
|
97
|
+
const storage = await this.#storage(collectionId);
|
|
98
|
+
const caps = storage.capabilities;
|
|
99
|
+
const storageKey = newId('obj');
|
|
100
|
+
const contentType = req.contentType || 'application/octet-stream';
|
|
101
|
+
|
|
102
|
+
const session = {
|
|
103
|
+
id: newId('up'),
|
|
104
|
+
storageKey,
|
|
105
|
+
collectionId,
|
|
106
|
+
name: req.name,
|
|
107
|
+
// Whether the caller asked to replace an existing item of this name. Carried on
|
|
108
|
+
// the session because the decision is made when the upload STARTS but has to be
|
|
109
|
+
// honoured when it COMPLETES, possibly much later.
|
|
110
|
+
overwrite: !!req.overwrite,
|
|
111
|
+
size: req.size,
|
|
112
|
+
contentType,
|
|
113
|
+
createdAt: Date.now(),
|
|
114
|
+
strategy: null,
|
|
115
|
+
partSize: this.partSize,
|
|
116
|
+
uploadId: null,
|
|
117
|
+
parts: {}, // partNumber -> { etag }
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
const limits = this.#limits();
|
|
121
|
+
|
|
122
|
+
// Small file + presign → single PUT straight to storage (never through us).
|
|
123
|
+
if (req.size <= SINGLE_PUT_LIMIT && caps.presignUpload) {
|
|
124
|
+
session.strategy = 'single';
|
|
125
|
+
await this.sessions.put(session);
|
|
126
|
+
const url = await storage.presignPut(storageKey, { contentType });
|
|
127
|
+
return { ...planSummary(session), strategy: 'single', multipart: false, presigned: true, url, limits };
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// Multipart (presigned parts straight to storage, or streamed through us).
|
|
131
|
+
if (caps.multipart) {
|
|
132
|
+
session.strategy = caps.presignUpload ? 'presign' : 'direct';
|
|
133
|
+
const partCount = Math.max(1, Math.ceil(req.size / this.partSize));
|
|
134
|
+
// `#limits()` advertises maxParts in the very same response, and nothing enforced
|
|
135
|
+
// it: a 150 GiB file planned 19,200 parts against a ceiling of 10,000, which S3
|
|
136
|
+
// rejects at part 10,001 — after the client has transferred 80 GiB. On a presign
|
|
137
|
+
// backend it also signed every part up front, so the plan itself came back as a
|
|
138
|
+
// 1.9 MB JSON document. Refuse at negotiation, where it costs nothing.
|
|
139
|
+
if (partCount > MAX_PARTS) {
|
|
140
|
+
throw TroveError.tooLarge(
|
|
141
|
+
`This file needs ${partCount.toLocaleString()} parts of ${this.partSize} bytes, over the ${MAX_PARTS.toLocaleString()}-part limit`,
|
|
142
|
+
{ details: { maxParts: MAX_PARTS, partCount, partSize: this.partSize, size: req.size } },
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
// Only now open the multipart. Refusing AFTER creating it left an upload open in
|
|
146
|
+
// the bucket with no session record, so nothing could ever abort it.
|
|
147
|
+
session.uploadId = await storage.createMultipart(storageKey, { contentType });
|
|
148
|
+
session.partCount = partCount;
|
|
149
|
+
await this.sessions.put(session);
|
|
150
|
+
const parts = [];
|
|
151
|
+
if (session.strategy === 'presign') {
|
|
152
|
+
for (let n = 1; n <= partCount; n++) {
|
|
153
|
+
parts.push({ partNumber: n, url: await storage.presignPart(storageKey, session.uploadId, n) });
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return { ...planSummary(session), strategy: session.strategy, multipart: true, presigned: session.strategy === 'presign', partCount, parts, limits };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// Fallback: whole-object PUT streamed through us (tiny/simple backends).
|
|
160
|
+
session.strategy = 'direct-single';
|
|
161
|
+
await this.sessions.put(session);
|
|
162
|
+
return { ...planSummary(session), strategy: 'direct-single', multipart: false, presigned: false, limits };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** Re-issue a signed URL for one part (resume after expiry). */
|
|
166
|
+
async signPart(uploadId, partNumber) {
|
|
167
|
+
const s = await this.#session(uploadId);
|
|
168
|
+
if (s.strategy !== 'presign') throw TroveError.invalid('signPart only applies to presigned uploads');
|
|
169
|
+
const storage = await this.#storage(s.collectionId);
|
|
170
|
+
return storage.presignPart(s.storageKey, s.uploadId, partNumber);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** Client reports a completed presigned part (with its ETag from S3). */
|
|
174
|
+
async reportPart(uploadId, partNumber, etag) {
|
|
175
|
+
const s = await this.#session(uploadId);
|
|
176
|
+
if (!etag) throw TroveError.invalid('Part ETag required');
|
|
177
|
+
// Same bounds as uploadPart. Unbounded, a client could report hundreds of thousands
|
|
178
|
+
// of out-of-plan parts, all retained for the session's 24h TTL — and `status` hands
|
|
179
|
+
// that list back as the parts it may SKIP.
|
|
180
|
+
if (!Number.isInteger(partNumber) || partNumber < 1 || (s.partCount && partNumber > s.partCount)) {
|
|
181
|
+
throw TroveError.invalid(`Part ${partNumber} is outside this upload's ${s.partCount} part(s)`);
|
|
182
|
+
}
|
|
183
|
+
s.parts[partNumber] = { etag };
|
|
184
|
+
await this.sessions.put(s);
|
|
185
|
+
return { ok: true };
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** Direct backends: stream one part through us to storage. */
|
|
189
|
+
async uploadPart(uploadId, partNumber, body, opts = {}) {
|
|
190
|
+
const s = await this.#session(uploadId);
|
|
191
|
+
const storage = await this.#storage(s.collectionId);
|
|
192
|
+
if (s.strategy === 'direct-single') {
|
|
193
|
+
// The proxied body is raw bytes streamed straight to storage — the JSON body cap
|
|
194
|
+
// deliberately doesn't apply — so this is the only place a client can be stopped
|
|
195
|
+
// from writing an unbounded object through us.
|
|
196
|
+
const capped = capStream(body, this.maxBytes);
|
|
197
|
+
const info = await storage.put(s.storageKey, capped, { contentType: s.contentType, ...opts });
|
|
198
|
+
s.parts[1] = { etag: info.etag || 'single' };
|
|
199
|
+
await this.sessions.put(s);
|
|
200
|
+
return { partNumber: 1, etag: s.parts[1].etag };
|
|
201
|
+
}
|
|
202
|
+
if (s.strategy !== 'direct') throw TroveError.invalid('uploadPart only applies to direct uploads');
|
|
203
|
+
// A part outside the plan is stored, billed, and never merged — `complete` only
|
|
204
|
+
// walks 1..partCount. Silently accepting one meant a client could believe it had
|
|
205
|
+
// uploaded bytes that would never become part of the file.
|
|
206
|
+
// `NaN < 1` and `NaN > partCount` are BOTH false, so a non-numeric part number
|
|
207
|
+
// walked through and its bytes were written under the key "NaN" — billed, and
|
|
208
|
+
// unmergeable, since `complete` only ever walks 1..partCount.
|
|
209
|
+
if (!Number.isInteger(partNumber) || partNumber < 1 || (s.partCount && partNumber > s.partCount)) {
|
|
210
|
+
throw TroveError.invalid(`Part ${partNumber} is outside this upload's ${s.partCount} part(s)`);
|
|
211
|
+
}
|
|
212
|
+
// A part is exactly `partSize` bytes, except the last, which is smaller.
|
|
213
|
+
const res = await storage.putPart(s.storageKey, s.uploadId, partNumber, capStream(body, s.partSize), opts);
|
|
214
|
+
s.parts[partNumber] = { etag: res.etag };
|
|
215
|
+
await this.sessions.put(s);
|
|
216
|
+
return res;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** Which parts are still outstanding (resume support). */
|
|
220
|
+
async status(uploadId) {
|
|
221
|
+
const s = await this.#session(uploadId);
|
|
222
|
+
const received = Object.keys(s.parts).map(Number).sort((a, b) => a - b);
|
|
223
|
+
return {
|
|
224
|
+
uploadId,
|
|
225
|
+
collectionId: s.collectionId,
|
|
226
|
+
strategy: s.strategy,
|
|
227
|
+
partCount: s.partCount ?? 1,
|
|
228
|
+
partSize: s.partSize,
|
|
229
|
+
received,
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Finalise. Verifies all parts are present, completes multipart, and returns
|
|
235
|
+
* the object descriptor so the VFS can create the node.
|
|
236
|
+
* @returns {Promise<{storageKey, size, contentType, etag, collectionId, name}>}
|
|
237
|
+
*/
|
|
238
|
+
async complete(uploadId, reportedParts) {
|
|
239
|
+
const s = await this.#session(uploadId);
|
|
240
|
+
let etag;
|
|
241
|
+
if (s.strategy === 'single' || s.strategy === 'direct-single') {
|
|
242
|
+
// Object already fully written by the client (single presigned PUT) or by us.
|
|
243
|
+
etag = s.parts[1]?.etag;
|
|
244
|
+
} else {
|
|
245
|
+
// Merge any client-reported ETags (presign) with what we recorded.
|
|
246
|
+
const parts = [];
|
|
247
|
+
const count = s.partCount ?? Object.keys(s.parts).length;
|
|
248
|
+
for (let n = 1; n <= count; n++) {
|
|
249
|
+
const fromClient = reportedParts?.find((p) => p.partNumber === n);
|
|
250
|
+
const etagN = fromClient?.etag || s.parts[n]?.etag;
|
|
251
|
+
if (!etagN) {
|
|
252
|
+
throw TroveError.invalid(`Missing part ${n} of ${count}`, { details: { partNumber: n } });
|
|
253
|
+
}
|
|
254
|
+
parts.push({ partNumber: n, etag: etagN });
|
|
255
|
+
}
|
|
256
|
+
const storage = await this.#storage(s.collectionId);
|
|
257
|
+
const res = await storage.completeMultipart(s.storageKey, s.uploadId, parts);
|
|
258
|
+
etag = res.etag;
|
|
259
|
+
}
|
|
260
|
+
// What actually landed, not what the client said it would send.
|
|
261
|
+
//
|
|
262
|
+
// `size` up to here has been the client's DECLARED size — the value the per-file
|
|
263
|
+
// limit was checked against at create(). Nothing re-checked the bytes, so declaring
|
|
264
|
+
// `size: 1` and then PUTting gigabytes walked straight past the limit and recorded a
|
|
265
|
+
// size that was simply false (which the quota meter, the UI and every later
|
|
266
|
+
// range read then believed). Ask the store.
|
|
267
|
+
const storage = await this.#storage(s.collectionId);
|
|
268
|
+
let size = s.size;
|
|
269
|
+
try {
|
|
270
|
+
const info = await storage.head(s.storageKey);
|
|
271
|
+
if (Number.isFinite(info?.size)) size = info.size;
|
|
272
|
+
} catch (err) {
|
|
273
|
+
// NOT_FOUND is the one error that means something specific: no bytes were ever
|
|
274
|
+
// stored. Swallowing it committed an item for an object that does not exist — the
|
|
275
|
+
// drive listing a file whose size is a number the client invented, whose download
|
|
276
|
+
// redirects to a 404, and which every indexer then raises a standing issue about.
|
|
277
|
+
// Reachable whenever a presigned PUT fails but `complete` still runs.
|
|
278
|
+
if (err?.code === ErrorCode.NOT_FOUND) {
|
|
279
|
+
await this.sessions.delete(uploadId);
|
|
280
|
+
throw TroveError.invalid('The upload was never written — no object exists for this session');
|
|
281
|
+
}
|
|
282
|
+
// Any other failure (a backend that can't answer a HEAD) leaves the declared size
|
|
283
|
+
// in place; better an approximate record than refusing an upload that succeeded.
|
|
284
|
+
}
|
|
285
|
+
if (this.maxBytes && size > this.maxBytes) {
|
|
286
|
+
// It is already in the store, so let it go rather than leaving an orphan that
|
|
287
|
+
// counts against the quota and belongs to nothing.
|
|
288
|
+
await storage.delete(s.storageKey).catch(() => {});
|
|
289
|
+
await this.sessions.delete(uploadId);
|
|
290
|
+
throw TroveError.tooLarge(
|
|
291
|
+
`File exceeds the maximum upload size of ${this.maxBytes} bytes`,
|
|
292
|
+
{ details: { maxBytes: this.maxBytes, size } },
|
|
293
|
+
);
|
|
294
|
+
}
|
|
295
|
+
await this.sessions.delete(uploadId);
|
|
296
|
+
return {
|
|
297
|
+
storageKey: s.storageKey,
|
|
298
|
+
size,
|
|
299
|
+
contentType: s.contentType,
|
|
300
|
+
etag,
|
|
301
|
+
collectionId: s.collectionId,
|
|
302
|
+
name: s.name,
|
|
303
|
+
overwrite: !!s.overwrite,
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* Reclaim uploads the client started and never finished.
|
|
309
|
+
*
|
|
310
|
+
* A dropped connection, a closed tab, a crash — all leave a live multipart upload
|
|
311
|
+
* whose parts are stored and billed until something aborts them. That something is
|
|
312
|
+
* this: it goes through `abort`, which tells the backend to let the parts go before
|
|
313
|
+
* the session record (and with it the uploadId) is discarded.
|
|
314
|
+
*
|
|
315
|
+
* @returns {Promise<{aborted: number, failed: number}>}
|
|
316
|
+
*/
|
|
317
|
+
async sweepExpired(now = Date.now()) {
|
|
318
|
+
const ids = (await this.sessions.expired?.(now)) || [];
|
|
319
|
+
let aborted = 0;
|
|
320
|
+
let failed = 0;
|
|
321
|
+
for (const id of ids) {
|
|
322
|
+
try {
|
|
323
|
+
await this.abort(id);
|
|
324
|
+
aborted++;
|
|
325
|
+
} catch (err) {
|
|
326
|
+
// Leave the session in place so the next sweep tries again — dropping it is
|
|
327
|
+
// exactly how the bytes became unreclaimable in the first place.
|
|
328
|
+
failed++;
|
|
329
|
+
console.error(`[trove] could not reclaim abandoned upload ${id}:`, err?.message || err);
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
return { aborted, failed };
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
async abort(uploadId) {
|
|
336
|
+
const s = await this.sessions.get(uploadId);
|
|
337
|
+
if (!s) return;
|
|
338
|
+
const storage = await this.#storage(s.collectionId);
|
|
339
|
+
// The session record holds the multipart uploadId, and that is the ONLY handle that
|
|
340
|
+
// can ever reclaim the staged parts. Swallowing the backend's failure and deleting
|
|
341
|
+
// the record anyway is precisely how the bytes became unreclaimable — and it made
|
|
342
|
+
// sweepExpired's failure branch, whose comment says exactly that, unreachable.
|
|
343
|
+
if (s.uploadId) await storage.abortMultipart(s.storageKey, s.uploadId);
|
|
344
|
+
else await storage.delete(s.storageKey).catch(() => {});
|
|
345
|
+
await this.sessions.delete(uploadId);
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
async #session(uploadId) {
|
|
349
|
+
const s = await this.sessions.get(uploadId);
|
|
350
|
+
if (!s) throw TroveError.notFound('Upload session');
|
|
351
|
+
return s;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
function planSummary(s) {
|
|
356
|
+
return { uploadId: s.id, storageKey: s.storageKey, partSize: s.partSize, size: s.size, name: s.name, contentType: s.contentType };
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* Wrap a streamed body so it cannot exceed `max` bytes.
|
|
361
|
+
*
|
|
362
|
+
* Uploads bypass the JSON body cap by design — their bytes stream to storage rather
|
|
363
|
+
* than being buffered — which left the proxied part route with no ceiling at all: the
|
|
364
|
+
* declared size was checked at create() and nothing checked what actually arrived.
|
|
365
|
+
* Non-stream bodies (a Uint8Array from a test, a Blob) pass through untouched; they are
|
|
366
|
+
* already resident, so capping them here would not save the memory.
|
|
367
|
+
*/
|
|
368
|
+
function capStream(body, max) {
|
|
369
|
+
if (!max || !body || typeof body.getReader !== 'function') return body;
|
|
370
|
+
const reader = body.getReader();
|
|
371
|
+
let total = 0;
|
|
372
|
+
return new ReadableStream({
|
|
373
|
+
async pull(controller) {
|
|
374
|
+
const { done, value } = await reader.read();
|
|
375
|
+
if (done) { controller.close(); return; }
|
|
376
|
+
total += value.byteLength;
|
|
377
|
+
if (total > max) {
|
|
378
|
+
await reader.cancel().catch(() => {});
|
|
379
|
+
controller.error(TroveError.tooLarge(`Upload body exceeds ${max} bytes`, { details: { maxBytes: max } }));
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
382
|
+
controller.enqueue(value);
|
|
383
|
+
},
|
|
384
|
+
cancel(reason) { reader.cancel(reason).catch(() => {}); },
|
|
385
|
+
});
|
|
386
|
+
}
|