@3sln/trove 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/README.md +1227 -0
  2. package/package.json +75 -0
  3. package/packages/core/src/collections/index.js +249 -0
  4. package/packages/core/src/errors.js +186 -0
  5. package/packages/core/src/identity/discovery.js +210 -0
  6. package/packages/core/src/identity/index.js +188 -0
  7. package/packages/core/src/identity/jwt.js +199 -0
  8. package/packages/core/src/index.js +104 -0
  9. package/packages/core/src/indexers/contribution.js +115 -0
  10. package/packages/core/src/indexers/registry.js +162 -0
  11. package/packages/core/src/indexing.js +340 -0
  12. package/packages/core/src/issues.js +150 -0
  13. package/packages/core/src/kv.js +0 -0
  14. package/packages/core/src/links.js +141 -0
  15. package/packages/core/src/metadata/cursor.js +73 -0
  16. package/packages/core/src/metadata/interface.js +244 -0
  17. package/packages/core/src/metadata/memory.js +270 -0
  18. package/packages/core/src/metadata/sqlite.js +412 -0
  19. package/packages/core/src/notifications/index.js +139 -0
  20. package/packages/core/src/notifications/webpush.js +217 -0
  21. package/packages/core/src/plugins/contributions.js +177 -0
  22. package/packages/core/src/plugins/identity.js +98 -0
  23. package/packages/core/src/plugins/index.js +225 -0
  24. package/packages/core/src/plugins/indexers.js +142 -0
  25. package/packages/core/src/plugins/installStore.js +134 -0
  26. package/packages/core/src/plugins/package.js +102 -0
  27. package/packages/core/src/plugins/packageStore.js +61 -0
  28. package/packages/core/src/plugins/runtime.js +101 -0
  29. package/packages/core/src/plugins/sql.js +52 -0
  30. package/packages/core/src/retry.js +74 -0
  31. package/packages/core/src/scan.js +302 -0
  32. package/packages/core/src/search/embeddings.js +128 -0
  33. package/packages/core/src/search/index.js +200 -0
  34. package/packages/core/src/search/keywordStore.js +107 -0
  35. package/packages/core/src/search/sqliteStores.js +455 -0
  36. package/packages/core/src/search/tagMatch.js +59 -0
  37. package/packages/core/src/search/transformer.js +195 -0
  38. package/packages/core/src/search/vectorStore.js +274 -0
  39. package/packages/core/src/search/vectorize.js +249 -0
  40. package/packages/core/src/sidecar/document.js +213 -0
  41. package/packages/core/src/sidecar/index.js +174 -0
  42. package/packages/core/src/sidecar/manager.js +239 -0
  43. package/packages/core/src/sidecar/store.js +46 -0
  44. package/packages/core/src/signedUrls.js +170 -0
  45. package/packages/core/src/sqlite-d1.js +162 -0
  46. package/packages/core/src/sqlite-driver.js +42 -0
  47. package/packages/core/src/sqlite.js +162 -0
  48. package/packages/core/src/storage/filesystem.js +283 -0
  49. package/packages/core/src/storage/interface.js +222 -0
  50. package/packages/core/src/storage/memory.js +113 -0
  51. package/packages/core/src/storage/prefixed.js +75 -0
  52. package/packages/core/src/storage/s3.js +316 -0
  53. package/packages/core/src/storage/s3sigv4.js +185 -0
  54. package/packages/core/src/tasks.js +228 -0
  55. package/packages/core/src/uploads.js +386 -0
  56. package/packages/core/src/util.js +125 -0
  57. package/packages/core/src/vfs.js +666 -0
  58. package/packages/plugin-sdk/src/browser.js +316 -0
  59. package/packages/plugin-sdk/src/index.js +32 -0
  60. package/packages/plugin-sdk/src/protocol.js +59 -0
  61. package/packages/plugin-sdk/src/rpc.js +95 -0
  62. package/packages/server/src/adapters/bun.js +78 -0
  63. package/packages/server/src/adapters/node.js +115 -0
  64. package/packages/server/src/adapters/staticAssets.js +123 -0
  65. package/packages/server/src/adapters/webDist.js +70 -0
  66. package/packages/server/src/adapters/worker-tasks.js +206 -0
  67. package/packages/server/src/adapters/worker.js +159 -0
  68. package/packages/server/src/cachePolicy.js +34 -0
  69. package/packages/server/src/engine/README.md +88 -0
  70. package/packages/server/src/engine/actions/scanCollection.js +114 -0
  71. package/packages/server/src/engine/index.js +95 -0
  72. package/packages/server/src/engine/lazy.js +25 -0
  73. package/packages/server/src/engine/providers/access.js +363 -0
  74. package/packages/server/src/engine/providers/core.js +405 -0
  75. package/packages/server/src/engine/providers/scan.js +67 -0
  76. package/packages/server/src/index.js +698 -0
  77. package/packages/server/src/manifest.js +98 -0
  78. package/packages/server/src/mcp/auth.js +40 -0
  79. package/packages/server/src/mcp/index.js +213 -0
  80. package/packages/server/src/mcp/protocol.js +181 -0
  81. package/packages/server/src/mcp/tools.js +351 -0
  82. package/packages/server/src/router.js +229 -0
  83. package/packages/server/src/routes.js +1066 -0
  84. package/packages/server/src/scope.js +43 -0
  85. package/packages/web/dist/assets/chunk-4xqbzebh.js +5 -0
  86. package/packages/web/dist/assets/chunk-4xqbzebh.js.map +9 -0
  87. package/packages/web/dist/assets/chunk-h05bxfbs.js +5 -0
  88. package/packages/web/dist/assets/chunk-h05bxfbs.js.map +10 -0
  89. package/packages/web/dist/assets/main-4cxs7prw.js +356 -0
  90. package/packages/web/dist/assets/main-4cxs7prw.js.map +103 -0
  91. package/packages/web/dist/assets/styles-kcx1x337.css +1 -0
  92. package/packages/web/dist/icon.svg +11 -0
  93. package/packages/web/dist/index.html +16 -0
  94. package/packages/web/dist/sql-wasm.wasm +0 -0
  95. package/packages/web/dist/sw.js +186 -0
  96. package/packages/web/src/bl/actions.js +410 -0
  97. package/packages/web/src/bl/activity.js +306 -0
  98. package/packages/web/src/bl/commands.js +274 -0
  99. package/packages/web/src/bl/fileType.js +49 -0
  100. package/packages/web/src/bl/index.js +70 -0
  101. package/packages/web/src/bl/links.js +54 -0
  102. package/packages/web/src/bl/offline.js +268 -0
  103. package/packages/web/src/bl/openers.js +71 -0
  104. package/packages/web/src/bl/pluginInstall.js +59 -0
  105. package/packages/web/src/bl/services.js +143 -0
  106. package/packages/web/src/bl/social.js +234 -0
  107. package/packages/web/src/bl/tagQuery.js +44 -0
  108. package/packages/web/src/main.js +10 -0
  109. package/packages/web/src/platform/api.js +529 -0
  110. package/packages/web/src/platform/commands.js +89 -0
  111. package/packages/web/src/platform/context.js +77 -0
  112. package/packages/web/src/platform/contributions.js +156 -0
  113. package/packages/web/src/platform/index.js +150 -0
  114. package/packages/web/src/platform/keybindings.js +199 -0
  115. package/packages/web/src/platform/mediaUrls.js +137 -0
  116. package/packages/web/src/platform/navigation.js +131 -0
  117. package/packages/web/src/platform/notifications.js +50 -0
  118. package/packages/web/src/platform/overlay.js +81 -0
  119. package/packages/web/src/platform/pluginClientDb.js +132 -0
  120. package/packages/web/src/platform/pluginDock.js +141 -0
  121. package/packages/web/src/platform/pluginFrames.js +194 -0
  122. package/packages/web/src/platform/pluginHost.js +648 -0
  123. package/packages/web/src/platform/pluginMedia.js +62 -0
  124. package/packages/web/src/platform/pluginModules.js +90 -0
  125. package/packages/web/src/platform/pluginNet.js +71 -0
  126. package/packages/web/src/platform/pluginPackage.js +247 -0
  127. package/packages/web/src/platform/pluginRpc.js +377 -0
  128. package/packages/web/src/platform/pluginSigning.js +168 -0
  129. package/packages/web/src/platform/pluginStore.js +67 -0
  130. package/packages/web/src/platform/settings.js +101 -0
  131. package/packages/web/src/platform/spatialNav.js +286 -0
  132. package/packages/web/src/platform/viewport.js +123 -0
  133. package/packages/web/src/platform/voice.js +133 -0
  134. package/packages/web/src/platform/voiceSearch.js +155 -0
  135. package/packages/web/src/platform/whenclause.js +162 -0
  136. package/packages/web/src/platform/workbench.js +156 -0
  137. package/packages/web/src/runtime.js +73 -0
  138. package/packages/web/src/styles.css +1382 -0
  139. package/packages/web/src/ui/components/activityBar.js +35 -0
  140. package/packages/web/src/ui/components/activityPanel.js +132 -0
  141. package/packages/web/src/ui/components/commandPalette.js +154 -0
  142. package/packages/web/src/ui/components/editorArea.js +75 -0
  143. package/packages/web/src/ui/components/launcher.js +392 -0
  144. package/packages/web/src/ui/components/openers/index.js +212 -0
  145. package/packages/web/src/ui/components/openers/markdown.js +222 -0
  146. package/packages/web/src/ui/components/overlays.js +255 -0
  147. package/packages/web/src/ui/components/phoneChrome.js +188 -0
  148. package/packages/web/src/ui/components/pluginReview.js +151 -0
  149. package/packages/web/src/ui/components/pluginsView.js +120 -0
  150. package/packages/web/src/ui/components/settingsView.js +258 -0
  151. package/packages/web/src/ui/components/social.js +290 -0
  152. package/packages/web/src/ui/components/statusBar.js +198 -0
  153. package/packages/web/src/ui/components/views/grid.js +115 -0
  154. package/packages/web/src/ui/components/views/index.js +155 -0
  155. package/packages/web/src/ui/components/views/list.js +50 -0
  156. package/packages/web/src/ui/components/views/parts.js +58 -0
  157. package/packages/web/src/ui/compositions/workbench.js +125 -0
  158. package/packages/web/src/ui/format.js +33 -0
  159. package/packages/web/src/ui/icon.js +81 -0
  160. package/packages/web/src/ui/media.js +114 -0
  161. package/packages/web/src/ui/sanitize.js +86 -0
  162. package/packages/web/src/workbench.js +205 -0
@@ -0,0 +1,228 @@
1
+ // TaskRegistry — work that is happening right now and takes long enough that someone
2
+ // should be able to watch it.
3
+ //
4
+ // Deliberately IN-MEMORY and per-process. A task describes work in flight, and work in
5
+ // flight does not survive the process doing it: if the server restarts mid-reindex,
6
+ // that reindex is not "still running", it is gone. Persisting tasks would mean
7
+ // resurrecting rows that claim to be running while nothing is, which is worse than
8
+ // forgetting them. What must outlive the process is the *consequence* of a failure —
9
+ // and that is an Issue (issues.js), which is durable precisely because it is a standing
10
+ // fact rather than an in-flight activity.
11
+ //
12
+ // Progress comes in two shapes, because honest progress does:
13
+ // • determinate — done / total, with a unit ("142 / 3,100 items")
14
+ // • indeterminate — total is null; we know it's working, not how far along
15
+ // A caller that doesn't know the total must NOT invent one. A progress bar that lies is
16
+ // worse than a spinner that doesn't.
17
+
18
+ import { newId } from './util.js';
19
+
20
+ /** How long a finished task stays listable, so a client polling every second sees it end. */
21
+ const DEFAULT_RETAIN_MS = 60_000;
22
+ /** Hard cap on retained finished tasks, so a chatty producer can't grow this forever. */
23
+ const MAX_FINISHED = 100;
24
+
25
+ export class TaskRegistry {
26
+ /**
27
+ * @param {{retainMs?: number, now?: () => number}} [opts]
28
+ */
29
+ constructor({ retainMs = DEFAULT_RETAIN_MS, now = () => Date.now() } = {}) {
30
+ this.retainMs = retainMs;
31
+ this.now = now;
32
+ this.tasks = new Map(); // id -> task record
33
+ this._controllers = new Map(); // id -> AbortController
34
+ this._inFlight = new Set(); // promises for tasks still running — see pending()
35
+ }
36
+
37
+ /**
38
+ * Begin a task. Prefer `run()` — it can't leak a task that never finishes.
39
+ *
40
+ * @param {object} spec
41
+ * @param {string} spec.title what the user sees ("Rebuilding the search index")
42
+ * @param {string} [spec.kind] coarse grouping ('index', 'transfer', …)
43
+ * @param {string} [spec.detail] a line under the title, updated as it goes
44
+ * @param {number|null} [spec.total] omit for indeterminate — do not guess
45
+ * @param {string} [spec.unit] 'items', 'files', 'bytes'
46
+ * @param {string} [spec.collectionId] scopes visibility to readers of that collection
47
+ * @param {boolean} [spec.cancellable]
48
+ */
49
+ start(spec = {}) {
50
+ const id = spec.id || newId('tsk');
51
+ const controller = spec.cancellable ? new AbortController() : null;
52
+ if (controller) this._controllers.set(id, controller);
53
+ const task = {
54
+ id,
55
+ kind: spec.kind || 'general',
56
+ title: spec.title || 'Working',
57
+ detail: spec.detail || null,
58
+ status: 'running',
59
+ done: 0,
60
+ total: spec.total ?? null,
61
+ unit: spec.unit || null,
62
+ collectionId: spec.collectionId ?? null,
63
+ cancellable: !!controller,
64
+ startedAt: this.now(),
65
+ endedAt: null,
66
+ error: null,
67
+ };
68
+ this.tasks.set(id, task);
69
+ this.#prune();
70
+ return this.#handle(task, controller);
71
+ }
72
+
73
+ /**
74
+ * Run `fn` as a task, finishing it however `fn` ends. This is the shape callers
75
+ * should reach for: a `start()` whose owner throws before `succeed()`/`fail()` leaves
76
+ * a task spinning in the UI forever, describing work that stopped long ago.
77
+ *
78
+ * @param {object} spec as `start`
79
+ * @param {(handle: object) => Promise<any>} fn
80
+ */
81
+ async run(spec, fn) {
82
+ return this.begin(spec, fn).done;
83
+ }
84
+
85
+ /**
86
+ * As `run()`, but hands back the task record immediately alongside the promise.
87
+ *
88
+ * A route that starts long work has to answer with something the client can watch,
89
+ * and it cannot wait for the work to finish to find out what that something is. The
90
+ * record exists the moment the task starts; this is how a caller gets at it without
91
+ * awaiting the whole job.
92
+ *
93
+ * @returns {{task: object, done: Promise<any>}}
94
+ */
95
+ begin(spec, fn) {
96
+ const handle = this.start(spec);
97
+ const promise = (async () => {
98
+ try {
99
+ const result = await fn(handle);
100
+ handle.succeed(result);
101
+ return result;
102
+ } catch (err) {
103
+ handle.fail(err);
104
+ throw err;
105
+ }
106
+ })();
107
+ // Track it so `pending()` can hand the whole set to a runtime that needs to be told
108
+ // the work exists — see below.
109
+ this._inFlight.add(promise);
110
+ promise.catch(() => {}).finally(() => this._inFlight.delete(promise));
111
+ return { task: this.get(handle.id), done: promise };
112
+ }
113
+
114
+ /**
115
+ * Every task still running, as one promise — or null when nothing is.
116
+ *
117
+ * A route starts a task and returns immediately, because the work takes minutes and
118
+ * holding the request open for it would just time out. On a long-lived process that
119
+ * is enough: the promise keeps running because the process does.
120
+ *
121
+ * On Cloudflare Workers it is not. The isolate may be discarded as soon as the
122
+ * response resolves, and a promise nobody declared is simply cancelled part-way — a
123
+ * scan that silently did a third of the bucket. `ctx.waitUntil` is how a Worker says
124
+ * "the response is done but I am not", and it needs something to wait on. This is it.
125
+ */
126
+ pending() {
127
+ if (!this._inFlight.size) return null;
128
+ return Promise.allSettled([...this._inFlight]);
129
+ }
130
+
131
+ #handle(task, controller) {
132
+ const set = (patch) => {
133
+ // A finished task is final: a late progress callback from work that has already
134
+ // been cancelled must not resurrect it as "running".
135
+ if (task.status !== 'running') return;
136
+ Object.assign(task, patch);
137
+ };
138
+ return {
139
+ id: task.id,
140
+ get signal() { return controller?.signal; },
141
+ get cancelled() { return !!controller?.signal.aborted; },
142
+ /** @param {{done?: number, total?: number|null, unit?: string, detail?: string}} p */
143
+ progress: (p = {}) => set({
144
+ done: p.done ?? task.done,
145
+ total: p.total === undefined ? task.total : p.total,
146
+ unit: p.unit ?? task.unit,
147
+ detail: p.detail ?? task.detail,
148
+ }),
149
+ succeed: (detail) => {
150
+ if (task.status !== 'running') return; // already cancelled/failed
151
+ Object.assign(task, {
152
+ status: 'done', endedAt: this.now(),
153
+ detail: typeof detail === 'string' ? detail : task.detail,
154
+ done: task.total ?? task.done,
155
+ });
156
+ this._controllers.delete(task.id);
157
+ },
158
+ fail: (err) => {
159
+ if (task.status !== 'running') return;
160
+ Object.assign(task, {
161
+ status: task.cancelled ? 'cancelled' : 'failed',
162
+ endedAt: this.now(),
163
+ error: err?.message || String(err || 'failed'),
164
+ });
165
+ this._controllers.delete(task.id);
166
+ },
167
+ };
168
+ }
169
+
170
+ /**
171
+ * Ask a task to stop. Cancellation is COOPERATIVE — this aborts the signal and marks
172
+ * the task, but the work itself decides when to notice. Marking it here rather than
173
+ * waiting for the worker means the UI reflects the click immediately, which is the
174
+ * behaviour a user expects from a Cancel button.
175
+ */
176
+ cancel(id) {
177
+ const task = this.tasks.get(id);
178
+ if (!task || task.status !== 'running') return false;
179
+ this._controllers.get(id)?.abort();
180
+ Object.assign(task, { status: 'cancelled', endedAt: this.now() });
181
+ this._controllers.delete(id);
182
+ return true;
183
+ }
184
+
185
+ get(id) {
186
+ return this.tasks.get(id) ? { ...this.tasks.get(id) } : null;
187
+ }
188
+
189
+ /**
190
+ * Running tasks first (they're what the user is waiting on), then recently finished
191
+ * newest-first — so a client that polls at 1 Hz still sees a task that started and
192
+ * ended between two polls.
193
+ *
194
+ * @param {{collectionIds?: string[]}} [opts] restrict to tasks the caller may see;
195
+ * a task with no collectionId is drive-wide and only listed when `collectionIds`
196
+ * is undefined (collections disabled) or the caller is explicitly allowed it.
197
+ */
198
+ list({ collectionIds, includeGlobal = true } = {}) {
199
+ this.#prune();
200
+ const visible = [...this.tasks.values()].filter((t) => {
201
+ if (t.collectionId == null) return includeGlobal;
202
+ return !collectionIds || collectionIds.includes(t.collectionId);
203
+ });
204
+ const running = visible.filter((t) => t.status === 'running').sort((a, b) => a.startedAt - b.startedAt);
205
+ const finished = visible.filter((t) => t.status !== 'running').sort((a, b) => b.endedAt - a.endedAt);
206
+ return [...running, ...finished].map((t) => ({ ...t }));
207
+ }
208
+
209
+ /** Drop a finished task from the list early (the user dismissed it). */
210
+ dismiss(id) {
211
+ const task = this.tasks.get(id);
212
+ if (task && task.status !== 'running') this.tasks.delete(id);
213
+ }
214
+
215
+ #prune() {
216
+ const cutoff = this.now() - this.retainMs;
217
+ const finished = [];
218
+ for (const task of this.tasks.values()) {
219
+ if (task.status === 'running') continue;
220
+ if (task.endedAt < cutoff) this.tasks.delete(task.id);
221
+ else finished.push(task);
222
+ }
223
+ if (finished.length > MAX_FINISHED) {
224
+ finished.sort((a, b) => a.endedAt - b.endedAt);
225
+ for (const task of finished.slice(0, finished.length - MAX_FINISHED)) this.tasks.delete(task.id);
226
+ }
227
+ }
228
+ }
@@ -0,0 +1,386 @@
1
+ // UploadManager — resumable, large-file uploads with a strategy chosen from the
2
+ // backend's capabilities:
3
+ // - 'single' : one presigned PUT (small files on a presign-capable backend).
4
+ // - 'presign' : multipart with a presigned URL per part — the client uploads
5
+ // parts straight to S3, we never see the bytes. The client
6
+ // reports each part's ETag back so we can complete.
7
+ // - 'direct' : multipart where each part is PUT to *our* server, which streams
8
+ // it to storage (filesystem/NAS/memory that can't presign).
9
+ //
10
+ // Sessions are held in a pluggable store (in-memory by default) so an upload can
11
+ // resume after a dropped connection: the client re-lists parts, re-requests any
12
+ // missing signed URL, and continues. `create` never allocates a node; the node
13
+ // appears only on `complete`, so a half-finished upload leaves no ghost files.
14
+
15
+ import { TroveError, ErrorCode } from './errors.js';
16
+ import { newId, isValidItemName } from './util.js';
17
+
18
+ export const DEFAULT_PART_SIZE = 8 * 1024 * 1024; // 8 MiB
19
+ const MIN_MULTIPART_PART = 5 * 1024 * 1024; // S3 floor (except final part)
20
+ const SINGLE_PUT_LIMIT = 5 * 1024 * 1024; // below this, one PUT beats multipart
21
+ const MAX_PARTS = 10_000; // S3 multipart ceiling
22
+ const SESSION_TTL_MS = 24 * 60 * 60 * 1000;
23
+
24
+ class MemorySessionStore {
25
+ constructor() {
26
+ this.map = new Map();
27
+ }
28
+ async get(id) {
29
+ return this.map.get(id) || null;
30
+ }
31
+ async put(session) {
32
+ this.map.set(session.id, session);
33
+ }
34
+ async delete(id) {
35
+ this.map.delete(id);
36
+ }
37
+ /**
38
+ * Which sessions have expired — WITHOUT deleting them.
39
+ *
40
+ * Deleting the record here is what leaked: the session holds the S3 `uploadId`, and
41
+ * that is the only handle that can abort the multipart. Drop it and the parts stay in
42
+ * the bucket, billed, with nothing left that could ever reclaim them (the collection
43
+ * scanner deliberately refuses to touch `obj_*` keys). Expiry has to go through
44
+ * UploadManager.sweepExpired, which aborts first and deletes after.
45
+ */
46
+ async expired(now) {
47
+ return [...this.map.entries()].filter(([, s]) => now - s.createdAt > SESSION_TTL_MS).map(([id]) => id);
48
+ }
49
+ }
50
+
51
+ export class UploadManager {
52
+ /**
53
+ * @param {object} deps
54
+ * @param {import('./storage/interface.js').StorageBackend} deps.storage
55
+ * @param {object} [deps.sessions] session store (defaults in-memory)
56
+ * @param {number} [deps.partSize]
57
+ */
58
+ constructor({ storage, storageFor, sessions, partSize = DEFAULT_PART_SIZE, maxBytes = null }) {
59
+ // Either a single backend, or a resolver keyed by collectionId (collections).
60
+ this.storageFor = storageFor ?? (async () => storage);
61
+ this.sessions = sessions ?? new MemorySessionStore();
62
+ this.partSize = partSize;
63
+ this.maxBytes = maxBytes || null; // per-file quota (null = unbounded)
64
+ }
65
+
66
+ // The self-describing limits the client gets back in every upload descriptor.
67
+ #limits() {
68
+ return {
69
+ maxBytes: this.maxBytes, // per-file quota (null = unbounded)
70
+ partSize: this.partSize,
71
+ minPartSize: MIN_MULTIPART_PART, // multipart floor, except the final part
72
+ singlePutLimit: SINGLE_PUT_LIMIT, // at/under this we use one PUT, not multipart
73
+ maxParts: MAX_PARTS,
74
+ };
75
+ }
76
+
77
+ #storage(collectionId) {
78
+ return this.storageFor(collectionId);
79
+ }
80
+
81
+ /**
82
+ * Begin an upload. Returns a plan the client follows.
83
+ * @param {{collectionId?:string, name:string, size:number, contentType?:string}} req
84
+ */
85
+ async create(req) {
86
+ if (!isValidItemName(req.name)) throw TroveError.invalid(`Invalid file name "${req.name}"`);
87
+ if (!(req.size >= 0)) throw TroveError.invalid('size must be a non-negative number');
88
+ if (this.maxBytes && req.size > this.maxBytes) {
89
+ // Deterministic per-file limit — retrying can't help, so it's non-retryable
90
+ // (capacity/rate quotas stay retryable via the default).
91
+ // TOO_LARGE (413), not QUOTA: the store has plenty of room, this file is simply
92
+ // bigger than this deployment permits. Reporting it as a capacity problem would
93
+ // send the user looking for space to free that would not help.
94
+ throw TroveError.tooLarge(`File exceeds the maximum upload size of ${this.maxBytes} bytes`, { details: { maxBytes: this.maxBytes, size: req.size } });
95
+ }
96
+ const collectionId = req.collectionId || 'default';
97
+ const storage = await this.#storage(collectionId);
98
+ const caps = storage.capabilities;
99
+ const storageKey = newId('obj');
100
+ const contentType = req.contentType || 'application/octet-stream';
101
+
102
+ const session = {
103
+ id: newId('up'),
104
+ storageKey,
105
+ collectionId,
106
+ name: req.name,
107
+ // Whether the caller asked to replace an existing item of this name. Carried on
108
+ // the session because the decision is made when the upload STARTS but has to be
109
+ // honoured when it COMPLETES, possibly much later.
110
+ overwrite: !!req.overwrite,
111
+ size: req.size,
112
+ contentType,
113
+ createdAt: Date.now(),
114
+ strategy: null,
115
+ partSize: this.partSize,
116
+ uploadId: null,
117
+ parts: {}, // partNumber -> { etag }
118
+ };
119
+
120
+ const limits = this.#limits();
121
+
122
+ // Small file + presign → single PUT straight to storage (never through us).
123
+ if (req.size <= SINGLE_PUT_LIMIT && caps.presignUpload) {
124
+ session.strategy = 'single';
125
+ await this.sessions.put(session);
126
+ const url = await storage.presignPut(storageKey, { contentType });
127
+ return { ...planSummary(session), strategy: 'single', multipart: false, presigned: true, url, limits };
128
+ }
129
+
130
+ // Multipart (presigned parts straight to storage, or streamed through us).
131
+ if (caps.multipart) {
132
+ session.strategy = caps.presignUpload ? 'presign' : 'direct';
133
+ const partCount = Math.max(1, Math.ceil(req.size / this.partSize));
134
+ // `#limits()` advertises maxParts in the very same response, and nothing enforced
135
+ // it: a 150 GiB file planned 19,200 parts against a ceiling of 10,000, which S3
136
+ // rejects at part 10,001 — after the client has transferred 80 GiB. On a presign
137
+ // backend it also signed every part up front, so the plan itself came back as a
138
+ // 1.9 MB JSON document. Refuse at negotiation, where it costs nothing.
139
+ if (partCount > MAX_PARTS) {
140
+ throw TroveError.tooLarge(
141
+ `This file needs ${partCount.toLocaleString()} parts of ${this.partSize} bytes, over the ${MAX_PARTS.toLocaleString()}-part limit`,
142
+ { details: { maxParts: MAX_PARTS, partCount, partSize: this.partSize, size: req.size } },
143
+ );
144
+ }
145
+ // Only now open the multipart. Refusing AFTER creating it left an upload open in
146
+ // the bucket with no session record, so nothing could ever abort it.
147
+ session.uploadId = await storage.createMultipart(storageKey, { contentType });
148
+ session.partCount = partCount;
149
+ await this.sessions.put(session);
150
+ const parts = [];
151
+ if (session.strategy === 'presign') {
152
+ for (let n = 1; n <= partCount; n++) {
153
+ parts.push({ partNumber: n, url: await storage.presignPart(storageKey, session.uploadId, n) });
154
+ }
155
+ }
156
+ return { ...planSummary(session), strategy: session.strategy, multipart: true, presigned: session.strategy === 'presign', partCount, parts, limits };
157
+ }
158
+
159
+ // Fallback: whole-object PUT streamed through us (tiny/simple backends).
160
+ session.strategy = 'direct-single';
161
+ await this.sessions.put(session);
162
+ return { ...planSummary(session), strategy: 'direct-single', multipart: false, presigned: false, limits };
163
+ }
164
+
165
+ /** Re-issue a signed URL for one part (resume after expiry). */
166
+ async signPart(uploadId, partNumber) {
167
+ const s = await this.#session(uploadId);
168
+ if (s.strategy !== 'presign') throw TroveError.invalid('signPart only applies to presigned uploads');
169
+ const storage = await this.#storage(s.collectionId);
170
+ return storage.presignPart(s.storageKey, s.uploadId, partNumber);
171
+ }
172
+
173
+ /** Client reports a completed presigned part (with its ETag from S3). */
174
+ async reportPart(uploadId, partNumber, etag) {
175
+ const s = await this.#session(uploadId);
176
+ if (!etag) throw TroveError.invalid('Part ETag required');
177
+ // Same bounds as uploadPart. Unbounded, a client could report hundreds of thousands
178
+ // of out-of-plan parts, all retained for the session's 24h TTL — and `status` hands
179
+ // that list back as the parts it may SKIP.
180
+ if (!Number.isInteger(partNumber) || partNumber < 1 || (s.partCount && partNumber > s.partCount)) {
181
+ throw TroveError.invalid(`Part ${partNumber} is outside this upload's ${s.partCount} part(s)`);
182
+ }
183
+ s.parts[partNumber] = { etag };
184
+ await this.sessions.put(s);
185
+ return { ok: true };
186
+ }
187
+
188
+ /** Direct backends: stream one part through us to storage. */
189
+ async uploadPart(uploadId, partNumber, body, opts = {}) {
190
+ const s = await this.#session(uploadId);
191
+ const storage = await this.#storage(s.collectionId);
192
+ if (s.strategy === 'direct-single') {
193
+ // The proxied body is raw bytes streamed straight to storage — the JSON body cap
194
+ // deliberately doesn't apply — so this is the only place a client can be stopped
195
+ // from writing an unbounded object through us.
196
+ const capped = capStream(body, this.maxBytes);
197
+ const info = await storage.put(s.storageKey, capped, { contentType: s.contentType, ...opts });
198
+ s.parts[1] = { etag: info.etag || 'single' };
199
+ await this.sessions.put(s);
200
+ return { partNumber: 1, etag: s.parts[1].etag };
201
+ }
202
+ if (s.strategy !== 'direct') throw TroveError.invalid('uploadPart only applies to direct uploads');
203
+ // A part outside the plan is stored, billed, and never merged — `complete` only
204
+ // walks 1..partCount. Silently accepting one meant a client could believe it had
205
+ // uploaded bytes that would never become part of the file.
206
+ // `NaN < 1` and `NaN > partCount` are BOTH false, so a non-numeric part number
207
+ // walked through and its bytes were written under the key "NaN" — billed, and
208
+ // unmergeable, since `complete` only ever walks 1..partCount.
209
+ if (!Number.isInteger(partNumber) || partNumber < 1 || (s.partCount && partNumber > s.partCount)) {
210
+ throw TroveError.invalid(`Part ${partNumber} is outside this upload's ${s.partCount} part(s)`);
211
+ }
212
+ // A part is exactly `partSize` bytes, except the last, which is smaller.
213
+ const res = await storage.putPart(s.storageKey, s.uploadId, partNumber, capStream(body, s.partSize), opts);
214
+ s.parts[partNumber] = { etag: res.etag };
215
+ await this.sessions.put(s);
216
+ return res;
217
+ }
218
+
219
+ /** Which parts are still outstanding (resume support). */
220
+ async status(uploadId) {
221
+ const s = await this.#session(uploadId);
222
+ const received = Object.keys(s.parts).map(Number).sort((a, b) => a - b);
223
+ return {
224
+ uploadId,
225
+ collectionId: s.collectionId,
226
+ strategy: s.strategy,
227
+ partCount: s.partCount ?? 1,
228
+ partSize: s.partSize,
229
+ received,
230
+ };
231
+ }
232
+
233
+ /**
234
+ * Finalise. Verifies all parts are present, completes multipart, and returns
235
+ * the object descriptor so the VFS can create the node.
236
+ * @returns {Promise<{storageKey, size, contentType, etag, collectionId, name}>}
237
+ */
238
+ async complete(uploadId, reportedParts) {
239
+ const s = await this.#session(uploadId);
240
+ let etag;
241
+ if (s.strategy === 'single' || s.strategy === 'direct-single') {
242
+ // Object already fully written by the client (single presigned PUT) or by us.
243
+ etag = s.parts[1]?.etag;
244
+ } else {
245
+ // Merge any client-reported ETags (presign) with what we recorded.
246
+ const parts = [];
247
+ const count = s.partCount ?? Object.keys(s.parts).length;
248
+ for (let n = 1; n <= count; n++) {
249
+ const fromClient = reportedParts?.find((p) => p.partNumber === n);
250
+ const etagN = fromClient?.etag || s.parts[n]?.etag;
251
+ if (!etagN) {
252
+ throw TroveError.invalid(`Missing part ${n} of ${count}`, { details: { partNumber: n } });
253
+ }
254
+ parts.push({ partNumber: n, etag: etagN });
255
+ }
256
+ const storage = await this.#storage(s.collectionId);
257
+ const res = await storage.completeMultipart(s.storageKey, s.uploadId, parts);
258
+ etag = res.etag;
259
+ }
260
+ // What actually landed, not what the client said it would send.
261
+ //
262
+ // `size` up to here has been the client's DECLARED size — the value the per-file
263
+ // limit was checked against at create(). Nothing re-checked the bytes, so declaring
264
+ // `size: 1` and then PUTting gigabytes walked straight past the limit and recorded a
265
+ // size that was simply false (which the quota meter, the UI and every later
266
+ // range read then believed). Ask the store.
267
+ const storage = await this.#storage(s.collectionId);
268
+ let size = s.size;
269
+ try {
270
+ const info = await storage.head(s.storageKey);
271
+ if (Number.isFinite(info?.size)) size = info.size;
272
+ } catch (err) {
273
+ // NOT_FOUND is the one error that means something specific: no bytes were ever
274
+ // stored. Swallowing it committed an item for an object that does not exist — the
275
+ // drive listing a file whose size is a number the client invented, whose download
276
+ // redirects to a 404, and which every indexer then raises a standing issue about.
277
+ // Reachable whenever a presigned PUT fails but `complete` still runs.
278
+ if (err?.code === ErrorCode.NOT_FOUND) {
279
+ await this.sessions.delete(uploadId);
280
+ throw TroveError.invalid('The upload was never written — no object exists for this session');
281
+ }
282
+ // Any other failure (a backend that can't answer a HEAD) leaves the declared size
283
+ // in place; better an approximate record than refusing an upload that succeeded.
284
+ }
285
+ if (this.maxBytes && size > this.maxBytes) {
286
+ // It is already in the store, so let it go rather than leaving an orphan that
287
+ // counts against the quota and belongs to nothing.
288
+ await storage.delete(s.storageKey).catch(() => {});
289
+ await this.sessions.delete(uploadId);
290
+ throw TroveError.tooLarge(
291
+ `File exceeds the maximum upload size of ${this.maxBytes} bytes`,
292
+ { details: { maxBytes: this.maxBytes, size } },
293
+ );
294
+ }
295
+ await this.sessions.delete(uploadId);
296
+ return {
297
+ storageKey: s.storageKey,
298
+ size,
299
+ contentType: s.contentType,
300
+ etag,
301
+ collectionId: s.collectionId,
302
+ name: s.name,
303
+ overwrite: !!s.overwrite,
304
+ };
305
+ }
306
+
307
+ /**
308
+ * Reclaim uploads the client started and never finished.
309
+ *
310
+ * A dropped connection, a closed tab, a crash — all leave a live multipart upload
311
+ * whose parts are stored and billed until something aborts them. That something is
312
+ * this: it goes through `abort`, which tells the backend to let the parts go before
313
+ * the session record (and with it the uploadId) is discarded.
314
+ *
315
+ * @returns {Promise<{aborted: number, failed: number}>}
316
+ */
317
+ async sweepExpired(now = Date.now()) {
318
+ const ids = (await this.sessions.expired?.(now)) || [];
319
+ let aborted = 0;
320
+ let failed = 0;
321
+ for (const id of ids) {
322
+ try {
323
+ await this.abort(id);
324
+ aborted++;
325
+ } catch (err) {
326
+ // Leave the session in place so the next sweep tries again — dropping it is
327
+ // exactly how the bytes became unreclaimable in the first place.
328
+ failed++;
329
+ console.error(`[trove] could not reclaim abandoned upload ${id}:`, err?.message || err);
330
+ }
331
+ }
332
+ return { aborted, failed };
333
+ }
334
+
335
+ async abort(uploadId) {
336
+ const s = await this.sessions.get(uploadId);
337
+ if (!s) return;
338
+ const storage = await this.#storage(s.collectionId);
339
+ // The session record holds the multipart uploadId, and that is the ONLY handle that
340
+ // can ever reclaim the staged parts. Swallowing the backend's failure and deleting
341
+ // the record anyway is precisely how the bytes became unreclaimable — and it made
342
+ // sweepExpired's failure branch, whose comment says exactly that, unreachable.
343
+ if (s.uploadId) await storage.abortMultipart(s.storageKey, s.uploadId);
344
+ else await storage.delete(s.storageKey).catch(() => {});
345
+ await this.sessions.delete(uploadId);
346
+ }
347
+
348
+ async #session(uploadId) {
349
+ const s = await this.sessions.get(uploadId);
350
+ if (!s) throw TroveError.notFound('Upload session');
351
+ return s;
352
+ }
353
+ }
354
+
355
+ function planSummary(s) {
356
+ return { uploadId: s.id, storageKey: s.storageKey, partSize: s.partSize, size: s.size, name: s.name, contentType: s.contentType };
357
+ }
358
+
359
+ /**
360
+ * Wrap a streamed body so it cannot exceed `max` bytes.
361
+ *
362
+ * Uploads bypass the JSON body cap by design — their bytes stream to storage rather
363
+ * than being buffered — which left the proxied part route with no ceiling at all: the
364
+ * declared size was checked at create() and nothing checked what actually arrived.
365
+ * Non-stream bodies (a Uint8Array from a test, a Blob) pass through untouched; they are
366
+ * already resident, so capping them here would not save the memory.
367
+ */
368
+ function capStream(body, max) {
369
+ if (!max || !body || typeof body.getReader !== 'function') return body;
370
+ const reader = body.getReader();
371
+ let total = 0;
372
+ return new ReadableStream({
373
+ async pull(controller) {
374
+ const { done, value } = await reader.read();
375
+ if (done) { controller.close(); return; }
376
+ total += value.byteLength;
377
+ if (total > max) {
378
+ await reader.cancel().catch(() => {});
379
+ controller.error(TroveError.tooLarge(`Upload body exceeds ${max} bytes`, { details: { maxBytes: max } }));
380
+ return;
381
+ }
382
+ controller.enqueue(value);
383
+ },
384
+ cancel(reason) { reader.cancel(reason).catch(() => {}); },
385
+ });
386
+ }