@3sln/trove 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/README.md +1227 -0
  2. package/package.json +75 -0
  3. package/packages/core/src/collections/index.js +249 -0
  4. package/packages/core/src/errors.js +186 -0
  5. package/packages/core/src/identity/discovery.js +210 -0
  6. package/packages/core/src/identity/index.js +188 -0
  7. package/packages/core/src/identity/jwt.js +199 -0
  8. package/packages/core/src/index.js +104 -0
  9. package/packages/core/src/indexers/contribution.js +115 -0
  10. package/packages/core/src/indexers/registry.js +162 -0
  11. package/packages/core/src/indexing.js +340 -0
  12. package/packages/core/src/issues.js +150 -0
  13. package/packages/core/src/kv.js +0 -0
  14. package/packages/core/src/links.js +141 -0
  15. package/packages/core/src/metadata/cursor.js +73 -0
  16. package/packages/core/src/metadata/interface.js +244 -0
  17. package/packages/core/src/metadata/memory.js +270 -0
  18. package/packages/core/src/metadata/sqlite.js +412 -0
  19. package/packages/core/src/notifications/index.js +139 -0
  20. package/packages/core/src/notifications/webpush.js +217 -0
  21. package/packages/core/src/plugins/contributions.js +177 -0
  22. package/packages/core/src/plugins/identity.js +98 -0
  23. package/packages/core/src/plugins/index.js +225 -0
  24. package/packages/core/src/plugins/indexers.js +142 -0
  25. package/packages/core/src/plugins/installStore.js +134 -0
  26. package/packages/core/src/plugins/package.js +102 -0
  27. package/packages/core/src/plugins/packageStore.js +61 -0
  28. package/packages/core/src/plugins/runtime.js +101 -0
  29. package/packages/core/src/plugins/sql.js +52 -0
  30. package/packages/core/src/retry.js +74 -0
  31. package/packages/core/src/scan.js +302 -0
  32. package/packages/core/src/search/embeddings.js +128 -0
  33. package/packages/core/src/search/index.js +200 -0
  34. package/packages/core/src/search/keywordStore.js +107 -0
  35. package/packages/core/src/search/sqliteStores.js +455 -0
  36. package/packages/core/src/search/tagMatch.js +59 -0
  37. package/packages/core/src/search/transformer.js +195 -0
  38. package/packages/core/src/search/vectorStore.js +274 -0
  39. package/packages/core/src/search/vectorize.js +249 -0
  40. package/packages/core/src/sidecar/document.js +213 -0
  41. package/packages/core/src/sidecar/index.js +174 -0
  42. package/packages/core/src/sidecar/manager.js +239 -0
  43. package/packages/core/src/sidecar/store.js +46 -0
  44. package/packages/core/src/signedUrls.js +170 -0
  45. package/packages/core/src/sqlite-d1.js +162 -0
  46. package/packages/core/src/sqlite-driver.js +42 -0
  47. package/packages/core/src/sqlite.js +162 -0
  48. package/packages/core/src/storage/filesystem.js +283 -0
  49. package/packages/core/src/storage/interface.js +222 -0
  50. package/packages/core/src/storage/memory.js +113 -0
  51. package/packages/core/src/storage/prefixed.js +75 -0
  52. package/packages/core/src/storage/s3.js +316 -0
  53. package/packages/core/src/storage/s3sigv4.js +185 -0
  54. package/packages/core/src/tasks.js +228 -0
  55. package/packages/core/src/uploads.js +386 -0
  56. package/packages/core/src/util.js +125 -0
  57. package/packages/core/src/vfs.js +666 -0
  58. package/packages/plugin-sdk/src/browser.js +316 -0
  59. package/packages/plugin-sdk/src/index.js +32 -0
  60. package/packages/plugin-sdk/src/protocol.js +59 -0
  61. package/packages/plugin-sdk/src/rpc.js +95 -0
  62. package/packages/server/src/adapters/bun.js +78 -0
  63. package/packages/server/src/adapters/node.js +115 -0
  64. package/packages/server/src/adapters/staticAssets.js +123 -0
  65. package/packages/server/src/adapters/webDist.js +70 -0
  66. package/packages/server/src/adapters/worker-tasks.js +206 -0
  67. package/packages/server/src/adapters/worker.js +159 -0
  68. package/packages/server/src/cachePolicy.js +34 -0
  69. package/packages/server/src/engine/README.md +88 -0
  70. package/packages/server/src/engine/actions/scanCollection.js +114 -0
  71. package/packages/server/src/engine/index.js +95 -0
  72. package/packages/server/src/engine/lazy.js +25 -0
  73. package/packages/server/src/engine/providers/access.js +363 -0
  74. package/packages/server/src/engine/providers/core.js +405 -0
  75. package/packages/server/src/engine/providers/scan.js +67 -0
  76. package/packages/server/src/index.js +698 -0
  77. package/packages/server/src/manifest.js +98 -0
  78. package/packages/server/src/mcp/auth.js +40 -0
  79. package/packages/server/src/mcp/index.js +213 -0
  80. package/packages/server/src/mcp/protocol.js +181 -0
  81. package/packages/server/src/mcp/tools.js +351 -0
  82. package/packages/server/src/router.js +229 -0
  83. package/packages/server/src/routes.js +1066 -0
  84. package/packages/server/src/scope.js +43 -0
  85. package/packages/web/dist/assets/chunk-4xqbzebh.js +5 -0
  86. package/packages/web/dist/assets/chunk-4xqbzebh.js.map +9 -0
  87. package/packages/web/dist/assets/chunk-h05bxfbs.js +5 -0
  88. package/packages/web/dist/assets/chunk-h05bxfbs.js.map +10 -0
  89. package/packages/web/dist/assets/main-4cxs7prw.js +356 -0
  90. package/packages/web/dist/assets/main-4cxs7prw.js.map +103 -0
  91. package/packages/web/dist/assets/styles-kcx1x337.css +1 -0
  92. package/packages/web/dist/icon.svg +11 -0
  93. package/packages/web/dist/index.html +16 -0
  94. package/packages/web/dist/sql-wasm.wasm +0 -0
  95. package/packages/web/dist/sw.js +186 -0
  96. package/packages/web/src/bl/actions.js +410 -0
  97. package/packages/web/src/bl/activity.js +306 -0
  98. package/packages/web/src/bl/commands.js +274 -0
  99. package/packages/web/src/bl/fileType.js +49 -0
  100. package/packages/web/src/bl/index.js +70 -0
  101. package/packages/web/src/bl/links.js +54 -0
  102. package/packages/web/src/bl/offline.js +268 -0
  103. package/packages/web/src/bl/openers.js +71 -0
  104. package/packages/web/src/bl/pluginInstall.js +59 -0
  105. package/packages/web/src/bl/services.js +143 -0
  106. package/packages/web/src/bl/social.js +234 -0
  107. package/packages/web/src/bl/tagQuery.js +44 -0
  108. package/packages/web/src/main.js +10 -0
  109. package/packages/web/src/platform/api.js +529 -0
  110. package/packages/web/src/platform/commands.js +89 -0
  111. package/packages/web/src/platform/context.js +77 -0
  112. package/packages/web/src/platform/contributions.js +156 -0
  113. package/packages/web/src/platform/index.js +150 -0
  114. package/packages/web/src/platform/keybindings.js +199 -0
  115. package/packages/web/src/platform/mediaUrls.js +137 -0
  116. package/packages/web/src/platform/navigation.js +131 -0
  117. package/packages/web/src/platform/notifications.js +50 -0
  118. package/packages/web/src/platform/overlay.js +81 -0
  119. package/packages/web/src/platform/pluginClientDb.js +132 -0
  120. package/packages/web/src/platform/pluginDock.js +141 -0
  121. package/packages/web/src/platform/pluginFrames.js +194 -0
  122. package/packages/web/src/platform/pluginHost.js +648 -0
  123. package/packages/web/src/platform/pluginMedia.js +62 -0
  124. package/packages/web/src/platform/pluginModules.js +90 -0
  125. package/packages/web/src/platform/pluginNet.js +71 -0
  126. package/packages/web/src/platform/pluginPackage.js +247 -0
  127. package/packages/web/src/platform/pluginRpc.js +377 -0
  128. package/packages/web/src/platform/pluginSigning.js +168 -0
  129. package/packages/web/src/platform/pluginStore.js +67 -0
  130. package/packages/web/src/platform/settings.js +101 -0
  131. package/packages/web/src/platform/spatialNav.js +286 -0
  132. package/packages/web/src/platform/viewport.js +123 -0
  133. package/packages/web/src/platform/voice.js +133 -0
  134. package/packages/web/src/platform/voiceSearch.js +155 -0
  135. package/packages/web/src/platform/whenclause.js +162 -0
  136. package/packages/web/src/platform/workbench.js +156 -0
  137. package/packages/web/src/runtime.js +73 -0
  138. package/packages/web/src/styles.css +1382 -0
  139. package/packages/web/src/ui/components/activityBar.js +35 -0
  140. package/packages/web/src/ui/components/activityPanel.js +132 -0
  141. package/packages/web/src/ui/components/commandPalette.js +154 -0
  142. package/packages/web/src/ui/components/editorArea.js +75 -0
  143. package/packages/web/src/ui/components/launcher.js +392 -0
  144. package/packages/web/src/ui/components/openers/index.js +212 -0
  145. package/packages/web/src/ui/components/openers/markdown.js +222 -0
  146. package/packages/web/src/ui/components/overlays.js +255 -0
  147. package/packages/web/src/ui/components/phoneChrome.js +188 -0
  148. package/packages/web/src/ui/components/pluginReview.js +151 -0
  149. package/packages/web/src/ui/components/pluginsView.js +120 -0
  150. package/packages/web/src/ui/components/settingsView.js +258 -0
  151. package/packages/web/src/ui/components/social.js +290 -0
  152. package/packages/web/src/ui/components/statusBar.js +198 -0
  153. package/packages/web/src/ui/components/views/grid.js +115 -0
  154. package/packages/web/src/ui/components/views/index.js +155 -0
  155. package/packages/web/src/ui/components/views/list.js +50 -0
  156. package/packages/web/src/ui/components/views/parts.js +58 -0
  157. package/packages/web/src/ui/compositions/workbench.js +125 -0
  158. package/packages/web/src/ui/format.js +33 -0
  159. package/packages/web/src/ui/icon.js +81 -0
  160. package/packages/web/src/ui/media.js +114 -0
  161. package/packages/web/src/ui/sanitize.js +86 -0
  162. package/packages/web/src/workbench.js +205 -0
@@ -0,0 +1,162 @@
1
+ // A tiny SQLite abstraction that every backend conforms to, so stores don't care
2
+ // whether they run on local files (bun:sqlite / node:sqlite), Cloudflare D1, a
3
+ // Durable Object, or an in-browser wasm build. The handle is ASYNC — the only
4
+ // shape D1/DO/wasm can satisfy — and the local backend just wraps its synchronous
5
+ // driver in resolved promises.
6
+ //
7
+ // const db = await provider.obtain({ key: 'metadata' });
8
+ // await db.exec('CREATE TABLE t (x)');
9
+ // await db.all('SELECT * FROM t WHERE x = ?', x);
10
+ //
11
+ // The provider is a keyed pool: obtain() lazily creates (and memoizes) the DB for
12
+ // a key, so per-plugin / per-domain databases are just distinct keys. The shell
13
+ // injects whichever provider fits the runtime.
14
+
15
+ import { TroveError } from './errors.js';
16
+
17
+ /** The database handle. All methods are async so every backend can implement it. */
18
+ export class SqliteDatabase {
19
+ async exec(sql) { throw TroveError.unsupported('SqliteDatabase.exec'); }
20
+ async run(sql, ...params) { throw TroveError.unsupported('SqliteDatabase.run'); }
21
+ async get(sql, ...params) { throw TroveError.unsupported('SqliteDatabase.get'); }
22
+ async all(sql, ...params) { throw TroveError.unsupported('SqliteDatabase.all'); }
23
+ /** Run statements atomically. @param {{sql:string, params?:any[]}[]} statements */
24
+ async batch(statements) { throw TroveError.unsupported('SqliteDatabase.batch'); }
25
+ async close() {}
26
+ }
27
+
28
+ // Plugin SQL safety lives in ./plugins/sql.js so the browser can use it without
29
+ // pulling a database driver in behind it. Re-exported here, where callers expect it.
30
+ export { assertSafePluginSql, stripSqlLiterals } from './plugins/sql.js';
31
+
32
+ /** A keyed pool/factory of databases. The shell injects one. */
33
+ export class SqliteProvider {
34
+ /**
35
+ * Does data written through this provider survive a restart? False unless a backend
36
+ * says otherwise, so "durable" is something a provider claims rather than something
37
+ * callers assume. The server asks before putting the search index in SQLite: an
38
+ * index in an ephemeral database is worse than one in memory, because it looks
39
+ * persistent right up until the restart that proves it isn't.
40
+ */
41
+ get durable() { return false; }
42
+ async init() {}
43
+ /** @returns {Promise<SqliteDatabase>} the lazily-created, memoized db for `key`. */
44
+ async obtain({ key }) { throw TroveError.unsupported('SqliteProvider.obtain'); }
45
+ /** Destroy the db for `key` (close + delete its backing store). */
46
+ async drop({ key }) {}
47
+ async close() {}
48
+ }
49
+
50
+ // --- local backend (bun:sqlite / node:sqlite) --------------------------------
51
+ // Kept in this module but with all node-specific bits (fs) imported lazily inside
52
+ // obtain(), so importing the interfaces stays safe in a Worker that never uses it.
53
+
54
+ const CORE_KEYS = new Set(['metadata', 'kv', 'plugins']); // these share one main db file
55
+
56
+ class LocalSqliteDatabase extends SqliteDatabase {
57
+ constructor(raw) {
58
+ super();
59
+ this.raw = raw;
60
+ }
61
+ async exec(sql) { this.raw.exec(sql); }
62
+ async run(sql, ...params) { return this.raw.prepare(sql).run(...params); }
63
+ async get(sql, ...params) { return this.raw.prepare(sql).get(...params) ?? null; }
64
+ async all(sql, ...params) { return this.raw.prepare(sql).all(...params); }
65
+ async batch(statements) {
66
+ this.raw.exec('BEGIN');
67
+ try {
68
+ for (const { sql, params = [] } of statements) this.raw.prepare(sql).run(...params);
69
+ this.raw.exec('COMMIT');
70
+ } catch (err) {
71
+ try { this.raw.exec('ROLLBACK'); } catch { /* already rolled back */ }
72
+ throw err;
73
+ }
74
+ }
75
+ async close() { this.raw.close?.(); }
76
+ }
77
+
78
+ export class LocalSqliteProvider extends SqliteProvider {
79
+ /**
80
+ * @param {{ path?: string }} opts Main db file. Core keys (metadata, kv) share it;
81
+ * other keys (e.g. plugin scopes) get an isolated sibling file under `stores/`.
82
+ * Use ':memory:' for an ephemeral shared in-memory db (tests).
83
+ */
84
+ constructor(opts = {}) {
85
+ super();
86
+ this.path = opts.path || ':memory:';
87
+ this._pool = new Map(); // resolvedPath -> LocalSqliteDatabase
88
+ }
89
+
90
+ get durable() { return this.path !== ':memory:'; }
91
+
92
+ // Resolve a key to a pool token. In-memory: core keys share one db, every other
93
+ // key gets its own isolated in-memory db (so plugin scopes never collide). On
94
+ // disk: core keys share the main file, others get isolated sibling files.
95
+ #resolve(key) {
96
+ if (this.path === ':memory:') return CORE_KEYS.has(key) ? ':memory:#core' : ':memory:#' + key;
97
+ if (CORE_KEYS.has(key)) return this.path;
98
+ return join(dirname(this.path), 'stores', sanitize(key) + '.db');
99
+ }
100
+
101
+ async obtain({ key }) {
102
+ const resolved = this.#resolve(key);
103
+ let db = this._pool.get(resolved);
104
+ if (!db) {
105
+ const file = resolved.startsWith(':memory:') ? ':memory:' : resolved;
106
+ if (file !== ':memory:') {
107
+ const { mkdir } = await import('node:fs/promises');
108
+ await mkdir(dirname(file), { recursive: true });
109
+ }
110
+ const { openDatabase } = await import('./sqlite-driver.js');
111
+ db = new LocalSqliteDatabase(await openDatabase(file));
112
+ this._pool.set(resolved, db);
113
+ }
114
+ return db;
115
+ }
116
+
117
+ async drop({ key }) {
118
+ const resolved = this.#resolve(key);
119
+ const db = this._pool.get(resolved);
120
+ if (db) { await db.close(); this._pool.delete(resolved); }
121
+ if (!resolved.startsWith(':memory:')) {
122
+ const { rm } = await import('node:fs/promises');
123
+ for (const suffix of ['', '-wal', '-shm']) await rm(resolved + suffix, { force: true }).catch(() => {});
124
+ }
125
+ }
126
+
127
+ async close() {
128
+ for (const db of this._pool.values()) await db.close();
129
+ this._pool.clear();
130
+ }
131
+ }
132
+
133
+ // Pure path helpers (no node:path, so this module loads anywhere).
134
+ function dirname(p) {
135
+ const i = p.lastIndexOf('/');
136
+ return i <= 0 ? '.' : p.slice(0, i);
137
+ }
138
+ function join(...parts) {
139
+ return parts.join('/').replace(/\/{2,}/g, '/');
140
+ }
141
+ // Map a store key to a filename that is safe AND unique.
142
+ //
143
+ // Sanitizing alone is lossy: every disallowed character collapses to `_`, so
144
+ // `pstore:alice@x.com:plg:p` and `pstore:alice_x.com:plg:p` land on the same file —
145
+ // two principals sharing one database. Since a principal id is often an email, and
146
+ // with a JWT it is whatever `sub` the IdP mints, that is a collision an attacker can
147
+ // sometimes choose. The hash suffix makes the mapping injective; the readable prefix
148
+ // is kept only so the directory is browsable.
149
+ function sanitize(key) {
150
+ return key.replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 96) + '.' + fnv1a(key);
151
+ }
152
+
153
+ // FNV-1a, 32-bit. Not a security hash — it only has to be collision-resistant enough
154
+ // to separate keys that sanitize to the same string, which are near-identical inputs.
155
+ function fnv1a(str) {
156
+ let h = 0x811c9dc5;
157
+ for (let i = 0; i < str.length; i++) {
158
+ h ^= str.charCodeAt(i);
159
+ h = Math.imul(h, 0x01000193) >>> 0;
160
+ }
161
+ return h.toString(16).padStart(8, '0');
162
+ }
@@ -0,0 +1,283 @@
1
+ // Filesystem storage backend (Node). Also the "NAS" backend — point `root` at a
2
+ // mounted network share. Stores each object as a file under a two-level sharded
3
+ // directory (ab/cd/<key>) to avoid huge flat directories. Supports range reads
4
+ // (media seeking, resumable downloads); no presigning, so the server proxies
5
+ // bytes for these backends. Multipart is emulated with per-part temp files that
6
+ // are concatenated on complete — writes are staged to a temp file and atomically
7
+ // renamed so a crash mid-write never yields a half-written object.
8
+
9
+ import { createReadStream, createWriteStream } from 'node:fs';
10
+ import fs from 'node:fs/promises';
11
+ import path from 'node:path';
12
+ import { Readable } from 'node:stream';
13
+ import { pipeline } from 'node:stream/promises';
14
+ import { StorageBackend, resolveRange } from './interface.js';
15
+ import { TroveError, wrapError } from '../errors.js';
16
+ import { newId } from '../util.js';
17
+
18
+ // A key becomes a filename, so anything a filename can't hold has to be encoded —
19
+ // REVERSIBLY. Replacing every disallowed character with `_` was not: `list()` could only
20
+ // ever report the mangled name back, so a collection using bucket-and-prefix over a
21
+ // filesystem (keys like `team-a/obj_9f…`) listed nothing at all, adopt and refresh
22
+ // silently never ran, and every one of its items came back as an orphan — a durable
23
+ // "your files are gone" warning on a drive where nothing was wrong.
24
+ //
25
+ // `~XX` over UTF-8 bytes, with `~` itself encoded, so decoding is exact. Keys made only
26
+ // of safe characters — which is every key Trove mints (`obj_<hex>`) — encode to
27
+ // themselves, so this changes no existing filename.
28
+ function encodeKey(key) {
29
+ let out = '';
30
+ for (const byte of new TextEncoder().encode(key)) {
31
+ const c = String.fromCharCode(byte);
32
+ out += /[a-zA-Z0-9._-]/.test(c) ? c : '~' + byte.toString(16).padStart(2, '0');
33
+ }
34
+ return out;
35
+ }
36
+ function decodeKey(name) {
37
+ const bytes = [];
38
+ for (let i = 0; i < name.length; i++) {
39
+ if (name[i] === '~' && /^[0-9a-f]{2}$/i.test(name.slice(i + 1, i + 3))) {
40
+ bytes.push(parseInt(name.slice(i + 1, i + 3), 16));
41
+ i += 2;
42
+ } else {
43
+ bytes.push(name.charCodeAt(i) & 0xff);
44
+ }
45
+ }
46
+ return new TextDecoder().decode(new Uint8Array(bytes));
47
+ }
48
+
49
+ function shardPath(root, key) {
50
+ const safe = encodeKey(key);
51
+ const a = (safe.slice(0, 2) || '__').padEnd(2, '_');
52
+ const b = (safe.slice(2, 4) || '__').padEnd(2, '_');
53
+ return path.join(root, 'objects', a, b, safe);
54
+ }
55
+
56
+ async function toNodeStream(body) {
57
+ if (body instanceof Readable) return body;
58
+ if (body instanceof Uint8Array || body instanceof ArrayBuffer) {
59
+ return Readable.from(Buffer.from(body instanceof ArrayBuffer ? new Uint8Array(body) : body));
60
+ }
61
+ if (typeof Blob !== 'undefined' && body instanceof Blob) {
62
+ return Readable.fromWeb(body.stream());
63
+ }
64
+ if (body instanceof ReadableStream || typeof body?.getReader === 'function') {
65
+ return Readable.fromWeb(body);
66
+ }
67
+ if (typeof body?.[Symbol.asyncIterator] === 'function') return Readable.from(body);
68
+ if (typeof body === 'string') return Readable.from(Buffer.from(body));
69
+ throw TroveError.invalid('Unsupported body type');
70
+ }
71
+
72
+ export class FilesystemStorage extends StorageBackend {
73
+ /** @param {{root: string}} opts */
74
+ constructor({ root }) {
75
+ super();
76
+ if (!root) throw TroveError.invalid('FilesystemStorage requires a root directory');
77
+ this.root = path.resolve(root);
78
+ this.tmp = path.join(this.root, 'tmp');
79
+ }
80
+
81
+ get capabilities() {
82
+ return { presignDownload: false, presignUpload: false, multipart: true, range: true, list: true, usage: true };
83
+ }
84
+
85
+ async #ensureDir(dir) {
86
+ await fs.mkdir(dir, { recursive: true });
87
+ }
88
+
89
+ /**
90
+ * Space on the filesystem holding this root.
91
+ *
92
+ * Reports the FILESYSTEM's numbers, not the drive's — if Trove shares a disk with
93
+ * everything else on the machine, "12 GB free" is the fact that actually governs
94
+ * whether the next upload succeeds, and reporting only Trove's own footprint would
95
+ * leave someone confident with a full disk. `bavail` (free to a non-root user), not
96
+ * `bfree`, for the same reason: the reserved blocks are not available to us.
97
+ */
98
+ async usage() {
99
+ // statfs needs a path that EXISTS, and the root is created lazily on first write —
100
+ // so asking before anything has been uploaded would report "unknown" on exactly the
101
+ // empty drive where someone is most likely to be checking. Walk up to the nearest
102
+ // existing ancestor; it is on the same filesystem, which is what we are measuring.
103
+ let dir = this.root;
104
+ for (let i = 0; i < 8; i++) {
105
+ try {
106
+ const st = await fs.statfs(dir);
107
+ const total = st.blocks * st.bsize;
108
+ const available = st.bavail * st.bsize;
109
+ return { total, available, used: total - available };
110
+ } catch (err) {
111
+ if (err?.code !== 'ENOENT') return null; // old runtime, exotic mount: don't guess
112
+ const parent = path.dirname(dir);
113
+ if (parent === dir) return null;
114
+ dir = parent;
115
+ }
116
+ }
117
+ return null;
118
+ }
119
+
120
+ /**
121
+ * Walk the object tree so a scan can see files that arrived without Trove.
122
+ *
123
+ * Keys here are shard-mapped through a LOSSY sanitizer, so the mapping is only
124
+ * reversible for keys that survive it unchanged — which every key Trove writes does
125
+ * (`obj_<hex>`), and a hand-dropped `My Photo.jpg` does not. Rather than guess, this
126
+ * reports only files sitting at the exact path their own name maps to, and counts the
127
+ * rest as `unaddressable` so the caller can say so out loud instead of silently
128
+ * ignoring them.
129
+ */
130
+ async list({ prefix = '', cursor = null, limit = 1000 } = {}) {
131
+ const base = path.join(this.root, 'objects');
132
+ let found;
133
+ try {
134
+ found = await fs.readdir(base, { recursive: true, withFileTypes: true });
135
+ } catch (err) {
136
+ if (err.code === 'ENOENT') return { objects: [], nextCursor: null, unaddressable: 0 };
137
+ throw wrapError(err);
138
+ }
139
+ const entries = [];
140
+ let unaddressable = 0;
141
+ for (const d of found) {
142
+ if (!d.isFile()) continue;
143
+ const full = path.join(d.parentPath || d.path, d.name);
144
+ // Decode back to the real key first — the filename is an encoding of it, and the
145
+ // prefix a caller filters on is expressed in key space, not filename space.
146
+ const key = decodeKey(d.name);
147
+ // It is addressable only if it round-trips: a file dropped into the shard tree by
148
+ // hand, or left by an older layout, doesn't map back to a key we can serve.
149
+ if (shardPath(this.root, key) !== full) { unaddressable++; continue; }
150
+ if (prefix && !key.startsWith(prefix)) continue;
151
+ entries.push({ key, full });
152
+ }
153
+ entries.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
154
+ const start = cursor ? entries.findIndex((e) => e.key > cursor) : 0;
155
+ const from = start === -1 ? entries.length : start;
156
+ const page = entries.slice(from, from + limit);
157
+ const objects = [];
158
+ for (const e of page) {
159
+ try {
160
+ const st = await fs.stat(e.full);
161
+ // etagOfStat, NOT a second hand-rolled format. An etag only means anything if
162
+ // it is computed identically everywhere: this listing is compared against the
163
+ // etag `put` recorded, and a different encoding of the same facts made every
164
+ // object look changed — a scan that re-read and re-indexed the entire drive.
165
+ objects.push({ key: e.key, size: st.size, etag: etagOfStat(st), modifiedAt: Math.floor(st.mtimeMs) });
166
+ } catch { /* vanished mid-walk; the next scan will see it */ }
167
+ }
168
+ return {
169
+ objects,
170
+ nextCursor: from + limit < entries.length ? page[page.length - 1].key : null,
171
+ // A figure about the WHOLE tree, so it is reported once — on the first page only.
172
+ // Returning it on every page made a caller that sums pages multiply it by the page
173
+ // count: one stray file in a 3,000-object drive was reported as seven.
174
+ ...(cursor ? {} : { unaddressable }),
175
+ };
176
+ }
177
+
178
+ async put(key, body, opts = {}) {
179
+ const dest = shardPath(this.root, key);
180
+ await this.#ensureDir(path.dirname(dest));
181
+ await this.#ensureDir(this.tmp);
182
+ const tmpFile = path.join(this.tmp, newId('put'));
183
+ let written = 0;
184
+ try {
185
+ const src = await toNodeStream(body);
186
+ if (opts.onProgress) {
187
+ src.on('data', (c) => {
188
+ written += c.length;
189
+ opts.onProgress(written);
190
+ });
191
+ }
192
+ const out = createWriteStream(tmpFile);
193
+ await pipeline(src, out, { signal: opts.signal });
194
+ await fs.rename(tmpFile, dest); // atomic within a filesystem
195
+ } catch (err) {
196
+ await fs.rm(tmpFile, { force: true }).catch(() => {});
197
+ throw wrapError(err);
198
+ }
199
+ const stat = await fs.stat(dest);
200
+ return { size: stat.size, contentType: opts.contentType, etag: etagOfStat(stat) };
201
+ }
202
+
203
+ async head(key) {
204
+ try {
205
+ const stat = await fs.stat(shardPath(this.root, key));
206
+ return { size: stat.size, etag: etagOfStat(stat) };
207
+ } catch (err) {
208
+ throw wrapError(err);
209
+ }
210
+ }
211
+
212
+ async get(key, opts = {}) {
213
+ const file = shardPath(this.root, key);
214
+ let stat;
215
+ try {
216
+ stat = await fs.stat(file);
217
+ } catch (err) {
218
+ throw wrapError(err);
219
+ }
220
+ const total = stat.size;
221
+ const range = resolveRange(opts.range, total);
222
+ const streamOpts = range ? { start: range.start, end: range.end } : {};
223
+ const nodeStream = createReadStream(file, { ...streamOpts, signal: opts.signal });
224
+ return {
225
+ stream: Readable.toWeb(nodeStream),
226
+ size: range ? range.end - range.start + 1 : total,
227
+ etag: etagOfStat(stat),
228
+ range,
229
+ };
230
+ }
231
+
232
+ async delete(key) {
233
+ await fs.rm(shardPath(this.root, key), { force: true });
234
+ }
235
+
236
+ // --- multipart (staged part files, concatenated on complete) --------------
237
+
238
+ async createMultipart(key) {
239
+ const uploadId = newId('mp');
240
+ await this.#ensureDir(path.join(this.tmp, uploadId));
241
+ return uploadId;
242
+ }
243
+
244
+ async putPart(key, uploadId, partNumber, body, opts = {}) {
245
+ const partFile = path.join(this.tmp, uploadId, String(partNumber));
246
+ const src = await toNodeStream(body);
247
+ await pipeline(src, createWriteStream(partFile), { signal: opts.signal });
248
+ const stat = await fs.stat(partFile);
249
+ return { partNumber, etag: etagOfStat(stat) };
250
+ }
251
+
252
+ async completeMultipart(key, uploadId, parts) {
253
+ const dir = path.join(this.tmp, uploadId);
254
+ const dest = shardPath(this.root, key);
255
+ await this.#ensureDir(path.dirname(dest));
256
+ const tmpFile = path.join(this.tmp, newId('complete'));
257
+ const out = createWriteStream(tmpFile);
258
+ const ordered = [...parts].sort((a, b) => a.partNumber - b.partNumber);
259
+ try {
260
+ for (const p of ordered) {
261
+ const partFile = path.join(dir, String(p.partNumber));
262
+ await pipeline(createReadStream(partFile), out, { end: false });
263
+ }
264
+ out.end();
265
+ await new Promise((res, rej) => out.on('finish', res).on('error', rej));
266
+ await fs.rename(tmpFile, dest);
267
+ } catch (err) {
268
+ await fs.rm(tmpFile, { force: true }).catch(() => {});
269
+ throw wrapError(err);
270
+ }
271
+ await fs.rm(dir, { recursive: true, force: true }).catch(() => {});
272
+ const stat = await fs.stat(dest);
273
+ return { size: stat.size, etag: etagOfStat(stat) };
274
+ }
275
+
276
+ async abortMultipart(key, uploadId) {
277
+ await fs.rm(path.join(this.tmp, uploadId), { recursive: true, force: true }).catch(() => {});
278
+ }
279
+ }
280
+
281
+ function etagOfStat(stat) {
282
+ return `"${stat.size.toString(16)}-${Math.floor(stat.mtimeMs).toString(16)}"`;
283
+ }
@@ -0,0 +1,222 @@
1
+ // StorageBackend — the pluggable blob store contract.
2
+ //
3
+ // A backend stores opaque byte blobs addressed by a *storage key* (an internal
4
+ // id, NOT the user-visible path — the VFS/metadata layer owns the path↔key map,
5
+ // so rename/move is a metadata edit and never copies bytes).
6
+ //
7
+ // Backends advertise `capabilities` so higher layers degrade gracefully:
8
+ // - presignDownload/presignUpload: hand the client a direct, time-limited URL
9
+ // (S3) so big transfers bypass our server entirely.
10
+ // - multipart: parallel, resumable multi-part uploads.
11
+ // - range: byte-range reads (needed for media seeking / resumable downloads).
12
+ // A backend that lacks presigning (filesystem, memory) still works — the server
13
+ // proxies the bytes instead.
14
+ //
15
+ // Every method that can fail must throw a TroveError (use wrapError at the
16
+ // boundary). Every long operation must honour an AbortSignal in `opts.signal`.
17
+
18
+ import { TroveError } from '../errors.js';
19
+ import { readAll, concatBytes } from '../util.js';
20
+
21
+ // Re-exported for storage backends (memory.js) that assemble chunks themselves.
22
+ export { concatBytes as concat };
23
+
24
+ /**
25
+ * @typedef {object} StorageCapabilities
26
+ * @property {boolean} presignDownload
27
+ * @property {boolean} presignUpload single-shot PUT via signed URL
28
+ * @property {boolean} multipart parallel multipart uploads
29
+ * @property {boolean} range byte-range GET
30
+ * @property {boolean} list can enumerate what the store actually holds
31
+ * @property {boolean} usage can report how much space is left
32
+ */
33
+
34
+ /**
35
+ * @typedef {object} ObjectInfo
36
+ * @property {number} size
37
+ * @property {string} [contentType]
38
+ * @property {string} [etag]
39
+ */
40
+
41
+ export class StorageBackend {
42
+ /** @type {StorageCapabilities} */
43
+ get capabilities() {
44
+ return { presignDownload: false, presignUpload: false, multipart: false, range: false, list: false, usage: false };
45
+ }
46
+
47
+ /**
48
+ * Store a whole object.
49
+ * @param {string} key
50
+ * @param {ReadableStream|Uint8Array|Blob|ArrayBuffer} body
51
+ * @param {{size?: number, contentType?: string, signal?: AbortSignal, onProgress?: (n:number)=>void}} [opts]
52
+ * @returns {Promise<ObjectInfo>}
53
+ */
54
+ async put(key, body, opts) {
55
+ throw TroveError.unsupported('put not implemented');
56
+ }
57
+
58
+ /**
59
+ * Read an object (optionally a byte range).
60
+ * @param {string} key
61
+ * @param {{range?: {start:number, end?:number}, signal?: AbortSignal}} [opts]
62
+ * @returns {Promise<{stream: ReadableStream, size: number, contentType?: string, etag?: string, range?: {start:number,end:number,total:number}}>}
63
+ */
64
+ async get(key, opts) {
65
+ throw TroveError.unsupported('get not implemented');
66
+ }
67
+
68
+ /** @returns {Promise<ObjectInfo>} throws NOT_FOUND if absent. */
69
+ async head(key) {
70
+ throw TroveError.unsupported('head not implemented');
71
+ }
72
+
73
+ /** @param {{signal?: AbortSignal}} [opts] */
74
+ async delete(key, opts) {
75
+ throw TroveError.unsupported('delete not implemented');
76
+ }
77
+
78
+ /**
79
+ * Page through the objects actually present in the store.
80
+ *
81
+ * Everything else here addresses a key the caller already knows. This is the one
82
+ * operation that asks the store what it HAS — which is what makes it possible to
83
+ * notice things that happened without Trove: a file dropped in the bucket by
84
+ * another tool, an object deleted out from under an item, bytes replaced in place.
85
+ * Without it the drive can only ever know what it did itself.
86
+ *
87
+ * Paged with an opaque cursor rather than returning everything, because a real
88
+ * bucket does not fit in memory. `capabilities.list` says whether a backend can
89
+ * answer at all.
90
+ *
91
+ * @param {{prefix?: string, cursor?: string|null, limit?: number, signal?: AbortSignal}} [opts]
92
+ * @returns {Promise<{objects: Array<{key: string, size: number, etag?: string, modifiedAt?: number}>, nextCursor: string|null}>}
93
+ */
94
+ async list(opts) {
95
+ throw TroveError.unsupported('list not implemented');
96
+ }
97
+
98
+ /**
99
+ * How much room is left.
100
+ *
101
+ * Only some backends can answer. A filesystem or a NAS mount knows exactly, and that
102
+ * is the case where it matters most: a disk fills up and every upload starts failing
103
+ * with no warning that anything was coming. An object store has no such number — S3
104
+ * is effectively unbounded and a bucket quota, if any, lives outside the API — so it
105
+ * returns null rather than inventing one. `capabilities.usage` says which you're
106
+ * dealing with, so a UI can show a real gauge or say nothing at all instead of
107
+ * displaying a meter that means nothing.
108
+ *
109
+ * @returns {Promise<{used: number, available: number, total: number}|null>}
110
+ */
111
+ async usage() {
112
+ return null;
113
+ }
114
+
115
+ // --- Optional: presigned direct access (S3) --------------------------------
116
+
117
+ /** @returns {Promise<string>} a URL the client GETs directly. */
118
+ async presignGet(key, opts) {
119
+ throw TroveError.unsupported('This backend cannot presign downloads');
120
+ }
121
+ /** @returns {Promise<string>} a URL the client PUTs a whole object to. */
122
+ async presignPut(key, opts) {
123
+ throw TroveError.unsupported('This backend cannot presign uploads');
124
+ }
125
+
126
+ // --- Optional: multipart (S3) ----------------------------------------------
127
+
128
+ /** @returns {Promise<string>} uploadId */
129
+ async createMultipart(key, opts) {
130
+ throw TroveError.unsupported('This backend does not support multipart uploads');
131
+ }
132
+ /** @returns {Promise<string>} a signed URL to PUT one part to. */
133
+ async presignPart(key, uploadId, partNumber, opts) {
134
+ throw TroveError.unsupported('This backend does not support multipart uploads');
135
+ }
136
+ /**
137
+ * Upload one part directly (non-presign backends).
138
+ * @returns {Promise<{partNumber:number, etag:string}>}
139
+ */
140
+ async putPart(key, uploadId, partNumber, body, opts) {
141
+ throw TroveError.unsupported('This backend does not support multipart uploads');
142
+ }
143
+ /** @param {{partNumber:number, etag:string}[]} parts */
144
+ async completeMultipart(key, uploadId, parts, opts) {
145
+ throw TroveError.unsupported('This backend does not support multipart uploads');
146
+ }
147
+ async abortMultipart(key, uploadId, opts) {
148
+ throw TroveError.unsupported('This backend does not support multipart uploads');
149
+ }
150
+ }
151
+
152
+ /** Coerce assorted body types into a Uint8Array (used by simple backends). */
153
+ export async function toBytes(body) {
154
+ if (body == null) return new Uint8Array(0);
155
+ if (body instanceof Uint8Array) return body;
156
+ if (body instanceof ArrayBuffer) return new Uint8Array(body);
157
+ if (typeof Blob !== 'undefined' && body instanceof Blob) {
158
+ return new Uint8Array(await body.arrayBuffer());
159
+ }
160
+ if (typeof body === 'string') return new TextEncoder().encode(body);
161
+ if (body instanceof ReadableStream || typeof body?.getReader === 'function') {
162
+ return await readAll(body);
163
+ }
164
+ // Async iterable (Node stream)
165
+ if (typeof body?.[Symbol.asyncIterator] === 'function') {
166
+ const chunks = [];
167
+ for await (const c of body) chunks.push(c instanceof Uint8Array ? c : new Uint8Array(c));
168
+ return concatBytes(chunks);
169
+ }
170
+ throw TroveError.invalid('Unsupported body type');
171
+ }
172
+
173
+ /**
174
+ * Resolve a requested byte range against an object's real size.
175
+ *
176
+ * Two cases the per-backend copies of this arithmetic each got wrong:
177
+ *
178
+ * The SUFFIX form — `bytes=-500` means "the last 500 bytes" (RFC 9110 §14.1.4), and
179
+ * media players and container-footer probes send it routinely. Read as `{start: 0}`
180
+ * it served the FRONT of the file under a 206 that claimed to be what was asked for,
181
+ * which a client has no way to detect.
182
+ *
183
+ * An EMPTY object is deliberately not treated as unsatisfiable. A 0-byte file is a
184
+ * real file, and `end = total - 1 = -1` made every one of them throw: permanently
185
+ * un-indexable (a standing issue whose Retry re-runs the same failure) and unopenable
186
+ * in the text viewer, which reads through a range. The whole of it is zero bytes, so
187
+ * that is what we serve.
188
+ *
189
+ * @returns {{start:number,end:number,total:number}|null} null → serve the whole object
190
+ */
191
+ export function resolveRange(range, total) {
192
+ if (!range || total === 0) return null;
193
+ let start;
194
+ let end;
195
+ if (range.suffix != null) {
196
+ const n = Math.min(range.suffix, total);
197
+ // `bytes=-0` asks for the last zero bytes, which is unsatisfiable by definition —
198
+ // and S3 answers it with the WHOLE object under a 200, so it has to be caught here
199
+ // rather than left to the backend.
200
+ if (n <= 0) throw TroveError.badRange('Range not satisfiable');
201
+ start = total - n;
202
+ end = total - 1;
203
+ } else {
204
+ start = range.start ?? 0;
205
+ end = range.end != null ? Math.min(range.end, total - 1) : total - 1;
206
+ }
207
+ if (start < 0 || start > end || start >= total) throw TroveError.badRange('Range not satisfiable');
208
+ return { start, end, total };
209
+ }
210
+
211
+ /** A ReadableStream over an in-memory byte array (with optional range). */
212
+ export function bytesStream(bytes, range) {
213
+ const start = range?.start ?? 0;
214
+ const end = range?.end != null ? Math.min(range.end + 1, bytes.length) : bytes.length;
215
+ const slice = bytes.subarray(start, end);
216
+ return new ReadableStream({
217
+ start(controller) {
218
+ controller.enqueue(slice);
219
+ controller.close();
220
+ },
221
+ });
222
+ }