@3sln/trove 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1227 -0
- package/package.json +75 -0
- package/packages/core/src/collections/index.js +249 -0
- package/packages/core/src/errors.js +186 -0
- package/packages/core/src/identity/discovery.js +210 -0
- package/packages/core/src/identity/index.js +188 -0
- package/packages/core/src/identity/jwt.js +199 -0
- package/packages/core/src/index.js +104 -0
- package/packages/core/src/indexers/contribution.js +115 -0
- package/packages/core/src/indexers/registry.js +162 -0
- package/packages/core/src/indexing.js +340 -0
- package/packages/core/src/issues.js +150 -0
- package/packages/core/src/kv.js +0 -0
- package/packages/core/src/links.js +141 -0
- package/packages/core/src/metadata/cursor.js +73 -0
- package/packages/core/src/metadata/interface.js +244 -0
- package/packages/core/src/metadata/memory.js +270 -0
- package/packages/core/src/metadata/sqlite.js +412 -0
- package/packages/core/src/notifications/index.js +139 -0
- package/packages/core/src/notifications/webpush.js +217 -0
- package/packages/core/src/plugins/contributions.js +177 -0
- package/packages/core/src/plugins/identity.js +98 -0
- package/packages/core/src/plugins/index.js +225 -0
- package/packages/core/src/plugins/indexers.js +142 -0
- package/packages/core/src/plugins/installStore.js +134 -0
- package/packages/core/src/plugins/package.js +102 -0
- package/packages/core/src/plugins/packageStore.js +61 -0
- package/packages/core/src/plugins/runtime.js +101 -0
- package/packages/core/src/plugins/sql.js +52 -0
- package/packages/core/src/retry.js +74 -0
- package/packages/core/src/scan.js +302 -0
- package/packages/core/src/search/embeddings.js +128 -0
- package/packages/core/src/search/index.js +200 -0
- package/packages/core/src/search/keywordStore.js +107 -0
- package/packages/core/src/search/sqliteStores.js +455 -0
- package/packages/core/src/search/tagMatch.js +59 -0
- package/packages/core/src/search/transformer.js +195 -0
- package/packages/core/src/search/vectorStore.js +274 -0
- package/packages/core/src/search/vectorize.js +249 -0
- package/packages/core/src/sidecar/document.js +213 -0
- package/packages/core/src/sidecar/index.js +174 -0
- package/packages/core/src/sidecar/manager.js +239 -0
- package/packages/core/src/sidecar/store.js +46 -0
- package/packages/core/src/signedUrls.js +170 -0
- package/packages/core/src/sqlite-d1.js +162 -0
- package/packages/core/src/sqlite-driver.js +42 -0
- package/packages/core/src/sqlite.js +162 -0
- package/packages/core/src/storage/filesystem.js +283 -0
- package/packages/core/src/storage/interface.js +222 -0
- package/packages/core/src/storage/memory.js +113 -0
- package/packages/core/src/storage/prefixed.js +75 -0
- package/packages/core/src/storage/s3.js +316 -0
- package/packages/core/src/storage/s3sigv4.js +185 -0
- package/packages/core/src/tasks.js +228 -0
- package/packages/core/src/uploads.js +386 -0
- package/packages/core/src/util.js +125 -0
- package/packages/core/src/vfs.js +666 -0
- package/packages/plugin-sdk/src/browser.js +316 -0
- package/packages/plugin-sdk/src/index.js +32 -0
- package/packages/plugin-sdk/src/protocol.js +59 -0
- package/packages/plugin-sdk/src/rpc.js +95 -0
- package/packages/server/src/adapters/bun.js +78 -0
- package/packages/server/src/adapters/node.js +115 -0
- package/packages/server/src/adapters/staticAssets.js +123 -0
- package/packages/server/src/adapters/webDist.js +70 -0
- package/packages/server/src/adapters/worker-tasks.js +206 -0
- package/packages/server/src/adapters/worker.js +159 -0
- package/packages/server/src/cachePolicy.js +34 -0
- package/packages/server/src/engine/README.md +88 -0
- package/packages/server/src/engine/actions/scanCollection.js +114 -0
- package/packages/server/src/engine/index.js +95 -0
- package/packages/server/src/engine/lazy.js +25 -0
- package/packages/server/src/engine/providers/access.js +363 -0
- package/packages/server/src/engine/providers/core.js +405 -0
- package/packages/server/src/engine/providers/scan.js +67 -0
- package/packages/server/src/index.js +698 -0
- package/packages/server/src/manifest.js +98 -0
- package/packages/server/src/mcp/auth.js +40 -0
- package/packages/server/src/mcp/index.js +213 -0
- package/packages/server/src/mcp/protocol.js +181 -0
- package/packages/server/src/mcp/tools.js +351 -0
- package/packages/server/src/router.js +229 -0
- package/packages/server/src/routes.js +1066 -0
- package/packages/server/src/scope.js +43 -0
- package/packages/web/dist/assets/chunk-4xqbzebh.js +5 -0
- package/packages/web/dist/assets/chunk-4xqbzebh.js.map +9 -0
- package/packages/web/dist/assets/chunk-h05bxfbs.js +5 -0
- package/packages/web/dist/assets/chunk-h05bxfbs.js.map +10 -0
- package/packages/web/dist/assets/main-4cxs7prw.js +356 -0
- package/packages/web/dist/assets/main-4cxs7prw.js.map +103 -0
- package/packages/web/dist/assets/styles-kcx1x337.css +1 -0
- package/packages/web/dist/icon.svg +11 -0
- package/packages/web/dist/index.html +16 -0
- package/packages/web/dist/sql-wasm.wasm +0 -0
- package/packages/web/dist/sw.js +186 -0
- package/packages/web/src/bl/actions.js +410 -0
- package/packages/web/src/bl/activity.js +306 -0
- package/packages/web/src/bl/commands.js +274 -0
- package/packages/web/src/bl/fileType.js +49 -0
- package/packages/web/src/bl/index.js +70 -0
- package/packages/web/src/bl/links.js +54 -0
- package/packages/web/src/bl/offline.js +268 -0
- package/packages/web/src/bl/openers.js +71 -0
- package/packages/web/src/bl/pluginInstall.js +59 -0
- package/packages/web/src/bl/services.js +143 -0
- package/packages/web/src/bl/social.js +234 -0
- package/packages/web/src/bl/tagQuery.js +44 -0
- package/packages/web/src/main.js +10 -0
- package/packages/web/src/platform/api.js +529 -0
- package/packages/web/src/platform/commands.js +89 -0
- package/packages/web/src/platform/context.js +77 -0
- package/packages/web/src/platform/contributions.js +156 -0
- package/packages/web/src/platform/index.js +150 -0
- package/packages/web/src/platform/keybindings.js +199 -0
- package/packages/web/src/platform/mediaUrls.js +137 -0
- package/packages/web/src/platform/navigation.js +131 -0
- package/packages/web/src/platform/notifications.js +50 -0
- package/packages/web/src/platform/overlay.js +81 -0
- package/packages/web/src/platform/pluginClientDb.js +132 -0
- package/packages/web/src/platform/pluginDock.js +141 -0
- package/packages/web/src/platform/pluginFrames.js +194 -0
- package/packages/web/src/platform/pluginHost.js +648 -0
- package/packages/web/src/platform/pluginMedia.js +62 -0
- package/packages/web/src/platform/pluginModules.js +90 -0
- package/packages/web/src/platform/pluginNet.js +71 -0
- package/packages/web/src/platform/pluginPackage.js +247 -0
- package/packages/web/src/platform/pluginRpc.js +377 -0
- package/packages/web/src/platform/pluginSigning.js +168 -0
- package/packages/web/src/platform/pluginStore.js +67 -0
- package/packages/web/src/platform/settings.js +101 -0
- package/packages/web/src/platform/spatialNav.js +286 -0
- package/packages/web/src/platform/viewport.js +123 -0
- package/packages/web/src/platform/voice.js +133 -0
- package/packages/web/src/platform/voiceSearch.js +155 -0
- package/packages/web/src/platform/whenclause.js +162 -0
- package/packages/web/src/platform/workbench.js +156 -0
- package/packages/web/src/runtime.js +73 -0
- package/packages/web/src/styles.css +1382 -0
- package/packages/web/src/ui/components/activityBar.js +35 -0
- package/packages/web/src/ui/components/activityPanel.js +132 -0
- package/packages/web/src/ui/components/commandPalette.js +154 -0
- package/packages/web/src/ui/components/editorArea.js +75 -0
- package/packages/web/src/ui/components/launcher.js +392 -0
- package/packages/web/src/ui/components/openers/index.js +212 -0
- package/packages/web/src/ui/components/openers/markdown.js +222 -0
- package/packages/web/src/ui/components/overlays.js +255 -0
- package/packages/web/src/ui/components/phoneChrome.js +188 -0
- package/packages/web/src/ui/components/pluginReview.js +151 -0
- package/packages/web/src/ui/components/pluginsView.js +120 -0
- package/packages/web/src/ui/components/settingsView.js +258 -0
- package/packages/web/src/ui/components/social.js +290 -0
- package/packages/web/src/ui/components/statusBar.js +198 -0
- package/packages/web/src/ui/components/views/grid.js +115 -0
- package/packages/web/src/ui/components/views/index.js +155 -0
- package/packages/web/src/ui/components/views/list.js +50 -0
- package/packages/web/src/ui/components/views/parts.js +58 -0
- package/packages/web/src/ui/compositions/workbench.js +125 -0
- package/packages/web/src/ui/format.js +33 -0
- package/packages/web/src/ui/icon.js +81 -0
- package/packages/web/src/ui/media.js +114 -0
- package/packages/web/src/ui/sanitize.js +86 -0
- package/packages/web/src/workbench.js +205 -0
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
// VectorizeVectorStore — an adapter over Cloudflare Vectorize, proving the
|
|
2
|
+
// VectorStore contract fits a serverless, metadata-filtered ANN index. Two
|
|
3
|
+
// access modes share one code path:
|
|
4
|
+
// • Workers binding — `new VectorizeVectorStore({ binding: env.VECTORIZE, dimensions })`.
|
|
5
|
+
// Calls the binding RPC surface directly (upsert/query/deleteByIds/getByIds).
|
|
6
|
+
// • REST API — `new VectorizeVectorStore({ accountId, apiKey, indexName, dimensions })`.
|
|
7
|
+
// Talks to the v2 REST endpoints over fetch (works on Node, Bun, Workers).
|
|
8
|
+
//
|
|
9
|
+
// Two quirks the interface must paper over:
|
|
10
|
+
// 1. Vector ids are capped at 64 bytes, so a docId longer than that collapses
|
|
11
|
+
// to a deterministic SHA-256 hex; the original docId always lives in
|
|
12
|
+
// metadata so query results and remove(docId) still round-trip.
|
|
13
|
+
// 2. Vectorize has NO delete-by-filter — only deleteByIds. So the wholesale
|
|
14
|
+
// removes (by node / indexer) are implemented query-then-delete: query a
|
|
15
|
+
// zero vector with a metadata filter, collect the matched ids, delete them,
|
|
16
|
+
// and repeat until a query comes back empty. This needs metadata indexes on
|
|
17
|
+
// the filtered properties, which we create lazily on first use (REST) or
|
|
18
|
+
// assume are provisioned out-of-band (binding).
|
|
19
|
+
|
|
20
|
+
import { VectorStore } from './vectorStore.js';
|
|
21
|
+
import { TroveError, wrapError } from '../errors.js';
|
|
22
|
+
import { withRetry } from '../retry.js';
|
|
23
|
+
|
|
24
|
+
// Reserved metadata keys — the routing columns, never treated as user fields.
|
|
25
|
+
const RESERVED = ['docId', 'nodeId', 'indexerId'];
|
|
26
|
+
// Vectorize caps topK at 20 when returnMetadata is "all" (full metadata is only
|
|
27
|
+
// returned for the top slice), so both querying and pagination stay within it.
|
|
28
|
+
const MAX_TOPK = 20;
|
|
29
|
+
|
|
30
|
+
export class VectorizeVectorStore extends VectorStore {
|
|
31
|
+
/**
|
|
32
|
+
* @param {object} cfg
|
|
33
|
+
* @param {number} cfg.dimensions vector dimensionality (required)
|
|
34
|
+
* @param {object} [cfg.binding] Workers Vectorize binding (binding mode)
|
|
35
|
+
* @param {string} [cfg.accountId] Cloudflare account id (REST mode)
|
|
36
|
+
* @param {string} [cfg.apiKey] API token, sent as Bearer (REST mode)
|
|
37
|
+
* @param {string} [cfg.indexName] Vectorize index name (REST mode)
|
|
38
|
+
* @param {string} [cfg.namespace] optional query namespace (REST mode)
|
|
39
|
+
*/
|
|
40
|
+
constructor(cfg = {}) {
|
|
41
|
+
super();
|
|
42
|
+
if (!cfg.dimensions) throw TroveError.invalid('VectorizeVectorStore requires "dimensions"');
|
|
43
|
+
this._dimensions = cfg.dimensions;
|
|
44
|
+
this.binding = cfg.binding || null;
|
|
45
|
+
if (!this.binding) {
|
|
46
|
+
for (const k of ['accountId', 'apiKey', 'indexName']) {
|
|
47
|
+
if (!cfg[k]) throw TroveError.invalid(`VectorizeVectorStore (REST mode) requires "${k}"`);
|
|
48
|
+
}
|
|
49
|
+
this._baseUrl = `https://api.cloudflare.com/client/v4/accounts/${cfg.accountId}/vectorize/v2/indexes/${cfg.indexName}`;
|
|
50
|
+
}
|
|
51
|
+
this.cfg = cfg;
|
|
52
|
+
this._ready = null;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// --- transport ----------------------------------------------------------
|
|
56
|
+
|
|
57
|
+
/** One REST call. Cloudflare wraps every response as { success, result, errors }. */
|
|
58
|
+
async #req(path, body, { ndjson } = {}) {
|
|
59
|
+
return withRetry(
|
|
60
|
+
async () => {
|
|
61
|
+
let res;
|
|
62
|
+
try {
|
|
63
|
+
res = await fetch(this._baseUrl + path, {
|
|
64
|
+
method: 'POST',
|
|
65
|
+
headers: {
|
|
66
|
+
authorization: `Bearer ${this.cfg.apiKey}`,
|
|
67
|
+
'content-type': ndjson ? 'application/x-ndjson' : 'application/json',
|
|
68
|
+
},
|
|
69
|
+
body: ndjson ? body : body !== undefined ? JSON.stringify(body) : undefined,
|
|
70
|
+
});
|
|
71
|
+
} catch (err) {
|
|
72
|
+
throw wrapError(err);
|
|
73
|
+
}
|
|
74
|
+
if (res.status === 429 || res.status >= 500) throw TroveError.transient(`Vectorize ${res.status}`);
|
|
75
|
+
const json = await res.json().catch(() => ({}));
|
|
76
|
+
if (!res.ok || json.success === false) {
|
|
77
|
+
const msg = json?.errors?.map?.((e) => e.message).filter(Boolean).join('; ') || res.status;
|
|
78
|
+
throw TroveError.internal(`Vectorize ${path} failed: ${msg}`);
|
|
79
|
+
}
|
|
80
|
+
return json.result ?? {};
|
|
81
|
+
},
|
|
82
|
+
{ retries: 3 },
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** One binding RPC. Retried on transient classification; errors normalised. */
|
|
87
|
+
async #binding(fn) {
|
|
88
|
+
return withRetry(
|
|
89
|
+
async () => {
|
|
90
|
+
try {
|
|
91
|
+
return await fn();
|
|
92
|
+
} catch (err) {
|
|
93
|
+
throw wrapError(err);
|
|
94
|
+
}
|
|
95
|
+
},
|
|
96
|
+
{ retries: 3 },
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Ensure the metadata indexes deletes rely on exist (idempotent). */
|
|
101
|
+
async #ensure() {
|
|
102
|
+
if (!this._ready) {
|
|
103
|
+
this._ready = (async () => {
|
|
104
|
+
// Binding mode: metadata indexes are provisioned out-of-band (wrangler),
|
|
105
|
+
// so there is nothing to create — just proceed.
|
|
106
|
+
if (this.binding) return;
|
|
107
|
+
for (const propertyName of ['nodeId', 'indexerId']) {
|
|
108
|
+
await this.#req('/metadata_index/create', { propertyName, indexType: 'string' }).catch((e) => {
|
|
109
|
+
// Tolerate a metadata index that already exists; surface real errors.
|
|
110
|
+
if (!/already|exist/i.test(e.message)) throw e;
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
})();
|
|
114
|
+
}
|
|
115
|
+
return this._ready;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// --- writes -------------------------------------------------------------
|
|
119
|
+
|
|
120
|
+
async add(docs) {
|
|
121
|
+
if (!docs.length) return;
|
|
122
|
+
await this.#ensure();
|
|
123
|
+
const vectors = await Promise.all(
|
|
124
|
+
docs.map(async (d) => ({
|
|
125
|
+
id: await vectorId(d.id),
|
|
126
|
+
values: Array.from(d.vector),
|
|
127
|
+
metadata: buildMetadata(d),
|
|
128
|
+
})),
|
|
129
|
+
);
|
|
130
|
+
if (this.binding) {
|
|
131
|
+
await this.#binding(() => this.binding.upsert(vectors));
|
|
132
|
+
} else {
|
|
133
|
+
// Upsert body is NDJSON: one JSON vector object per line.
|
|
134
|
+
const body = vectors.map((v) => JSON.stringify(v)).join('\n');
|
|
135
|
+
await this.#req('/upsert', body, { ndjson: true });
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async remove(docId) {
|
|
140
|
+
await this.#ensure();
|
|
141
|
+
await this.#deleteByIds([await vectorId(docId)]);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
async removeByNode(nodeId) {
|
|
145
|
+
await this.#purge({ nodeId: { $eq: nodeId } });
|
|
146
|
+
}
|
|
147
|
+
async removeByIndexer(indexerId) {
|
|
148
|
+
await this.#purge({ indexerId: { $eq: indexerId } });
|
|
149
|
+
}
|
|
150
|
+
async removeByNodeIndexer(nodeId, indexerId) {
|
|
151
|
+
await this.#purge({ nodeId: { $eq: nodeId }, indexerId: { $eq: indexerId } });
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Query-then-delete: Vectorize can't delete by filter, so page through the
|
|
156
|
+
* matching vectors (topK capped at MAX_TOPK) and delete their ids until a
|
|
157
|
+
* query returns nothing — so a file with many chunks is fully cleared.
|
|
158
|
+
*/
|
|
159
|
+
async #purge(filter) {
|
|
160
|
+
await this.#ensure();
|
|
161
|
+
const zero = new Array(this._dimensions).fill(0);
|
|
162
|
+
// Bounded loop guards against an eventually-consistent delete looping forever.
|
|
163
|
+
for (let page = 0; page < 10000; page++) {
|
|
164
|
+
const matches = await this.#queryRaw(zero, { topK: MAX_TOPK, filter });
|
|
165
|
+
if (!matches.length) break;
|
|
166
|
+
await this.#deleteByIds(matches.map((m) => m.id));
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
async #deleteByIds(ids) {
|
|
171
|
+
if (!ids.length) return;
|
|
172
|
+
if (this.binding) await this.#binding(() => this.binding.deleteByIds(ids));
|
|
173
|
+
else await this.#req('/delete_by_ids', { ids });
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// --- reads --------------------------------------------------------------
|
|
177
|
+
|
|
178
|
+
async query(vector, opts = {}) {
|
|
179
|
+
await this.#ensure();
|
|
180
|
+
const limit = opts.limit ?? 20;
|
|
181
|
+
// Over-fetch a little (limit*4) but never past the returnMetadata="all" cap.
|
|
182
|
+
const topK = Math.min(limit * 4, MAX_TOPK);
|
|
183
|
+
const filter = opts.indexers?.length ? { indexerId: { $in: opts.indexers } } : undefined;
|
|
184
|
+
const matches = await this.#queryRaw(vector, { topK, filter });
|
|
185
|
+
return matches.slice(0, limit).map(toResult);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** Raw nearest-neighbour lookup. Normalises the two match-envelope shapes. */
|
|
189
|
+
async #queryRaw(vector, { topK, filter }) {
|
|
190
|
+
const values = Array.from(vector);
|
|
191
|
+
let res;
|
|
192
|
+
if (this.binding) {
|
|
193
|
+
res = await this.#binding(() =>
|
|
194
|
+
this.binding.query(values, {
|
|
195
|
+
topK,
|
|
196
|
+
returnValues: false,
|
|
197
|
+
returnMetadata: 'all',
|
|
198
|
+
...(filter ? { filter } : {}),
|
|
199
|
+
}),
|
|
200
|
+
);
|
|
201
|
+
} else {
|
|
202
|
+
res = await this.#req('/query', {
|
|
203
|
+
vector: values,
|
|
204
|
+
topK,
|
|
205
|
+
returnValues: false,
|
|
206
|
+
returnMetadata: 'all',
|
|
207
|
+
...(filter ? { filter } : {}),
|
|
208
|
+
...(this.cfg.namespace ? { namespace: this.cfg.namespace } : {}),
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
// Both { matches } and { count, matches } are accepted.
|
|
212
|
+
return res?.matches || [];
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// --- helpers --------------------------------------------------------------
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Vector id: ids must be ≤ 64 bytes. A short docId passes through so it stays
|
|
220
|
+
* human-readable; a longer one collapses to a deterministic SHA-256 hex (64
|
|
221
|
+
* chars) so remove(docId) recomputes the same id. The original docId always
|
|
222
|
+
* lives in metadata regardless (see buildMetadata).
|
|
223
|
+
*/
|
|
224
|
+
async function vectorId(docId) {
|
|
225
|
+
const bytes = new TextEncoder().encode(docId);
|
|
226
|
+
if (bytes.length <= 64) return docId;
|
|
227
|
+
const buf = new Uint8Array(await crypto.subtle.digest('SHA-256', bytes));
|
|
228
|
+
return [...buf].map((b) => b.toString(16).padStart(2, '0')).join('');
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** Metadata = routing keys + the flat, primitive user fields Vectorize can index. */
|
|
232
|
+
function buildMetadata(doc) {
|
|
233
|
+
const md = { docId: doc.id, nodeId: doc.nodeId, indexerId: doc.indexerId };
|
|
234
|
+
for (const [k, v] of Object.entries(doc.fields || {})) {
|
|
235
|
+
if (RESERVED.includes(k) || v == null) continue; // never let a field shadow a routing key
|
|
236
|
+
const t = typeof v;
|
|
237
|
+
// Vectorize metadata only stores flat primitives; skip nested objects/arrays.
|
|
238
|
+
if (t === 'string' || t === 'number' || t === 'boolean') md[k] = v;
|
|
239
|
+
}
|
|
240
|
+
return md;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Map a Vectorize match to the VectorStore result shape (fields = non-reserved metadata). */
|
|
244
|
+
function toResult(m) {
|
|
245
|
+
const md = m.metadata || {};
|
|
246
|
+
const fields = {};
|
|
247
|
+
for (const [k, v] of Object.entries(md)) if (!RESERVED.includes(k)) fields[k] = v;
|
|
248
|
+
return { docId: md.docId, nodeId: md.nodeId, indexerId: md.indexerId, score: m.score, fields };
|
|
249
|
+
}
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
// The per-file sidecar document — a small CRDT that holds everything mutable and
|
|
2
|
+
// social about a file WITHOUT touching the file bytes: its conversation
|
|
3
|
+
// (threaded comments + reactions), its tags, its per-indexer facet data (scoped
|
|
4
|
+
// to the indexer that produced it), and who's subscribed to the thread.
|
|
5
|
+
//
|
|
6
|
+
// It's designed to live as cold JSON in object storage and be merged whenever
|
|
7
|
+
// it's read-before-write, so two servers (or a stale hot copy vs the cold one)
|
|
8
|
+
// converge without a lock or a conflict. Every field is a CRDT register:
|
|
9
|
+
// • tags — LWW-Element-Set (add/remove wins by Lamport stamp)
|
|
10
|
+
// • facets — per-indexer LWW register
|
|
11
|
+
// • comments — grow-only map; body edit & deletion are LWW registers; reactions
|
|
12
|
+
// are an OR-map (per user, per emoji, LWW on/off)
|
|
13
|
+
// • subscribers — LWW register (subscribed / muted)
|
|
14
|
+
// A Lamport `clock` orders concurrent ops; ties break on the actor id, so merges
|
|
15
|
+
// are deterministic and commutative.
|
|
16
|
+
|
|
17
|
+
export const SIDECAR_VERSION = 1;
|
|
18
|
+
|
|
19
|
+
export function emptyDoc(nodeId) {
|
|
20
|
+
return { v: SIDECAR_VERSION, nodeId, clock: 0, tags: {}, facets: {}, comments: {}, subscribers: {} };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// A stamp orders and tie-breaks a write. Higher clock wins; equal clock → higher
|
|
24
|
+
// actor string wins (arbitrary but total and deterministic).
|
|
25
|
+
function newer(a, b) {
|
|
26
|
+
if (!a) return false;
|
|
27
|
+
if (!b) return true;
|
|
28
|
+
if (a.at !== b.at) return a.at > b.at;
|
|
29
|
+
return String(a.actor) >= String(b.actor);
|
|
30
|
+
}
|
|
31
|
+
function tick(doc) {
|
|
32
|
+
doc.clock = (doc.clock || 0) + 1;
|
|
33
|
+
return doc.clock;
|
|
34
|
+
}
|
|
35
|
+
function stamp(doc, actor, at) {
|
|
36
|
+
return { at: at ?? tick(doc), actor: actor || 'system' };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// ---- mutators (each returns the affected entity) ---------------------------
|
|
40
|
+
|
|
41
|
+
export function addComment(doc, { id, parentId = null, author, body, mentions = [], actor, at, ts }) {
|
|
42
|
+
const s = stamp(doc, actor ?? author?.id, at);
|
|
43
|
+
const comment = {
|
|
44
|
+
id, parentId, author, body: { text: body, ...s },
|
|
45
|
+
// `at` (Lamport) orders/merges; `createdAt` (wall clock) is for display only.
|
|
46
|
+
mentions, createdAt: ts ?? wallClock(), reactions: {}, deleted: null, ...s,
|
|
47
|
+
};
|
|
48
|
+
doc.comments[id] = comment;
|
|
49
|
+
subscribe(doc, author?.id, { actor: author?.id, at: s.at });
|
|
50
|
+
return comment;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function wallClock() {
|
|
54
|
+
try {
|
|
55
|
+
return Date.now();
|
|
56
|
+
} catch {
|
|
57
|
+
return 0;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function editComment(doc, id, { body, actor, at }) {
|
|
62
|
+
const c = doc.comments[id];
|
|
63
|
+
if (!c) return null;
|
|
64
|
+
const s = stamp(doc, actor, at);
|
|
65
|
+
if (newer(s, c.body)) c.body = { text: body, ...s };
|
|
66
|
+
return c;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function deleteComment(doc, id, { actor, at } = {}) {
|
|
70
|
+
const c = doc.comments[id];
|
|
71
|
+
if (!c) return null;
|
|
72
|
+
const s = stamp(doc, actor, at);
|
|
73
|
+
if (newer(s, c.deleted)) c.deleted = s;
|
|
74
|
+
return c;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function react(doc, id, emoji, userId, on = true, { at } = {}) {
|
|
78
|
+
const c = doc.comments[id];
|
|
79
|
+
if (!c) return null;
|
|
80
|
+
const s = stamp(doc, userId, at);
|
|
81
|
+
const map = (c.reactions[emoji] ??= {});
|
|
82
|
+
const cur = map[userId];
|
|
83
|
+
if (newer(s, cur)) map[userId] = { on, ...s };
|
|
84
|
+
return c;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export function setTag(doc, name, { value = true, actor, at } = {}) {
|
|
88
|
+
const s = stamp(doc, actor, at);
|
|
89
|
+
const cur = doc.tags[name];
|
|
90
|
+
if (newer(s, cur)) doc.tags[name] = { present: true, value, ...s };
|
|
91
|
+
return doc.tags[name];
|
|
92
|
+
}
|
|
93
|
+
export function removeTag(doc, name, { actor, at } = {}) {
|
|
94
|
+
const s = stamp(doc, actor, at);
|
|
95
|
+
const cur = doc.tags[name];
|
|
96
|
+
if (newer(s, cur)) doc.tags[name] = { present: false, value: cur?.value, ...s };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export function setFacet(doc, indexerId, data, { actor, at } = {}) {
|
|
100
|
+
const s = stamp(doc, actor ?? indexerId, at);
|
|
101
|
+
const cur = doc.facets[indexerId];
|
|
102
|
+
if (newer(s, cur)) doc.facets[indexerId] = { data, ...s };
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export function subscribe(doc, userId, { muted = false, actor, at } = {}) {
|
|
106
|
+
if (!userId) return;
|
|
107
|
+
const s = stamp(doc, actor ?? userId, at);
|
|
108
|
+
const cur = doc.subscribers[userId];
|
|
109
|
+
if (newer(s, cur)) doc.subscribers[userId] = { subscribed: true, muted, ...s };
|
|
110
|
+
}
|
|
111
|
+
export function unsubscribe(doc, userId, { actor, at } = {}) {
|
|
112
|
+
if (!userId) return;
|
|
113
|
+
const s = stamp(doc, actor ?? userId, at);
|
|
114
|
+
const cur = doc.subscribers[userId];
|
|
115
|
+
if (newer(s, cur)) doc.subscribers[userId] = { subscribed: false, muted: cur?.muted, ...s };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// ---- merge (CRDT join) -----------------------------------------------------
|
|
119
|
+
|
|
120
|
+
export function mergeDoc(a, b) {
|
|
121
|
+
if (!a) return structuredCloneSafe(b);
|
|
122
|
+
if (!b) return structuredCloneSafe(a);
|
|
123
|
+
const out = { v: SIDECAR_VERSION, nodeId: a.nodeId || b.nodeId, clock: Math.max(a.clock || 0, b.clock || 0), tags: {}, facets: {}, comments: {}, subscribers: {} };
|
|
124
|
+
|
|
125
|
+
for (const key of union(a.tags, b.tags)) out.tags[key] = pick(a.tags[key], b.tags[key]);
|
|
126
|
+
for (const key of union(a.facets, b.facets)) out.facets[key] = pick(a.facets[key], b.facets[key]);
|
|
127
|
+
for (const key of union(a.subscribers, b.subscribers)) out.subscribers[key] = pick(a.subscribers[key], b.subscribers[key]);
|
|
128
|
+
|
|
129
|
+
for (const id of union(a.comments, b.comments)) {
|
|
130
|
+
out.comments[id] = mergeComment(a.comments[id], b.comments[id]);
|
|
131
|
+
}
|
|
132
|
+
return out;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function mergeComment(x, y) {
|
|
136
|
+
if (!x) return structuredCloneSafe(y);
|
|
137
|
+
if (!y) return structuredCloneSafe(x);
|
|
138
|
+
const base = newer({ at: x.at, actor: x.actor }, { at: y.at, actor: y.actor }) ? x : y;
|
|
139
|
+
const out = { ...structuredCloneSafe(base) };
|
|
140
|
+
out.body = pick(x.body, y.body);
|
|
141
|
+
out.deleted = pick(x.deleted, y.deleted);
|
|
142
|
+
out.reactions = {};
|
|
143
|
+
for (const emoji of union(x.reactions, y.reactions)) {
|
|
144
|
+
out.reactions[emoji] = {};
|
|
145
|
+
const users = union(x.reactions?.[emoji], y.reactions?.[emoji]);
|
|
146
|
+
for (const u of users) out.reactions[emoji][u] = pick(x.reactions?.[emoji]?.[u], y.reactions?.[emoji]?.[u]);
|
|
147
|
+
}
|
|
148
|
+
return out;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function pick(a, b) {
|
|
152
|
+
return newer(a, b) ? a : b;
|
|
153
|
+
}
|
|
154
|
+
function union(a = {}, b = {}) {
|
|
155
|
+
return new Set([...Object.keys(a || {}), ...Object.keys(b || {})]);
|
|
156
|
+
}
|
|
157
|
+
function structuredCloneSafe(o) {
|
|
158
|
+
return JSON.parse(JSON.stringify(o));
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// ---- view (derive a UI-friendly shape) -------------------------------------
|
|
162
|
+
|
|
163
|
+
export function viewDoc(doc) {
|
|
164
|
+
const tags = Object.entries(doc.tags || {})
|
|
165
|
+
.filter(([, t]) => t.present)
|
|
166
|
+
.map(([name, t]) => ({ name, value: t.value === true ? null : t.value }))
|
|
167
|
+
.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); // canonical order
|
|
168
|
+
|
|
169
|
+
const all = Object.values(doc.comments || {}).map((c) => ({
|
|
170
|
+
id: c.id, parentId: c.parentId, author: c.author,
|
|
171
|
+
body: c.deleted ? null : c.body?.text, deleted: !!c.deleted,
|
|
172
|
+
// Edited iff the body's Lamport stamp advanced past the comment's own.
|
|
173
|
+
edited: !!c.body && c.body.at !== c.at, createdAt: c.createdAt, at: c.at,
|
|
174
|
+
mentions: c.mentions || [],
|
|
175
|
+
reactions: summariseReactions(c.reactions),
|
|
176
|
+
}));
|
|
177
|
+
all.sort((x, y) => x.createdAt - y.createdAt || (x.id < y.id ? -1 : 1)); // stable, canonical
|
|
178
|
+
|
|
179
|
+
// Thread into a tree.
|
|
180
|
+
const byId = new Map(all.map((c) => [c.id, { ...c, replies: [] }]));
|
|
181
|
+
const roots = [];
|
|
182
|
+
for (const c of byId.values()) {
|
|
183
|
+
if (c.parentId && byId.has(c.parentId)) byId.get(c.parentId).replies.push(c);
|
|
184
|
+
else roots.push(c);
|
|
185
|
+
}
|
|
186
|
+
const facets = Object.fromEntries(Object.entries(doc.facets || {}).map(([k, v]) => [k, v.data]));
|
|
187
|
+
const subscribers = Object.entries(doc.subscribers || {}).filter(([, s]) => s.subscribed).map(([id]) => id).sort();
|
|
188
|
+
return { nodeId: doc.nodeId, tags, comments: roots, commentCount: all.filter((c) => !c.deleted).length, facets, subscribers };
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function summariseReactions(reactions = {}) {
|
|
192
|
+
const out = {};
|
|
193
|
+
for (const [emoji, users] of Object.entries(reactions)) {
|
|
194
|
+
const on = Object.entries(users).filter(([, r]) => r.on).map(([u]) => u);
|
|
195
|
+
if (on.length) out[emoji] = on;
|
|
196
|
+
}
|
|
197
|
+
return out;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// Extract mentioned user ids from a body. Supports rich `@[Name](id)` tokens
|
|
201
|
+
// (from a mention picker) and bare `@handle` tokens (handy when the identity is
|
|
202
|
+
// an email/handle and there's no user directory — BYO-IdP deployments).
|
|
203
|
+
export function extractMentions(body) {
|
|
204
|
+
const ids = new Set();
|
|
205
|
+
let text = String(body || '');
|
|
206
|
+
const rich = /@\[[^\]]+\]\(([^)]+)\)/g;
|
|
207
|
+
let m;
|
|
208
|
+
while ((m = rich.exec(text))) ids.add(m[1]);
|
|
209
|
+
text = text.replace(rich, ' '); // remove rich tokens before scanning bare ones
|
|
210
|
+
const bare = /(?:^|\s)@([a-zA-Z0-9._@-]{2,})/g;
|
|
211
|
+
while ((m = bare.exec(text))) ids.add(m[1].replace(/[.@-]+$/, ''));
|
|
212
|
+
return [...ids];
|
|
213
|
+
}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
// SidecarService — the API surface over sidecar documents: conversations
|
|
2
|
+
// (threaded comments, edits, reactions), tags, per-indexer facets, and thread
|
|
3
|
+
// subscriptions. It resolves the acting Principal into comment authorship,
|
|
4
|
+
// extracts @mentions, auto-subscribes participants, and emits mention events to
|
|
5
|
+
// a callback (wired to the notification batcher). Routes call these methods.
|
|
6
|
+
|
|
7
|
+
import { SidecarStore } from './store.js';
|
|
8
|
+
import { SidecarManager } from './manager.js';
|
|
9
|
+
import {
|
|
10
|
+
addComment, editComment, deleteComment, react, setTag, removeTag, setFacet,
|
|
11
|
+
subscribe, unsubscribe, viewDoc, extractMentions,
|
|
12
|
+
} from './document.js';
|
|
13
|
+
import { newId } from '../util.js';
|
|
14
|
+
import { TroveError } from '../errors.js';
|
|
15
|
+
|
|
16
|
+
export class SidecarService {
|
|
17
|
+
/**
|
|
18
|
+
* @param {object} deps
|
|
19
|
+
* @param {import('../storage/interface.js').StorageBackend} deps.storage
|
|
20
|
+
* @param {(mentions: Array) => void} [deps.onMentions] batch-notify sink
|
|
21
|
+
* @param {SidecarManager} [deps.manager]
|
|
22
|
+
* @param {import('../issues.js').IssueRegistry} [deps.issues] so a write-back that
|
|
23
|
+
* keeps failing becomes a standing problem rather than a console line
|
|
24
|
+
*/
|
|
25
|
+
constructor({ storage, onMentions, manager, issues } = {}) {
|
|
26
|
+
this.store = new SidecarStore({ storage });
|
|
27
|
+
this.manager = manager ?? new SidecarManager({ store: this.store, issues });
|
|
28
|
+
this.onMentions = onMentions ?? (() => {});
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async view(nodeId) {
|
|
32
|
+
return viewDoc(await this.manager.get(nodeId));
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Flush and evict idle documents — the maintenance timer's entry point.
|
|
37
|
+
*
|
|
38
|
+
* It called `sidecar.sweep?.()`, which did not exist here; the optional call swallowed
|
|
39
|
+
* it, so nothing ever evicted and every sidecar ever loaded (one per file opened,
|
|
40
|
+
* commented on, or tagged) stayed resident for the life of the process. Delegating
|
|
41
|
+
* makes the name the caller already uses the real one.
|
|
42
|
+
*/
|
|
43
|
+
sweep() {
|
|
44
|
+
return this.manager.sweep();
|
|
45
|
+
}
|
|
46
|
+
/** Write back everything still holding unsaved changes (shutdown). */
|
|
47
|
+
flushAll() {
|
|
48
|
+
return this.manager.flushAll();
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// --- conversation ----------------------------------------------------------
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* @param {string} nodeId
|
|
55
|
+
* @param {{ body: string, parentId?: string, mentions?: string[] }} input
|
|
56
|
+
* @param {object} principal { id, name, picture }
|
|
57
|
+
*/
|
|
58
|
+
async addComment(nodeId, input, principal) {
|
|
59
|
+
if (!principal?.id) throw TroveError.unauthorized('Sign in to comment');
|
|
60
|
+
if (!input.body?.trim()) throw TroveError.invalid('Comment body is required');
|
|
61
|
+
const id = newId('cmt');
|
|
62
|
+
const author = { id: principal.id, name: principal.name || principal.id, picture: principal.picture || null };
|
|
63
|
+
const mentions = uniq([...(input.mentions || []), ...extractMentions(input.body)]).filter((m) => m !== principal.id);
|
|
64
|
+
|
|
65
|
+
const comment = await this.manager.mutate(nodeId, (doc) => {
|
|
66
|
+
const c = addComment(doc, { id, parentId: input.parentId || null, author, body: input.body, mentions, actor: principal.id });
|
|
67
|
+
// Auto-subscribe mentioned users and the parent author to the thread.
|
|
68
|
+
for (const uid of mentions) subscribe(doc, uid, { actor: principal.id });
|
|
69
|
+
if (input.parentId && doc.comments[input.parentId]) subscribe(doc, doc.comments[input.parentId].author?.id, { actor: principal.id });
|
|
70
|
+
return c;
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
if (mentions.length) {
|
|
74
|
+
this.onMentions(mentions.map((userId) => ({
|
|
75
|
+
userId, nodeId, commentId: id, by: author, at: Date.now(),
|
|
76
|
+
excerpt: excerpt(input.body),
|
|
77
|
+
})));
|
|
78
|
+
}
|
|
79
|
+
return this.#commentView(nodeId, comment.id);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async editComment(nodeId, commentId, body, principal) {
|
|
83
|
+
await this.#assertAuthor(nodeId, commentId, principal);
|
|
84
|
+
await this.manager.mutate(nodeId, (doc) => editComment(doc, commentId, { body, actor: principal.id }));
|
|
85
|
+
return this.#commentView(nodeId, commentId);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async deleteComment(nodeId, commentId, principal) {
|
|
89
|
+
await this.#assertAuthor(nodeId, commentId, principal);
|
|
90
|
+
await this.manager.mutate(nodeId, (doc) => deleteComment(doc, commentId, { actor: principal.id }));
|
|
91
|
+
return { ok: true };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async react(nodeId, commentId, emoji, on, principal) {
|
|
95
|
+
if (!principal?.id) throw TroveError.unauthorized('Sign in to react');
|
|
96
|
+
await this.manager.mutate(nodeId, (doc) => react(doc, commentId, emoji, principal.id, on));
|
|
97
|
+
return this.#commentView(nodeId, commentId);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// --- tags ------------------------------------------------------------------
|
|
101
|
+
|
|
102
|
+
async setTag(nodeId, name, value, principal) {
|
|
103
|
+
await this.manager.mutate(nodeId, (doc) => setTag(doc, name, { value: value ?? true, actor: principal?.id }));
|
|
104
|
+
return this.view(nodeId);
|
|
105
|
+
}
|
|
106
|
+
async removeTag(nodeId, name, principal) {
|
|
107
|
+
await this.manager.mutate(nodeId, (doc) => removeTag(doc, name, { actor: principal?.id }));
|
|
108
|
+
return this.view(nodeId);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// --- facets (indexer-scoped) ----------------------------------------------
|
|
112
|
+
|
|
113
|
+
/** Write an indexer's facet into the sidecar, namespaced under indexerId. */
|
|
114
|
+
async setFacet(nodeId, indexerId, data) {
|
|
115
|
+
await this.manager.mutate(nodeId, (doc) => setFacet(doc, indexerId, data, { actor: indexerId }));
|
|
116
|
+
return { ok: true };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// --- subscriptions ---------------------------------------------------------
|
|
120
|
+
|
|
121
|
+
async subscribe(nodeId, principal, muted = false) {
|
|
122
|
+
if (!principal?.id) throw TroveError.unauthorized('Sign in to follow');
|
|
123
|
+
await this.manager.mutate(nodeId, (doc) => subscribe(doc, principal.id, { muted, actor: principal.id }));
|
|
124
|
+
return { ok: true };
|
|
125
|
+
}
|
|
126
|
+
async unsubscribe(nodeId, principal) {
|
|
127
|
+
if (!principal?.id) throw TroveError.unauthorized('Sign in');
|
|
128
|
+
await this.manager.mutate(nodeId, (doc) => unsubscribe(doc, principal.id, { actor: principal.id }));
|
|
129
|
+
return { ok: true };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Delete a file's sidecar (called when the file is deleted). */
|
|
133
|
+
async remove(nodeId) {
|
|
134
|
+
await this.store.remove(nodeId);
|
|
135
|
+
this.manager.hot.delete(nodeId);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async dispose() {
|
|
139
|
+
await this.manager.dispose();
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// --- helpers ---------------------------------------------------------------
|
|
143
|
+
|
|
144
|
+
async #assertAuthor(nodeId, commentId, principal) {
|
|
145
|
+
if (!principal?.id) throw TroveError.unauthorized('Sign in');
|
|
146
|
+
const doc = await this.manager.get(nodeId);
|
|
147
|
+
const c = doc.comments[commentId];
|
|
148
|
+
if (!c) throw TroveError.notFound('Comment');
|
|
149
|
+
if (c.author?.id !== principal.id) throw TroveError.forbidden('You can only edit your own comments');
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
async #commentView(nodeId, commentId) {
|
|
153
|
+
const view = viewDoc(await this.manager.get(nodeId));
|
|
154
|
+
const find = (list) => {
|
|
155
|
+
for (const c of list) {
|
|
156
|
+
if (c.id === commentId) return c;
|
|
157
|
+
const inner = find(c.replies || []);
|
|
158
|
+
if (inner) return inner;
|
|
159
|
+
}
|
|
160
|
+
return null;
|
|
161
|
+
};
|
|
162
|
+
return find(view.comments) || { id: commentId };
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function uniq(arr) {
|
|
167
|
+
return [...new Set(arr)];
|
|
168
|
+
}
|
|
169
|
+
function excerpt(body, n = 140) {
|
|
170
|
+
const clean = body.replace(/@\[([^\]]+)\]\([^)]+\)/g, '@$1').replace(/\s+/g, ' ').trim();
|
|
171
|
+
return clean.length > n ? clean.slice(0, n) + '…' : clean;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export { SidecarStore, SidecarManager };
|