@xnetjs/data-bridge 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/chunk-25WNZV7W.js +831 -0
- package/dist/chunk-5GTIP33X.js +201 -0
- package/dist/chunk-EPNW4GGU.js +460 -0
- package/dist/index.d.ts +259 -449
- package/dist/index.js +1335 -575
- package/dist/native-bridge.d.ts +126 -0
- package/dist/native-bridge.js +13 -0
- package/dist/query-descriptor-D0k2gUQ0.d.ts +298 -0
- package/dist/types-BRvuTwEn.d.ts +547 -0
- package/dist/types.d.ts +4 -0
- package/dist/types.js +0 -0
- package/dist/worker/data-worker.d.ts +161 -1
- package/dist/worker/data-worker.js +468 -144
- package/package.json +16 -6
- package/dist/chunk-X6F5CPJI.js +0 -386
|
@@ -1,57 +1,170 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
PortSQLiteAdapter,
|
|
3
|
+
encodeWorkerQuerySnapshot,
|
|
4
|
+
groupNodeChangeEventsBySchema
|
|
5
|
+
} from "../chunk-EPNW4GGU.js";
|
|
6
|
+
import {
|
|
7
|
+
applyNodeChangeToBoundedQueryResult,
|
|
8
|
+
applyNodeChangeToQueryResult,
|
|
9
|
+
createBoundedWorkingSet,
|
|
10
|
+
createBoundedWorkingSetDescriptor,
|
|
11
|
+
createQueryDescriptor,
|
|
12
|
+
queryDescriptorSupportsBoundedDelta,
|
|
13
|
+
reuseEquivalentNodeReferences
|
|
14
|
+
} from "../chunk-5GTIP33X.js";
|
|
4
15
|
|
|
5
16
|
// src/worker/data-worker.ts
|
|
17
|
+
import { expose } from "comlink";
|
|
18
|
+
|
|
19
|
+
// src/worker/data-worker-host.ts
|
|
6
20
|
import {
|
|
7
21
|
NodeStore,
|
|
8
|
-
MemoryNodeStorageAdapter
|
|
22
|
+
MemoryNodeStorageAdapter,
|
|
23
|
+
SQLiteNodeStorageAdapter
|
|
9
24
|
} from "@xnetjs/data";
|
|
10
|
-
import {
|
|
25
|
+
import { createWebCryptoChangeSigner } from "@xnetjs/sync";
|
|
26
|
+
import { proxy, transfer } from "comlink";
|
|
11
27
|
import * as Y from "yjs";
|
|
28
|
+
var BULK_STORE_CHANGE_RELOAD_THRESHOLD = 250;
|
|
12
29
|
var MAX_DOC_POOL_SIZE = 50;
|
|
13
30
|
var MIN_DOC_AGE_FOR_EVICTION = 6e4;
|
|
31
|
+
function mergePreviousNodeReferences(nextNodes, previousNodes, previousWorkingSet) {
|
|
32
|
+
return reuseEquivalentNodeReferences(nextNodes, [
|
|
33
|
+
...previousWorkingSet?.nodes ?? [],
|
|
34
|
+
...previousNodes ?? []
|
|
35
|
+
]);
|
|
36
|
+
}
|
|
37
|
+
function computeQueryDelta(previous, next) {
|
|
38
|
+
const previousById = new Map(previous.map((node) => [node.id, node]));
|
|
39
|
+
const nextIds = new Set(next.map((node) => node.id));
|
|
40
|
+
const added = next.filter((node) => !previousById.has(node.id));
|
|
41
|
+
const removed = previous.filter((node) => !nextIds.has(node.id));
|
|
42
|
+
if (added.length === 0 && removed.length === 0) {
|
|
43
|
+
const changed = next.filter((node) => previousById.get(node.id) !== node);
|
|
44
|
+
if (changed.length === 0) return null;
|
|
45
|
+
if (changed.length === 1) {
|
|
46
|
+
return { type: "update", nodeId: changed[0].id, node: changed[0] };
|
|
47
|
+
}
|
|
48
|
+
return { type: "reload", data: next };
|
|
49
|
+
}
|
|
50
|
+
if (added.length === 1 && removed.length === 0) {
|
|
51
|
+
const othersUntouched = next.every(
|
|
52
|
+
(node) => node.id === added[0].id || previousById.get(node.id) === node
|
|
53
|
+
);
|
|
54
|
+
if (othersUntouched) {
|
|
55
|
+
return {
|
|
56
|
+
type: "add",
|
|
57
|
+
node: added[0],
|
|
58
|
+
index: next.findIndex((node) => node.id === added[0].id)
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
return { type: "reload", data: next };
|
|
62
|
+
}
|
|
63
|
+
if (removed.length === 1 && added.length === 0) {
|
|
64
|
+
const othersUntouched = next.every((node) => previousById.get(node.id) === node);
|
|
65
|
+
if (othersUntouched) {
|
|
66
|
+
return { type: "remove", nodeId: removed[0].id };
|
|
67
|
+
}
|
|
68
|
+
return { type: "reload", data: next };
|
|
69
|
+
}
|
|
70
|
+
return { type: "reload", data: next };
|
|
71
|
+
}
|
|
14
72
|
var DataWorker = class {
|
|
15
73
|
store = null;
|
|
16
74
|
storage = null;
|
|
17
75
|
subscriptions = /* @__PURE__ */ new Map();
|
|
18
|
-
cache = new QueryCache();
|
|
19
76
|
status = "disconnected";
|
|
20
77
|
statusHandlers = /* @__PURE__ */ new Set();
|
|
78
|
+
changeFeedHandlers = /* @__PURE__ */ new Set();
|
|
21
79
|
storeUnsubscribe = null;
|
|
80
|
+
storeBatchUnsubscribe = null;
|
|
81
|
+
pendingStoreChanges = [];
|
|
82
|
+
storeChangeFlushQueued = false;
|
|
22
83
|
// Y.Doc pool - the "source of truth" for all documents
|
|
23
84
|
docPool = /* @__PURE__ */ new Map();
|
|
24
85
|
// Client ID counter for Y.Doc instances
|
|
25
86
|
nextClientId = Math.floor(Math.random() * 2147483647);
|
|
26
87
|
async initialize(config) {
|
|
27
|
-
this.storage =
|
|
88
|
+
this.storage = await this.createStorageAdapter(config);
|
|
89
|
+
const signingKey = new Uint8Array(config.signingKey);
|
|
28
90
|
this.store = new NodeStore({
|
|
29
91
|
storage: this.storage,
|
|
30
92
|
authorDID: config.authorDID,
|
|
31
|
-
signingKey
|
|
93
|
+
signingKey,
|
|
94
|
+
// Signing already runs off the main thread here, but WebCrypto keeps
|
|
95
|
+
// signature bursts (imports, transactions) from blocking the worker's
|
|
96
|
+
// own event loop — queries and deltas stay responsive. Byte-identical
|
|
97
|
+
// to the synchronous path; null when the runtime lacks SubtleCrypto.
|
|
98
|
+
changeSigner: createWebCryptoChangeSigner(signingKey) ?? void 0
|
|
32
99
|
});
|
|
33
100
|
await this.store.initialize();
|
|
34
101
|
this.storeUnsubscribe = this.store.subscribe((event) => {
|
|
35
|
-
this.
|
|
102
|
+
this.enqueueStoreChange(event);
|
|
103
|
+
this.emitChangeFeedEvent(event);
|
|
104
|
+
});
|
|
105
|
+
this.storeBatchUnsubscribe = this.store.subscribeToBatchChanges((event) => {
|
|
106
|
+
void this.handleStoreBatchChange(event);
|
|
36
107
|
});
|
|
37
108
|
this.setStatus("connected");
|
|
38
109
|
}
|
|
110
|
+
/**
|
|
111
|
+
* Create the worker's storage adapter.
|
|
112
|
+
*
|
|
113
|
+
* With a forwarded `storagePort`, persistence goes through the existing
|
|
114
|
+
* SQLite worker via PortSQLiteAdapter (worker-to-worker, no main-thread
|
|
115
|
+
* hop). Without one, storage is in-memory.
|
|
116
|
+
*/
|
|
117
|
+
async createStorageAdapter(config) {
|
|
118
|
+
if (config.storagePort) {
|
|
119
|
+
const portAdapter = new PortSQLiteAdapter(config.storagePort);
|
|
120
|
+
await portAdapter.open();
|
|
121
|
+
const storage = new SQLiteNodeStorageAdapter(portAdapter);
|
|
122
|
+
await storage.open();
|
|
123
|
+
return storage;
|
|
124
|
+
}
|
|
125
|
+
return new MemoryNodeStorageAdapter();
|
|
126
|
+
}
|
|
39
127
|
async subscribe(queryId, schemaId, options, onDelta) {
|
|
40
128
|
if (!this.store) {
|
|
41
129
|
throw new Error("DataWorker not initialized");
|
|
42
130
|
}
|
|
43
|
-
const
|
|
131
|
+
const descriptor = createQueryDescriptor(schemaId, options);
|
|
132
|
+
const existing = this.subscriptions.get(queryId);
|
|
133
|
+
const loaded = await this.loadQueryState(descriptor, existing?.lastResult ?? null);
|
|
44
134
|
this.subscriptions.set(queryId, {
|
|
45
135
|
schemaId,
|
|
136
|
+
descriptor,
|
|
46
137
|
options,
|
|
47
|
-
lastResult:
|
|
138
|
+
lastResult: loaded.visible,
|
|
139
|
+
workingSet: loaded.workingSet,
|
|
48
140
|
onDelta: proxy(onDelta)
|
|
49
141
|
});
|
|
50
|
-
return
|
|
142
|
+
return this.toWireSnapshot(loaded.visible);
|
|
51
143
|
}
|
|
52
144
|
async unsubscribe(queryId) {
|
|
53
145
|
this.subscriptions.delete(queryId);
|
|
54
146
|
}
|
|
147
|
+
async reloadQuery(queryId) {
|
|
148
|
+
const sub = this.subscriptions.get(queryId);
|
|
149
|
+
if (!sub) {
|
|
150
|
+
return { encoding: "json", nodes: [] };
|
|
151
|
+
}
|
|
152
|
+
const loaded = await this.loadQueryState(sub.descriptor, sub.lastResult, sub.workingSet);
|
|
153
|
+
sub.lastResult = loaded.visible;
|
|
154
|
+
sub.workingSet = loaded.workingSet;
|
|
155
|
+
return this.toWireSnapshot(loaded.visible);
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Encode a snapshot for the wire. Binary payloads ride a freshly
|
|
159
|
+
* allocated buffer, so it is transferred (zero-copy) instead of cloned.
|
|
160
|
+
*/
|
|
161
|
+
toWireSnapshot(nodes) {
|
|
162
|
+
const snapshot = encodeWorkerQuerySnapshot(nodes);
|
|
163
|
+
if (snapshot.encoding === "binary") {
|
|
164
|
+
return transfer(snapshot, [snapshot.data.buffer]);
|
|
165
|
+
}
|
|
166
|
+
return snapshot;
|
|
167
|
+
}
|
|
55
168
|
async create(schemaId, data, id) {
|
|
56
169
|
if (!this.store) {
|
|
57
170
|
throw new Error("DataWorker not initialized");
|
|
@@ -66,13 +179,33 @@ var DataWorker = class {
|
|
|
66
179
|
if (!this.store) {
|
|
67
180
|
throw new Error("DataWorker not initialized");
|
|
68
181
|
}
|
|
69
|
-
|
|
182
|
+
const revert = this.applyOptimisticNodeChange(nodeId, (node) => ({
|
|
183
|
+
...node,
|
|
184
|
+
properties: { ...node.properties, ...changes },
|
|
185
|
+
updatedAt: Date.now()
|
|
186
|
+
}));
|
|
187
|
+
try {
|
|
188
|
+
return await this.store.update(nodeId, { properties: changes });
|
|
189
|
+
} catch (err) {
|
|
190
|
+
await revert();
|
|
191
|
+
throw err;
|
|
192
|
+
}
|
|
70
193
|
}
|
|
71
194
|
async delete(nodeId) {
|
|
72
195
|
if (!this.store) {
|
|
73
196
|
throw new Error("DataWorker not initialized");
|
|
74
197
|
}
|
|
75
|
-
|
|
198
|
+
const revert = this.applyOptimisticNodeChange(nodeId, (node) => ({
|
|
199
|
+
...node,
|
|
200
|
+
deleted: true,
|
|
201
|
+
updatedAt: Date.now()
|
|
202
|
+
}));
|
|
203
|
+
try {
|
|
204
|
+
await this.store.delete(nodeId);
|
|
205
|
+
} catch (err) {
|
|
206
|
+
await revert();
|
|
207
|
+
throw err;
|
|
208
|
+
}
|
|
76
209
|
}
|
|
77
210
|
async restore(nodeId) {
|
|
78
211
|
if (!this.store) {
|
|
@@ -80,6 +213,19 @@ var DataWorker = class {
|
|
|
80
213
|
}
|
|
81
214
|
return this.store.restore(nodeId);
|
|
82
215
|
}
|
|
216
|
+
async bulkWrite(input) {
|
|
217
|
+
if (!this.store) {
|
|
218
|
+
throw new Error("DataWorker not initialized");
|
|
219
|
+
}
|
|
220
|
+
return this.store.batchWrite(input);
|
|
221
|
+
}
|
|
222
|
+
async transaction(operations) {
|
|
223
|
+
if (!this.store) {
|
|
224
|
+
throw new Error("DataWorker not initialized");
|
|
225
|
+
}
|
|
226
|
+
const tx = await this.store.transaction(operations);
|
|
227
|
+
return { batchId: tx.batchId, results: tx.results, tempIds: tx.tempIds };
|
|
228
|
+
}
|
|
83
229
|
async get(nodeId) {
|
|
84
230
|
if (!this.store) {
|
|
85
231
|
throw new Error("DataWorker not initialized");
|
|
@@ -91,51 +237,8 @@ var DataWorker = class {
|
|
|
91
237
|
if (!this.storage) {
|
|
92
238
|
throw new Error("DataWorker not initialized");
|
|
93
239
|
}
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
entry.lastAccessed = Date.now();
|
|
97
|
-
} else {
|
|
98
|
-
const doc = new Y.Doc({ guid: nodeId, gc: false });
|
|
99
|
-
const storedContent = await this.storage.getDocumentContent(nodeId);
|
|
100
|
-
if (storedContent && storedContent.length > 0) {
|
|
101
|
-
Y.applyUpdate(doc, storedContent, "storage");
|
|
102
|
-
}
|
|
103
|
-
entry = {
|
|
104
|
-
doc,
|
|
105
|
-
refCount: 0,
|
|
106
|
-
updateHandlers: /* @__PURE__ */ new Set(),
|
|
107
|
-
lastAccessed: Date.now()
|
|
108
|
-
};
|
|
109
|
-
this.evictOldDocs();
|
|
110
|
-
doc.on("update", (update, origin) => {
|
|
111
|
-
const content = Y.encodeStateAsUpdate(doc);
|
|
112
|
-
this.storage?.setDocumentContent(nodeId, content).catch((err) => {
|
|
113
|
-
console.error("[DataWorker] Failed to persist doc:", err);
|
|
114
|
-
});
|
|
115
|
-
if (origin === "remote") {
|
|
116
|
-
const handlers = Array.from(entry.updateHandlers);
|
|
117
|
-
for (let i = 0; i < handlers.length; i++) {
|
|
118
|
-
try {
|
|
119
|
-
const isLast = i === handlers.length - 1;
|
|
120
|
-
const updateToSend = isLast ? update : new Uint8Array(update);
|
|
121
|
-
if (isLast && update.buffer.byteLength === update.byteLength) {
|
|
122
|
-
handlers[i](
|
|
123
|
-
transfer(updateToSend, [updateToSend.buffer]),
|
|
124
|
-
"remote"
|
|
125
|
-
);
|
|
126
|
-
} else {
|
|
127
|
-
handlers[i](updateToSend, "remote");
|
|
128
|
-
}
|
|
129
|
-
} catch (err) {
|
|
130
|
-
console.error("[DataWorker] Update handler error:", err);
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
});
|
|
135
|
-
this.docPool.set(nodeId, entry);
|
|
136
|
-
}
|
|
137
|
-
const proxiedHandler = proxy(onUpdate);
|
|
138
|
-
entry.updateHandlers.add(proxiedHandler);
|
|
240
|
+
const entry = await this.acquireDocEntry(nodeId);
|
|
241
|
+
entry.updateHandlers.add(proxy(onUpdate));
|
|
139
242
|
entry.refCount++;
|
|
140
243
|
const state = Y.encodeStateAsUpdate(entry.doc);
|
|
141
244
|
return transfer(
|
|
@@ -147,6 +250,61 @@ var DataWorker = class {
|
|
|
147
250
|
[state.buffer]
|
|
148
251
|
);
|
|
149
252
|
}
|
|
253
|
+
async acquireDocEntry(nodeId) {
|
|
254
|
+
const existing = this.docPool.get(nodeId);
|
|
255
|
+
if (existing) {
|
|
256
|
+
existing.lastAccessed = Date.now();
|
|
257
|
+
return existing;
|
|
258
|
+
}
|
|
259
|
+
return this.createDocEntry(nodeId);
|
|
260
|
+
}
|
|
261
|
+
async createDocEntry(nodeId) {
|
|
262
|
+
const doc = new Y.Doc({ guid: nodeId, gc: false });
|
|
263
|
+
const storedContent = await this.storage.getDocumentContent(nodeId);
|
|
264
|
+
if (storedContent && storedContent.length > 0) {
|
|
265
|
+
Y.applyUpdate(doc, storedContent, "storage");
|
|
266
|
+
}
|
|
267
|
+
const entry = {
|
|
268
|
+
doc,
|
|
269
|
+
refCount: 0,
|
|
270
|
+
updateHandlers: /* @__PURE__ */ new Set(),
|
|
271
|
+
lastAccessed: Date.now()
|
|
272
|
+
};
|
|
273
|
+
this.evictOldDocs();
|
|
274
|
+
doc.on("update", (update, origin) => {
|
|
275
|
+
this.handleDocUpdate(nodeId, entry, update, origin);
|
|
276
|
+
});
|
|
277
|
+
this.docPool.set(nodeId, entry);
|
|
278
|
+
return entry;
|
|
279
|
+
}
|
|
280
|
+
handleDocUpdate(nodeId, entry, update, origin) {
|
|
281
|
+
this.persistDocState(nodeId, entry.doc);
|
|
282
|
+
if (origin !== "remote") return;
|
|
283
|
+
this.forwardRemoteDocUpdate(entry, update);
|
|
284
|
+
}
|
|
285
|
+
persistDocState(nodeId, doc) {
|
|
286
|
+
const content = Y.encodeStateAsUpdate(doc);
|
|
287
|
+
this.storage?.setDocumentContent(nodeId, content).catch((err) => {
|
|
288
|
+
console.error("[DataWorker] Failed to persist doc:", err);
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
forwardRemoteDocUpdate(entry, update) {
|
|
292
|
+
const handlers = Array.from(entry.updateHandlers);
|
|
293
|
+
for (let i = 0; i < handlers.length; i++) {
|
|
294
|
+
try {
|
|
295
|
+
this.sendDocUpdate(handlers[i], update, i === handlers.length - 1);
|
|
296
|
+
} catch (err) {
|
|
297
|
+
console.error("[DataWorker] Update handler error:", err);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
sendDocUpdate(handler, update, isLast) {
|
|
302
|
+
if (isLast && update.buffer.byteLength === update.byteLength) {
|
|
303
|
+
handler(transfer(update, [update.buffer]), "remote");
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
handler(isLast ? update : new Uint8Array(update), "remote");
|
|
307
|
+
}
|
|
150
308
|
releaseDoc(nodeId) {
|
|
151
309
|
const entry = this.docPool.get(nodeId);
|
|
152
310
|
if (!entry) return;
|
|
@@ -167,102 +325,256 @@ var DataWorker = class {
|
|
|
167
325
|
onStatusChange(handler) {
|
|
168
326
|
this.statusHandlers.add(proxy(handler));
|
|
169
327
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
328
|
+
/**
|
|
329
|
+
* Subscribe to the worker's raw store change feed (devtools and other
|
|
330
|
+
* instrumentation). Events are structured-clone-safe NodeChangeEvents.
|
|
331
|
+
* The bridge registers a single forwarder and fans out locally, so the
|
|
332
|
+
* worker keeps at most one handler per bridge.
|
|
333
|
+
*/
|
|
334
|
+
subscribeToChanges(handler) {
|
|
335
|
+
this.changeFeedHandlers.add(proxy(handler));
|
|
336
|
+
}
|
|
337
|
+
emitChangeFeedEvent(event) {
|
|
338
|
+
for (const handler of this.changeFeedHandlers) {
|
|
339
|
+
try {
|
|
340
|
+
handler(event);
|
|
341
|
+
} catch (err) {
|
|
342
|
+
console.error("[DataWorker] Change feed handler error:", err);
|
|
343
|
+
}
|
|
174
344
|
}
|
|
345
|
+
}
|
|
346
|
+
async destroy() {
|
|
347
|
+
this.detachStoreListeners();
|
|
175
348
|
for (const entry of this.docPool.values()) {
|
|
176
349
|
entry.doc.destroy();
|
|
177
350
|
}
|
|
178
351
|
this.docPool.clear();
|
|
179
|
-
|
|
180
|
-
await this.storage.close();
|
|
181
|
-
}
|
|
182
|
-
this.storage = null;
|
|
352
|
+
await this.closeStorage();
|
|
183
353
|
this.store = null;
|
|
184
354
|
this.subscriptions.clear();
|
|
185
|
-
this.cache.clear();
|
|
186
355
|
this.statusHandlers.clear();
|
|
356
|
+
this.changeFeedHandlers.clear();
|
|
187
357
|
this.setStatus("disconnected");
|
|
188
358
|
}
|
|
359
|
+
detachStoreListeners() {
|
|
360
|
+
this.storeUnsubscribe?.();
|
|
361
|
+
this.storeUnsubscribe = null;
|
|
362
|
+
this.storeBatchUnsubscribe?.();
|
|
363
|
+
this.storeBatchUnsubscribe = null;
|
|
364
|
+
}
|
|
365
|
+
async closeStorage() {
|
|
366
|
+
if (this.storage?.close) {
|
|
367
|
+
await this.storage.close();
|
|
368
|
+
}
|
|
369
|
+
this.storage = null;
|
|
370
|
+
}
|
|
189
371
|
// ─── Private Methods ─────────────────────────────────────────────────────────
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
372
|
+
/**
|
|
373
|
+
* Execute a subscription's query against storage. Bounded descriptors
|
|
374
|
+
* overfetch a small buffer so later node changes can be applied in
|
|
375
|
+
* memory, and re-queries graft previous node references back in wherever
|
|
376
|
+
* the snapshots are equivalent (so reference-based delta math and
|
|
377
|
+
* downstream identity caches keep working).
|
|
378
|
+
*/
|
|
379
|
+
async loadQueryState(descriptor, previousNodes, previousWorkingSet) {
|
|
380
|
+
if (!this.store) {
|
|
381
|
+
return { visible: [], workingSet: null };
|
|
382
|
+
}
|
|
383
|
+
if (!queryDescriptorSupportsBoundedDelta(descriptor)) {
|
|
384
|
+
const result2 = await this.store.query(descriptor);
|
|
385
|
+
const merged2 = mergePreviousNodeReferences(result2.nodes, previousNodes, previousWorkingSet);
|
|
386
|
+
return { visible: merged2, workingSet: null };
|
|
387
|
+
}
|
|
388
|
+
const result = await this.store.query(createBoundedWorkingSetDescriptor(descriptor));
|
|
389
|
+
const merged = mergePreviousNodeReferences(result.nodes, previousNodes, previousWorkingSet);
|
|
390
|
+
return {
|
|
391
|
+
visible: merged.slice(0, descriptor.limit),
|
|
392
|
+
workingSet: createBoundedWorkingSet(descriptor, merged)
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
async reloadSubscription(sub) {
|
|
396
|
+
const loaded = await this.loadQueryState(sub.descriptor, sub.lastResult, sub.workingSet);
|
|
397
|
+
sub.lastResult = loaded.visible;
|
|
398
|
+
sub.workingSet = loaded.workingSet;
|
|
399
|
+
sub.onDelta({ type: "reload", data: loaded.visible });
|
|
400
|
+
}
|
|
401
|
+
enqueueStoreChange(event) {
|
|
402
|
+
this.pendingStoreChanges.push(event);
|
|
403
|
+
if (this.storeChangeFlushQueued) {
|
|
404
|
+
return;
|
|
203
405
|
}
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
const
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
406
|
+
this.storeChangeFlushQueued = true;
|
|
407
|
+
queueMicrotask(() => {
|
|
408
|
+
void this.flushStoreChanges();
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
async flushStoreChanges() {
|
|
412
|
+
const events = this.pendingStoreChanges;
|
|
413
|
+
this.pendingStoreChanges = [];
|
|
414
|
+
this.storeChangeFlushQueued = false;
|
|
415
|
+
if (events.length === 0) {
|
|
416
|
+
return;
|
|
417
|
+
}
|
|
418
|
+
await this.handleStoreChangeSet(events);
|
|
419
|
+
}
|
|
420
|
+
isBulkStoreChangeSet(events) {
|
|
421
|
+
return events.length > BULK_STORE_CHANGE_RELOAD_THRESHOLD || events.some((event) => (event.change.batchSize ?? 1) > BULK_STORE_CHANGE_RELOAD_THRESHOLD);
|
|
422
|
+
}
|
|
423
|
+
*subscriptionsForSchema(schemaId) {
|
|
424
|
+
for (const sub of this.subscriptions.values()) {
|
|
425
|
+
if (sub.schemaId === schemaId) yield sub;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
async handleStoreChangeSet(events) {
|
|
429
|
+
const eventsBySchema = groupNodeChangeEventsBySchema(events);
|
|
430
|
+
for (const [schemaId, schemaEvents] of eventsBySchema) {
|
|
431
|
+
const shouldReload = this.isBulkStoreChangeSet(schemaEvents);
|
|
432
|
+
const changes = schemaEvents.map((event) => ({
|
|
433
|
+
nodeId: event.change.payload.nodeId,
|
|
434
|
+
nextNode: event.node ?? null
|
|
435
|
+
}));
|
|
436
|
+
for (const sub of this.subscriptionsForSchema(schemaId)) {
|
|
437
|
+
if (shouldReload) {
|
|
438
|
+
await this.reloadSubscription(sub);
|
|
439
|
+
continue;
|
|
440
|
+
}
|
|
441
|
+
await this.applyChangesToSubscription(sub, changes);
|
|
218
442
|
}
|
|
219
443
|
}
|
|
220
444
|
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
445
|
+
/**
|
|
446
|
+
* Batch notifications carry node ids only. Small batches hydrate the
|
|
447
|
+
* touched nodes once and flow through the same delta path as regular
|
|
448
|
+
* change events; only genuinely bulk batches re-query each subscription.
|
|
449
|
+
*/
|
|
450
|
+
async handleStoreBatchChange(event) {
|
|
451
|
+
if (event.nodeIds.length > BULK_STORE_CHANGE_RELOAD_THRESHOLD) {
|
|
452
|
+
await this.reloadSubscriptionsForSchemas(event.schemaIds);
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
455
|
+
try {
|
|
456
|
+
await this.applyStoreBatchChangeDeltas(event);
|
|
457
|
+
} catch (err) {
|
|
458
|
+
console.error("[DataWorker] Failed to apply batch change deltas:", err);
|
|
459
|
+
await this.reloadSubscriptionsForSchemas(event.schemaIds);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
async reloadSubscriptionsForSchemas(schemaIds) {
|
|
463
|
+
for (const schemaId of schemaIds) {
|
|
464
|
+
for (const sub of this.subscriptionsForSchema(schemaId)) {
|
|
465
|
+
await this.reloadSubscription(sub);
|
|
230
466
|
}
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
async applyStoreBatchChangeDeltas(event) {
|
|
470
|
+
if (!this.store) return;
|
|
471
|
+
const nodes = await Promise.all(event.nodeIds.map((nodeId) => this.store.get(nodeId)));
|
|
472
|
+
const changes = event.nodeIds.map((nodeId, index) => ({
|
|
473
|
+
nodeId,
|
|
474
|
+
nextNode: nodes[index]
|
|
475
|
+
}));
|
|
476
|
+
for (const schemaId of event.schemaIds) {
|
|
477
|
+
for (const sub of this.subscriptionsForSchema(schemaId)) {
|
|
478
|
+
await this.applyChangesToSubscription(sub, changes);
|
|
234
479
|
}
|
|
235
480
|
}
|
|
236
|
-
return null;
|
|
237
481
|
}
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
482
|
+
/**
|
|
483
|
+
* Apply a list of node changes to one subscription, falling back to a
|
|
484
|
+
* storage re-query only when a delta is ambiguous. Emits at most one
|
|
485
|
+
* wire delta per change.
|
|
486
|
+
*/
|
|
487
|
+
async applyChangesToSubscription(sub, changes, options) {
|
|
488
|
+
const skipAmbiguous = options?.onAmbiguous === "skip";
|
|
489
|
+
for (const change of changes) {
|
|
490
|
+
const handled = await this.applyChangeAndEmit(sub, change, skipAmbiguous);
|
|
491
|
+
if (!handled) return;
|
|
241
492
|
}
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
493
|
+
}
|
|
494
|
+
/**
|
|
495
|
+
* Apply one change to a subscription and emit its wire delta.
|
|
496
|
+
* Returns false when the change was ambiguous (the subscription was
|
|
497
|
+
* reloaded — or skipped — wholesale, so remaining changes are moot).
|
|
498
|
+
*/
|
|
499
|
+
async applyChangeAndEmit(sub, change, skipAmbiguous) {
|
|
500
|
+
const applied = this.applyChangeToSubscriptionState(sub, change.nodeId, change.nextNode);
|
|
501
|
+
if (applied.kind === "reload") {
|
|
502
|
+
if (!skipAmbiguous) {
|
|
503
|
+
await this.reloadSubscription(sub);
|
|
247
504
|
}
|
|
505
|
+
return false;
|
|
506
|
+
}
|
|
507
|
+
if (applied.kind === "ok") {
|
|
508
|
+
this.emitSubscriptionDelta(sub, applied);
|
|
248
509
|
}
|
|
249
510
|
return true;
|
|
250
511
|
}
|
|
251
|
-
|
|
252
|
-
const
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
break;
|
|
258
|
-
case "remove":
|
|
259
|
-
sub.lastResult = sub.lastResult.filter((n) => n.id !== delta.nodeId);
|
|
260
|
-
break;
|
|
261
|
-
case "update":
|
|
262
|
-
sub.lastResult = sub.lastResult.map((n) => n.id === delta.nodeId ? delta.node : n);
|
|
263
|
-
break;
|
|
512
|
+
emitSubscriptionDelta(sub, applied) {
|
|
513
|
+
const delta = computeQueryDelta(sub.lastResult, applied.data);
|
|
514
|
+
sub.lastResult = applied.data;
|
|
515
|
+
sub.workingSet = applied.workingSet;
|
|
516
|
+
if (delta) {
|
|
517
|
+
sub.onDelta(delta);
|
|
264
518
|
}
|
|
265
519
|
}
|
|
520
|
+
/**
|
|
521
|
+
* Find the freshest cached snapshot of a node across all subscriptions.
|
|
522
|
+
*/
|
|
523
|
+
findCachedNode(nodeId) {
|
|
524
|
+
for (const sub of this.subscriptions.values()) {
|
|
525
|
+
const fromWorkingSet = sub.workingSet?.nodes.find((node) => node.id === nodeId);
|
|
526
|
+
if (fromWorkingSet) return fromWorkingSet;
|
|
527
|
+
const fromData = sub.lastResult.find((node) => node.id === nodeId);
|
|
528
|
+
if (fromData) return fromData;
|
|
529
|
+
}
|
|
530
|
+
return null;
|
|
531
|
+
}
|
|
532
|
+
/**
|
|
533
|
+
* Synchronously apply an optimistic node mutation to every affected
|
|
534
|
+
* subscription before persistence, so the main thread sees the edit one
|
|
535
|
+
* postMessage later (~next microtask) instead of after storage commits.
|
|
536
|
+
* Ambiguous deltas are skipped, not reloaded — storage still holds the
|
|
537
|
+
* OLD state, and the durable change event that follows reconciles.
|
|
538
|
+
* Returns a revert function that re-queries authoritative state (used
|
|
539
|
+
* when persistence fails).
|
|
540
|
+
*/
|
|
541
|
+
applyOptimisticNodeChange(nodeId, mutate) {
|
|
542
|
+
const current = this.findCachedNode(nodeId);
|
|
543
|
+
if (!current) {
|
|
544
|
+
return async () => {
|
|
545
|
+
};
|
|
546
|
+
}
|
|
547
|
+
const nextNode = mutate(current);
|
|
548
|
+
const changes = [{ nodeId, nextNode }];
|
|
549
|
+
for (const sub of this.subscriptions.values()) {
|
|
550
|
+
if (sub.schemaId !== current.schemaId) continue;
|
|
551
|
+
void this.applyChangesToSubscription(sub, changes, { onAmbiguous: "skip" });
|
|
552
|
+
}
|
|
553
|
+
return async () => {
|
|
554
|
+
await this.reloadSubscriptionsForSchemas([current.schemaId]);
|
|
555
|
+
};
|
|
556
|
+
}
|
|
557
|
+
applyChangeToSubscriptionState(sub, nodeId, nextNode) {
|
|
558
|
+
if (sub.workingSet && queryDescriptorSupportsBoundedDelta(sub.descriptor)) {
|
|
559
|
+
return this.applyBoundedChange(sub.descriptor, sub.workingSet, nodeId, nextNode);
|
|
560
|
+
}
|
|
561
|
+
return this.applyUnboundedChange(sub, nodeId, nextNode);
|
|
562
|
+
}
|
|
563
|
+
applyBoundedChange(descriptor, workingSet, nodeId, nextNode) {
|
|
564
|
+
const delta = applyNodeChangeToBoundedQueryResult({ descriptor, workingSet, nodeId, nextNode });
|
|
565
|
+
if (delta.kind !== "set") return { kind: delta.kind };
|
|
566
|
+
return { kind: "ok", data: delta.data, workingSet: delta.workingSet };
|
|
567
|
+
}
|
|
568
|
+
applyUnboundedChange(sub, nodeId, nextNode) {
|
|
569
|
+
const delta = applyNodeChangeToQueryResult({
|
|
570
|
+
descriptor: sub.descriptor,
|
|
571
|
+
currentData: sub.lastResult,
|
|
572
|
+
nodeId,
|
|
573
|
+
nextNode
|
|
574
|
+
});
|
|
575
|
+
if (delta.kind !== "set") return { kind: delta.kind };
|
|
576
|
+
return { kind: "ok", data: delta.data, workingSet: null };
|
|
577
|
+
}
|
|
266
578
|
setStatus(status) {
|
|
267
579
|
this.status = status;
|
|
268
580
|
for (const handler of this.statusHandlers) {
|
|
@@ -275,6 +587,18 @@ var DataWorker = class {
|
|
|
275
587
|
*/
|
|
276
588
|
evictOldDocs() {
|
|
277
589
|
if (this.docPool.size < MAX_DOC_POOL_SIZE) return;
|
|
590
|
+
const candidates = this.collectDocEvictionCandidates();
|
|
591
|
+
const targetSize = Math.floor(MAX_DOC_POOL_SIZE * 0.8);
|
|
592
|
+
const toEvict = Math.min(this.docPool.size - targetSize, candidates.length);
|
|
593
|
+
for (let i = 0; i < toEvict; i++) {
|
|
594
|
+
this.evictDoc(candidates[i].nodeId);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
/**
|
|
598
|
+
* Find eviction candidates — docs with no refs and old enough — ordered
|
|
599
|
+
* oldest-accessed first.
|
|
600
|
+
*/
|
|
601
|
+
collectDocEvictionCandidates() {
|
|
278
602
|
const now = Date.now();
|
|
279
603
|
const candidates = [];
|
|
280
604
|
for (const [nodeId, entry] of this.docPool) {
|
|
@@ -282,23 +606,23 @@ var DataWorker = class {
|
|
|
282
606
|
candidates.push({ nodeId, lastAccessed: entry.lastAccessed });
|
|
283
607
|
}
|
|
284
608
|
}
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
const
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
const
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
const content = Y.encodeStateAsUpdate(entry.doc);
|
|
295
|
-
this.storage.setDocumentContent(nodeId, content).catch(() => {
|
|
296
|
-
});
|
|
297
|
-
}
|
|
298
|
-
entry.doc.destroy();
|
|
299
|
-
this.docPool.delete(nodeId);
|
|
300
|
-
}
|
|
609
|
+
return candidates.sort((a, b) => a.lastAccessed - b.lastAccessed);
|
|
610
|
+
}
|
|
611
|
+
evictDoc(nodeId) {
|
|
612
|
+
const entry = this.docPool.get(nodeId);
|
|
613
|
+
if (!entry) return;
|
|
614
|
+
if (this.storage) {
|
|
615
|
+
const content = Y.encodeStateAsUpdate(entry.doc);
|
|
616
|
+
this.storage.setDocumentContent(nodeId, content).catch(() => {
|
|
617
|
+
});
|
|
301
618
|
}
|
|
619
|
+
entry.doc.destroy();
|
|
620
|
+
this.docPool.delete(nodeId);
|
|
302
621
|
}
|
|
303
622
|
};
|
|
623
|
+
|
|
624
|
+
// src/worker/data-worker.ts
|
|
304
625
|
expose(new DataWorker());
|
|
626
|
+
export {
|
|
627
|
+
DataWorker
|
|
628
|
+
};
|