cojson-storage-indexeddb 0.8.11 → 0.8.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -1,781 +1,723 @@
1
1
  import {
2
- cojsonInternals,
3
- SessionID,
4
- SyncMessage,
5
- Peer,
6
- CojsonInternalTypes,
7
- MAX_RECOMMENDED_TX_SIZE,
8
- RawAccountID,
9
- IncomingSyncStream,
10
- OutgoingSyncQueue,
2
+ CojsonInternalTypes,
3
+ IncomingSyncStream,
4
+ MAX_RECOMMENDED_TX_SIZE,
5
+ OutgoingSyncQueue,
6
+ Peer,
7
+ RawAccountID,
8
+ SessionID,
9
+ SyncMessage,
10
+ cojsonInternals,
11
11
  } from "cojson";
12
12
  import { SyncPromise } from "./syncPromises.js";
13
13
 
14
14
  type CoValueRow = {
15
- id: CojsonInternalTypes.RawCoID;
16
- header: CojsonInternalTypes.CoValueHeader;
15
+ id: CojsonInternalTypes.RawCoID;
16
+ header: CojsonInternalTypes.CoValueHeader;
17
17
  };
18
18
 
19
19
  type StoredCoValueRow = CoValueRow & { rowID: number };
20
20
 
21
21
  type SessionRow = {
22
- coValue: number;
23
- sessionID: SessionID;
24
- lastIdx: number;
25
- lastSignature: CojsonInternalTypes.Signature;
26
- bytesSinceLastSignature?: number;
22
+ coValue: number;
23
+ sessionID: SessionID;
24
+ lastIdx: number;
25
+ lastSignature: CojsonInternalTypes.Signature;
26
+ bytesSinceLastSignature?: number;
27
27
  };
28
28
 
29
29
  type StoredSessionRow = SessionRow & { rowID: number };
30
30
 
31
31
  type TransactionRow = {
32
- ses: number;
33
- idx: number;
34
- tx: CojsonInternalTypes.Transaction;
32
+ ses: number;
33
+ idx: number;
34
+ tx: CojsonInternalTypes.Transaction;
35
35
  };
36
36
 
37
37
  type SignatureAfterRow = {
38
- ses: number;
39
- idx: number;
40
- signature: CojsonInternalTypes.Signature;
38
+ ses: number;
39
+ idx: number;
40
+ signature: CojsonInternalTypes.Signature;
41
41
  };
42
42
 
43
43
  export class IDBStorage {
44
- db: IDBDatabase;
45
- toLocalNode: OutgoingSyncQueue;
46
-
47
- constructor(
48
- db: IDBDatabase,
49
- fromLocalNode: IncomingSyncStream,
50
- toLocalNode: OutgoingSyncQueue,
51
- ) {
52
- this.db = db;
53
- this.toLocalNode = toLocalNode;
54
-
55
- const processMessages = async () => {
56
- for await (const msg of fromLocalNode) {
57
- try {
58
- if (msg === "Disconnected" || msg === "PingTimeout") {
59
- throw new Error("Unexpected Disconnected message");
60
- }
61
- await this.handleSyncMessage(msg);
62
- } catch (e) {
63
- console.error(
64
- new Error(
65
- `Error reading from localNode, handling msg\n\n${JSON.stringify(
66
- msg,
67
- (k, v) =>
68
- k === "changes" || k === "encryptedChanges"
69
- ? v.slice(0, 20) + "..."
70
- : v,
71
- )}`,
72
- { cause: e },
73
- ),
74
- );
75
- }
76
- }
77
- };
78
-
79
- processMessages().catch((e) =>
80
- console.error("Error in processMessages in IndexedDB", e),
81
- );
82
- }
83
-
84
- static async asPeer(
85
- {
86
- trace,
87
- localNodeName = "local",
88
- }: { trace?: boolean; localNodeName?: string } | undefined = {
89
- localNodeName: "local",
90
- },
91
- ): Promise<Peer> {
92
- const [localNodeAsPeer, storageAsPeer] = cojsonInternals.connectedPeers(
93
- localNodeName,
94
- "storage",
95
- {
96
- peer1role: "client",
97
- peer2role: "server",
98
- trace,
99
- crashOnClose: true,
100
- },
101
- );
102
-
103
- await IDBStorage.open(
104
- localNodeAsPeer.incoming,
105
- localNodeAsPeer.outgoing,
106
- );
107
-
108
- return { ...storageAsPeer, priority: 100 };
109
- }
110
-
111
- static async open(
112
- fromLocalNode: IncomingSyncStream,
113
- toLocalNode: OutgoingSyncQueue,
114
- ) {
115
- const dbPromise = new Promise<IDBDatabase>((resolve, reject) => {
116
- const request = indexedDB.open("jazz-storage", 4);
117
- request.onerror = () => {
118
- reject(request.error);
119
- };
120
- request.onsuccess = () => {
121
- resolve(request.result);
122
- };
123
- request.onupgradeneeded = async (ev) => {
124
- const db = request.result;
125
- if (ev.oldVersion === 0) {
126
- const coValues = db.createObjectStore("coValues", {
127
- autoIncrement: true,
128
- keyPath: "rowID",
129
- });
130
-
131
- coValues.createIndex("coValuesById", "id", {
132
- unique: true,
133
- });
134
-
135
- const sessions = db.createObjectStore("sessions", {
136
- autoIncrement: true,
137
- keyPath: "rowID",
138
- });
139
-
140
- sessions.createIndex("sessionsByCoValue", "coValue");
141
- sessions.createIndex(
142
- "uniqueSessions",
143
- ["coValue", "sessionID"],
144
- {
145
- unique: true,
146
- },
147
- );
148
-
149
- db.createObjectStore("transactions", {
150
- keyPath: ["ses", "idx"],
151
- });
152
- }
153
- if (ev.oldVersion <= 1) {
154
- db.createObjectStore("signatureAfter", {
155
- keyPath: ["ses", "idx"],
156
- });
157
- }
158
- };
159
- });
160
-
161
- return new IDBStorage(await dbPromise, fromLocalNode, toLocalNode);
162
- }
163
-
164
- async handleSyncMessage(msg: SyncMessage) {
165
- switch (msg.action) {
166
- case "load":
167
- await this.handleLoad(msg);
168
- break;
169
- case "content":
170
- await this.handleContent(msg);
171
- break;
172
- case "known":
173
- await this.handleKnown(msg);
174
- break;
175
- case "done":
176
- await this.handleDone(msg);
177
- break;
44
+ db: IDBDatabase;
45
+ toLocalNode: OutgoingSyncQueue;
46
+
47
+ constructor(
48
+ db: IDBDatabase,
49
+ fromLocalNode: IncomingSyncStream,
50
+ toLocalNode: OutgoingSyncQueue,
51
+ ) {
52
+ this.db = db;
53
+ this.toLocalNode = toLocalNode;
54
+
55
+ const processMessages = async () => {
56
+ for await (const msg of fromLocalNode) {
57
+ try {
58
+ if (msg === "Disconnected" || msg === "PingTimeout") {
59
+ throw new Error("Unexpected Disconnected message");
60
+ }
61
+ await this.handleSyncMessage(msg);
62
+ } catch (e) {
63
+ console.error(
64
+ new Error(
65
+ `Error reading from localNode, handling msg\n\n${JSON.stringify(
66
+ msg,
67
+ (k, v) =>
68
+ k === "changes" || k === "encryptedChanges"
69
+ ? v.slice(0, 20) + "..."
70
+ : v,
71
+ )}`,
72
+ { cause: e },
73
+ ),
74
+ );
75
+ }
76
+ }
77
+ };
78
+
79
+ processMessages().catch((e) =>
80
+ console.error("Error in processMessages in IndexedDB", e),
81
+ );
82
+ }
83
+
84
+ static async asPeer(
85
+ {
86
+ trace,
87
+ localNodeName = "local",
88
+ }: { trace?: boolean; localNodeName?: string } | undefined = {
89
+ localNodeName: "local",
90
+ },
91
+ ): Promise<Peer> {
92
+ const [localNodeAsPeer, storageAsPeer] = cojsonInternals.connectedPeers(
93
+ localNodeName,
94
+ "storage",
95
+ {
96
+ peer1role: "client",
97
+ peer2role: "storage",
98
+ trace,
99
+ crashOnClose: true,
100
+ },
101
+ );
102
+
103
+ await IDBStorage.open(localNodeAsPeer.incoming, localNodeAsPeer.outgoing);
104
+
105
+ return { ...storageAsPeer, priority: 100 };
106
+ }
107
+
108
+ static async open(
109
+ fromLocalNode: IncomingSyncStream,
110
+ toLocalNode: OutgoingSyncQueue,
111
+ ) {
112
+ const dbPromise = new Promise<IDBDatabase>((resolve, reject) => {
113
+ const request = indexedDB.open("jazz-storage", 4);
114
+ request.onerror = () => {
115
+ reject(request.error);
116
+ };
117
+ request.onsuccess = () => {
118
+ resolve(request.result);
119
+ };
120
+ request.onupgradeneeded = async (ev) => {
121
+ const db = request.result;
122
+ if (ev.oldVersion === 0) {
123
+ const coValues = db.createObjectStore("coValues", {
124
+ autoIncrement: true,
125
+ keyPath: "rowID",
126
+ });
127
+
128
+ coValues.createIndex("coValuesById", "id", {
129
+ unique: true,
130
+ });
131
+
132
+ const sessions = db.createObjectStore("sessions", {
133
+ autoIncrement: true,
134
+ keyPath: "rowID",
135
+ });
136
+
137
+ sessions.createIndex("sessionsByCoValue", "coValue");
138
+ sessions.createIndex("uniqueSessions", ["coValue", "sessionID"], {
139
+ unique: true,
140
+ });
141
+
142
+ db.createObjectStore("transactions", {
143
+ keyPath: ["ses", "idx"],
144
+ });
145
+ }
146
+ if (ev.oldVersion <= 1) {
147
+ db.createObjectStore("signatureAfter", {
148
+ keyPath: ["ses", "idx"],
149
+ });
178
150
  }
151
+ };
152
+ });
153
+
154
+ return new IDBStorage(await dbPromise, fromLocalNode, toLocalNode);
155
+ }
156
+
157
+ async handleSyncMessage(msg: SyncMessage) {
158
+ switch (msg.action) {
159
+ case "load":
160
+ await this.handleLoad(msg);
161
+ break;
162
+ case "content":
163
+ await this.handleContent(msg);
164
+ break;
165
+ case "known":
166
+ await this.handleKnown(msg);
167
+ break;
168
+ case "done":
169
+ await this.handleDone(msg);
170
+ break;
179
171
  }
180
-
181
- currentTx:
182
- | {
183
- id: number;
184
- tx: IDBTransaction;
185
- stores: {
186
- coValues: IDBObjectStore;
187
- sessions: IDBObjectStore;
188
- transactions: IDBObjectStore;
189
- signatureAfter: IDBObjectStore;
190
- };
191
- startedAt: number;
192
- pendingRequests: ((txEntry: {
193
- stores: {
194
- coValues: IDBObjectStore;
195
- sessions: IDBObjectStore;
196
- transactions: IDBObjectStore;
197
- signatureAfter: IDBObjectStore;
198
- };
199
- }) => void)[];
200
- }
201
- | undefined;
202
- currentTxID = 0;
203
-
204
- makeRequest<T>(
205
- handler: (stores: {
172
+ }
173
+
174
+ currentTx:
175
+ | {
176
+ id: number;
177
+ tx: IDBTransaction;
178
+ stores: {
179
+ coValues: IDBObjectStore;
180
+ sessions: IDBObjectStore;
181
+ transactions: IDBObjectStore;
182
+ signatureAfter: IDBObjectStore;
183
+ };
184
+ startedAt: number;
185
+ pendingRequests: ((txEntry: {
186
+ stores: {
206
187
  coValues: IDBObjectStore;
207
188
  sessions: IDBObjectStore;
208
189
  transactions: IDBObjectStore;
209
190
  signatureAfter: IDBObjectStore;
210
- }) => IDBRequest,
211
- ): SyncPromise<T> {
212
- return new SyncPromise((resolve, reject) => {
213
- let txEntry = this.currentTx;
214
-
215
- const requestEntry = ({
216
- stores,
217
- }: {
218
- stores: {
219
- coValues: IDBObjectStore;
220
- sessions: IDBObjectStore;
221
- transactions: IDBObjectStore;
222
- signatureAfter: IDBObjectStore;
223
- };
224
- }) => {
225
- const request = handler(stores);
226
- request.onerror = () => {
227
- console.error("Error in request", request.error);
228
- this.currentTx = undefined;
229
- reject(request.error);
230
- // TODO: recover pending requests in new tx
231
- };
232
- request.onsuccess = () => {
233
- const value = request.result as T;
234
- resolve(value);
235
-
236
- const next = txEntry!.pendingRequests.shift();
237
-
238
- if (next) {
239
- next({ stores });
240
- } else {
241
- if (this.currentTx === txEntry) {
242
- this.currentTx = undefined;
243
- }
244
- }
245
- };
246
- };
247
-
248
- if (!txEntry || performance.now() - txEntry.startedAt > 20) {
249
- const tx = this.db.transaction(
250
- ["coValues", "sessions", "transactions", "signatureAfter"],
251
- "readwrite",
252
- );
253
- txEntry = {
254
- id: this.currentTxID++,
255
- tx,
256
- stores: {
257
- coValues: tx.objectStore("coValues"),
258
- sessions: tx.objectStore("sessions"),
259
- transactions: tx.objectStore("transactions"),
260
- signatureAfter: tx.objectStore("signatureAfter"),
261
- },
262
- startedAt: performance.now(),
263
- pendingRequests: [],
264
- };
265
-
266
- // console.time("IndexedDB TX" + txEntry.id);
267
-
268
- // txEntry.tx.oncomplete = () => {
269
- // console.timeEnd("IndexedDB TX" + txEntry!.id);
270
- // };
271
-
272
- this.currentTx = txEntry;
273
-
274
- requestEntry(txEntry);
275
- } else {
276
- txEntry.pendingRequests.push(requestEntry);
277
- // console.log(
278
- // "Queued request in TX " + txEntry.id,
279
- // txEntry.pendingRequests.length
280
- // );
191
+ };
192
+ }) => void)[];
193
+ }
194
+ | undefined;
195
+ currentTxID = 0;
196
+
197
+ makeRequest<T>(
198
+ handler: (stores: {
199
+ coValues: IDBObjectStore;
200
+ sessions: IDBObjectStore;
201
+ transactions: IDBObjectStore;
202
+ signatureAfter: IDBObjectStore;
203
+ }) => IDBRequest,
204
+ ): SyncPromise<T> {
205
+ return new SyncPromise((resolve, reject) => {
206
+ let txEntry = this.currentTx;
207
+
208
+ const requestEntry = ({
209
+ stores,
210
+ }: {
211
+ stores: {
212
+ coValues: IDBObjectStore;
213
+ sessions: IDBObjectStore;
214
+ transactions: IDBObjectStore;
215
+ signatureAfter: IDBObjectStore;
216
+ };
217
+ }) => {
218
+ const request = handler(stores);
219
+ request.onerror = () => {
220
+ console.error("Error in request", request.error);
221
+ this.currentTx = undefined;
222
+ reject(request.error);
223
+ // TODO: recover pending requests in new tx
224
+ };
225
+ request.onsuccess = () => {
226
+ const value = request.result as T;
227
+ resolve(value);
228
+
229
+ const next = txEntry!.pendingRequests.shift();
230
+
231
+ if (next) {
232
+ next({ stores });
233
+ } else {
234
+ if (this.currentTx === txEntry) {
235
+ this.currentTx = undefined;
281
236
  }
282
- });
283
- }
237
+ }
238
+ };
239
+ };
284
240
 
285
- sendNewContentAfter(
286
- theirKnown: CojsonInternalTypes.CoValueKnownState,
287
- asDependencyOf?: CojsonInternalTypes.RawCoID,
288
- ): SyncPromise<void> {
289
- return this.makeRequest<StoredCoValueRow | undefined>(({ coValues }) =>
290
- coValues.index("coValuesById").get(theirKnown.id),
291
- )
292
- .then((coValueRow) => {
293
- return (
294
- coValueRow
295
- ? this.makeRequest<StoredSessionRow[]>(({ sessions }) =>
296
- sessions
297
- .index("sessionsByCoValue")
298
- .getAll(coValueRow.rowID),
299
- )
300
- : SyncPromise.resolve([])
301
- ).then((allOurSessions) => {
302
- const ourKnown: CojsonInternalTypes.CoValueKnownState = {
303
- id: theirKnown.id,
304
- header: !!coValueRow,
305
- sessions: {},
306
- };
307
-
308
- const newContentPieces: CojsonInternalTypes.NewContentMessage[] =
309
- [
310
- {
311
- action: "content",
312
- id: theirKnown.id,
313
- header: theirKnown.header
314
- ? undefined
315
- : coValueRow?.header,
316
- new: {},
317
- priority: cojsonInternals.getPriorityFromHeader(
318
- coValueRow?.header,
319
- ),
320
- },
321
- ];
322
-
323
- return SyncPromise.all(
324
- allOurSessions.map((sessionRow) => {
325
- ourKnown.sessions[sessionRow.sessionID] =
326
- sessionRow.lastIdx;
327
-
328
- if (
329
- sessionRow.lastIdx >
330
- (theirKnown.sessions[sessionRow.sessionID] || 0)
331
- ) {
332
- const firstNewTxIdx =
333
- theirKnown.sessions[sessionRow.sessionID] ||
334
- 0;
335
-
336
- return this.makeRequest<SignatureAfterRow[]>(
337
- ({ signatureAfter }) =>
338
- signatureAfter.getAll(
339
- IDBKeyRange.bound(
340
- [
341
- sessionRow.rowID,
342
- firstNewTxIdx,
343
- ],
344
- [sessionRow.rowID, Infinity],
345
- ),
346
- ),
347
- ).then((signaturesAndIdxs) => {
348
- // console.log(
349
- // theirKnown.id,
350
- // "signaturesAndIdxs",
351
- // JSON.stringify(signaturesAndIdxs)
352
- // );
353
-
354
- return this.makeRequest<TransactionRow[]>(
355
- ({ transactions }) =>
356
- transactions.getAll(
357
- IDBKeyRange.bound(
358
- [
359
- sessionRow.rowID,
360
- firstNewTxIdx,
361
- ],
362
- [
363
- sessionRow.rowID,
364
- Infinity,
365
- ],
366
- ),
367
- ),
368
- ).then((newTxsInSession) => {
369
- collectNewTxs(
370
- newTxsInSession,
371
- newContentPieces,
372
- sessionRow,
373
- signaturesAndIdxs,
374
- theirKnown,
375
- firstNewTxIdx,
376
- );
377
- });
378
- });
379
- } else {
380
- return SyncPromise.resolve();
381
- }
382
- }),
383
- ).then(() => {
384
- const dependedOnCoValues = getDependedOnCoValues(
385
- coValueRow,
386
- newContentPieces,
387
- theirKnown,
388
- );
389
-
390
- return SyncPromise.all(
391
- dependedOnCoValues.map((dependedOnCoValue) =>
392
- this.sendNewContentAfter(
393
- {
394
- id: dependedOnCoValue,
395
- header: false,
396
- sessions: {},
397
- },
398
- asDependencyOf || theirKnown.id,
399
- ),
400
- ),
401
- ).then(() => {
402
- // we're done with IndexedDB stuff here so can use native Promises again
403
- setTimeout(() => {
404
- this.toLocalNode
405
- .push({
406
- action: "known",
407
- ...ourKnown,
408
- asDependencyOf,
409
- })
410
- .catch((e) =>
411
- console.error(
412
- "Error sending known state",
413
- e,
414
- ),
415
- );
416
-
417
- const nonEmptyNewContentPieces =
418
- newContentPieces.filter(
419
- (piece) =>
420
- piece.header ||
421
- Object.keys(piece.new).length > 0,
422
- );
423
-
424
- // console.log(theirKnown.id, nonEmptyNewContentPieces);
425
-
426
- for (const piece of nonEmptyNewContentPieces) {
427
- this.toLocalNode
428
- .push(piece)
429
- .catch((e) =>
430
- console.error(
431
- "Error sending new content piece",
432
- e,
433
- ),
434
- );
435
- }
436
- });
437
-
438
- return Promise.resolve();
439
- });
440
- });
441
- });
442
- })
443
- .then(() => {});
444
- }
241
+ if (!txEntry || performance.now() - txEntry.startedAt > 20) {
242
+ const tx = this.db.transaction(
243
+ ["coValues", "sessions", "transactions", "signatureAfter"],
244
+ "readwrite",
245
+ );
246
+ txEntry = {
247
+ id: this.currentTxID++,
248
+ tx,
249
+ stores: {
250
+ coValues: tx.objectStore("coValues"),
251
+ sessions: tx.objectStore("sessions"),
252
+ transactions: tx.objectStore("transactions"),
253
+ signatureAfter: tx.objectStore("signatureAfter"),
254
+ },
255
+ startedAt: performance.now(),
256
+ pendingRequests: [],
257
+ };
445
258
 
446
- handleLoad(msg: CojsonInternalTypes.LoadMessage) {
447
- return this.sendNewContentAfter(msg);
448
- }
259
+ // console.time("IndexedDB TX" + txEntry.id);
449
260
 
450
- handleContent(
451
- msg: CojsonInternalTypes.NewContentMessage,
452
- ): SyncPromise<void> {
453
- return this.makeRequest<StoredCoValueRow | undefined>(({ coValues }) =>
454
- coValues.index("coValuesById").get(msg.id),
455
- )
456
- .then((coValueRow) => {
457
- if (coValueRow?.rowID === undefined) {
458
- const header = msg.header;
459
- if (!header) {
460
- console.error("Expected to be sent header first");
461
- this.toLocalNode
462
- .push({
463
- action: "known",
464
- id: msg.id,
465
- header: false,
466
- sessions: {},
467
- isCorrection: true,
468
- })
469
- .catch((e) =>
470
- console.error("Error sending known state", e),
471
- );
472
- return SyncPromise.resolve();
473
- }
474
-
475
- return this.makeRequest<IDBValidKey>(({ coValues }) =>
476
- coValues.put({
477
- id: msg.id,
478
- header: header,
479
- } satisfies CoValueRow),
480
- ) as SyncPromise<number>;
481
- } else {
482
- return SyncPromise.resolve(coValueRow.rowID);
483
- }
484
- })
485
- .then((storedCoValueRowID: number) => {
486
- void this.makeRequest<StoredSessionRow[]>(({ sessions }) =>
487
- sessions
488
- .index("sessionsByCoValue")
489
- .getAll(storedCoValueRowID),
490
- ).then((allOurSessionsEntries) => {
491
- const allOurSessions: {
492
- [sessionID: SessionID]: StoredSessionRow;
493
- } = Object.fromEntries(
494
- allOurSessionsEntries.map((row) => [
495
- row.sessionID,
496
- row,
497
- ]),
498
- );
261
+ // txEntry.tx.oncomplete = () => {
262
+ // console.timeEnd("IndexedDB TX" + txEntry!.id);
263
+ // };
264
+
265
+ this.currentTx = txEntry;
499
266
 
500
- const ourKnown: CojsonInternalTypes.CoValueKnownState = {
501
- id: msg.id,
502
- header: true,
503
- sessions: {},
504
- };
505
- let invalidAssumptions = false;
506
-
507
- return Promise.all(
508
- (Object.keys(msg.new) as SessionID[]).map(
509
- (sessionID) => {
510
- const sessionRow = allOurSessions[sessionID];
511
- if (sessionRow) {
512
- ourKnown.sessions[sessionRow.sessionID] =
513
- sessionRow.lastIdx;
514
- }
515
-
516
- if (
517
- (sessionRow?.lastIdx || 0) <
518
- (msg.new[sessionID]?.after || 0)
519
- ) {
520
- invalidAssumptions = true;
521
- } else {
522
- return this.putNewTxs(
523
- msg,
524
- sessionID,
525
- sessionRow,
526
- storedCoValueRowID,
527
- );
528
- }
529
- },
267
+ requestEntry(txEntry);
268
+ } else {
269
+ txEntry.pendingRequests.push(requestEntry);
270
+ // console.log(
271
+ // "Queued request in TX " + txEntry.id,
272
+ // txEntry.pendingRequests.length
273
+ // );
274
+ }
275
+ });
276
+ }
277
+
278
+ sendNewContentAfter(
279
+ theirKnown: CojsonInternalTypes.CoValueKnownState,
280
+ asDependencyOf?: CojsonInternalTypes.RawCoID,
281
+ ): SyncPromise<void> {
282
+ return this.makeRequest<StoredCoValueRow | undefined>(({ coValues }) =>
283
+ coValues.index("coValuesById").get(theirKnown.id),
284
+ )
285
+ .then((coValueRow) => {
286
+ return (
287
+ coValueRow
288
+ ? this.makeRequest<StoredSessionRow[]>(({ sessions }) =>
289
+ sessions.index("sessionsByCoValue").getAll(coValueRow.rowID),
290
+ )
291
+ : SyncPromise.resolve([])
292
+ ).then((allOurSessions) => {
293
+ const ourKnown: CojsonInternalTypes.CoValueKnownState = {
294
+ id: theirKnown.id,
295
+ header: !!coValueRow,
296
+ sessions: {},
297
+ };
298
+
299
+ const newContentPieces: CojsonInternalTypes.NewContentMessage[] = [
300
+ {
301
+ action: "content",
302
+ id: theirKnown.id,
303
+ header: theirKnown.header ? undefined : coValueRow?.header,
304
+ new: {},
305
+ priority: cojsonInternals.getPriorityFromHeader(
306
+ coValueRow?.header,
307
+ ),
308
+ },
309
+ ];
310
+
311
+ return SyncPromise.all(
312
+ allOurSessions.map((sessionRow) => {
313
+ ourKnown.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
314
+
315
+ if (
316
+ sessionRow.lastIdx >
317
+ (theirKnown.sessions[sessionRow.sessionID] || 0)
318
+ ) {
319
+ const firstNewTxIdx =
320
+ theirKnown.sessions[sessionRow.sessionID] || 0;
321
+
322
+ return this.makeRequest<SignatureAfterRow[]>(
323
+ ({ signatureAfter }) =>
324
+ signatureAfter.getAll(
325
+ IDBKeyRange.bound(
326
+ [sessionRow.rowID, firstNewTxIdx],
327
+ [sessionRow.rowID, Infinity],
328
+ ),
329
+ ),
330
+ ).then((signaturesAndIdxs) => {
331
+ // console.log(
332
+ // theirKnown.id,
333
+ // "signaturesAndIdxs",
334
+ // JSON.stringify(signaturesAndIdxs)
335
+ // );
336
+
337
+ return this.makeRequest<TransactionRow[]>(
338
+ ({ transactions }) =>
339
+ transactions.getAll(
340
+ IDBKeyRange.bound(
341
+ [sessionRow.rowID, firstNewTxIdx],
342
+ [sessionRow.rowID, Infinity],
530
343
  ),
531
- ).then(() => {
532
- if (invalidAssumptions) {
533
- this.toLocalNode
534
- .push({
535
- action: "known",
536
- ...ourKnown,
537
- isCorrection: invalidAssumptions,
538
- })
539
- .catch((e) =>
540
- console.error(
541
- "Error sending known state",
542
- e,
543
- ),
544
- );
545
- }
546
- });
344
+ ),
345
+ ).then((newTxsInSession) => {
346
+ collectNewTxs(
347
+ newTxsInSession,
348
+ newContentPieces,
349
+ sessionRow,
350
+ signaturesAndIdxs,
351
+ theirKnown,
352
+ firstNewTxIdx,
353
+ );
354
+ });
547
355
  });
548
- });
549
- }
550
-
551
- private putNewTxs(
552
- msg: CojsonInternalTypes.NewContentMessage,
553
- sessionID: SessionID,
554
- sessionRow: StoredSessionRow | undefined,
555
- storedCoValueRowID: number,
556
- ) {
557
- const newTransactions = msg.new[sessionID]?.newTransactions || [];
558
-
559
- const actuallyNewOffset =
560
- (sessionRow?.lastIdx || 0) - (msg.new[sessionID]?.after || 0);
561
-
562
- const actuallyNewTransactions =
563
- newTransactions.slice(actuallyNewOffset);
564
-
565
- let newBytesSinceLastSignature =
566
- (sessionRow?.bytesSinceLastSignature || 0) +
567
- actuallyNewTransactions.reduce(
568
- (sum, tx) =>
569
- sum +
570
- (tx.privacy === "private"
571
- ? tx.encryptedChanges.length
572
- : tx.changes.length),
573
- 0,
356
+ } else {
357
+ return SyncPromise.resolve();
358
+ }
359
+ }),
360
+ ).then(() => {
361
+ const dependedOnCoValues = getDependedOnCoValues(
362
+ coValueRow,
363
+ newContentPieces,
364
+ theirKnown,
574
365
  );
575
366
 
576
- const newLastIdx =
577
- (sessionRow?.lastIdx || 0) + actuallyNewTransactions.length;
578
-
579
- let shouldWriteSignature = false;
367
+ return SyncPromise.all(
368
+ dependedOnCoValues.map((dependedOnCoValue) =>
369
+ this.sendNewContentAfter(
370
+ {
371
+ id: dependedOnCoValue,
372
+ header: false,
373
+ sessions: {},
374
+ },
375
+ asDependencyOf || theirKnown.id,
376
+ ),
377
+ ),
378
+ ).then(() => {
379
+ // we're done with IndexedDB stuff here so can use native Promises again
380
+ setTimeout(() => {
381
+ this.toLocalNode
382
+ .push({
383
+ action: "known",
384
+ ...ourKnown,
385
+ asDependencyOf,
386
+ })
387
+ .catch((e) => console.error("Error sending known state", e));
388
+
389
+ const nonEmptyNewContentPieces = newContentPieces.filter(
390
+ (piece) => piece.header || Object.keys(piece.new).length > 0,
391
+ );
580
392
 
581
- if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
582
- shouldWriteSignature = true;
583
- newBytesSinceLastSignature = 0;
584
- }
393
+ // console.log(theirKnown.id, nonEmptyNewContentPieces);
585
394
 
586
- const nextIdx = sessionRow?.lastIdx || 0;
395
+ for (const piece of nonEmptyNewContentPieces) {
396
+ this.toLocalNode
397
+ .push(piece)
398
+ .catch((e) =>
399
+ console.error("Error sending new content piece", e),
400
+ );
401
+ }
402
+ });
587
403
 
588
- const sessionUpdate = {
589
- coValue: storedCoValueRowID,
590
- sessionID: sessionID,
591
- lastIdx: newLastIdx,
592
- lastSignature: msg.new[sessionID]!.lastSignature,
593
- bytesSinceLastSignature: newBytesSinceLastSignature,
594
- };
404
+ return Promise.resolve();
405
+ });
406
+ });
407
+ });
408
+ })
409
+ .then(() => {});
410
+ }
411
+
412
+ handleLoad(msg: CojsonInternalTypes.LoadMessage) {
413
+ return this.sendNewContentAfter(msg);
414
+ }
415
+
416
+ handleContent(msg: CojsonInternalTypes.NewContentMessage): SyncPromise<void> {
417
+ return this.makeRequest<StoredCoValueRow | undefined>(({ coValues }) =>
418
+ coValues.index("coValuesById").get(msg.id),
419
+ )
420
+ .then((coValueRow) => {
421
+ if (coValueRow?.rowID === undefined) {
422
+ const header = msg.header;
423
+ if (!header) {
424
+ console.error("Expected to be sent header first");
425
+ this.toLocalNode
426
+ .push({
427
+ action: "known",
428
+ id: msg.id,
429
+ header: false,
430
+ sessions: {},
431
+ isCorrection: true,
432
+ })
433
+ .catch((e) => console.error("Error sending known state", e));
434
+ return SyncPromise.resolve();
435
+ }
595
436
 
596
- return this.makeRequest<number>(({ sessions }) =>
597
- sessions.put(
598
- sessionRow?.rowID
599
- ? {
600
- rowID: sessionRow.rowID,
601
- ...sessionUpdate,
602
- }
603
- : sessionUpdate,
604
- ),
605
- ).then((sessionRowID) => {
606
- let maybePutRequest;
607
- if (shouldWriteSignature) {
608
- maybePutRequest = this.makeRequest(({ signatureAfter }) =>
609
- signatureAfter.put({
610
- ses: sessionRowID,
611
- // TODO: newLastIdx is a misnomer, it's actually more like nextIdx or length
612
- idx: newLastIdx - 1,
613
- signature: msg.new[sessionID]!.lastSignature,
614
- } satisfies SignatureAfterRow),
437
+ return this.makeRequest<IDBValidKey>(({ coValues }) =>
438
+ coValues.put({
439
+ id: msg.id,
440
+ header: header,
441
+ } satisfies CoValueRow),
442
+ ) as SyncPromise<number>;
443
+ } else {
444
+ return SyncPromise.resolve(coValueRow.rowID);
445
+ }
446
+ })
447
+ .then((storedCoValueRowID: number) => {
448
+ void this.makeRequest<StoredSessionRow[]>(({ sessions }) =>
449
+ sessions.index("sessionsByCoValue").getAll(storedCoValueRowID),
450
+ ).then((allOurSessionsEntries) => {
451
+ const allOurSessions: {
452
+ [sessionID: SessionID]: StoredSessionRow;
453
+ } = Object.fromEntries(
454
+ allOurSessionsEntries.map((row) => [row.sessionID, row]),
455
+ );
456
+
457
+ const ourKnown: CojsonInternalTypes.CoValueKnownState = {
458
+ id: msg.id,
459
+ header: true,
460
+ sessions: {},
461
+ };
462
+ let invalidAssumptions = false;
463
+
464
+ return Promise.all(
465
+ (Object.keys(msg.new) as SessionID[]).map((sessionID) => {
466
+ const sessionRow = allOurSessions[sessionID];
467
+ if (sessionRow) {
468
+ ourKnown.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
469
+ }
470
+
471
+ if (
472
+ (sessionRow?.lastIdx || 0) < (msg.new[sessionID]?.after || 0)
473
+ ) {
474
+ invalidAssumptions = true;
475
+ } else {
476
+ return this.putNewTxs(
477
+ msg,
478
+ sessionID,
479
+ sessionRow,
480
+ storedCoValueRowID,
615
481
  );
616
- } else {
617
- maybePutRequest = SyncPromise.resolve();
482
+ }
483
+ }),
484
+ ).then(() => {
485
+ if (invalidAssumptions) {
486
+ this.toLocalNode
487
+ .push({
488
+ action: "known",
489
+ ...ourKnown,
490
+ isCorrection: invalidAssumptions,
491
+ })
492
+ .catch((e) => console.error("Error sending known state", e));
618
493
  }
619
-
620
- return maybePutRequest.then(() =>
621
- Promise.all(
622
- actuallyNewTransactions.map((newTransaction, i) => {
623
- return this.makeRequest(({ transactions }) =>
624
- transactions.add({
625
- ses: sessionRowID,
626
- idx: nextIdx + i,
627
- tx: newTransaction,
628
- } satisfies TransactionRow),
629
- );
630
- }),
631
- ),
632
- );
494
+ });
633
495
  });
496
+ });
497
+ }
498
+
499
+ private putNewTxs(
500
+ msg: CojsonInternalTypes.NewContentMessage,
501
+ sessionID: SessionID,
502
+ sessionRow: StoredSessionRow | undefined,
503
+ storedCoValueRowID: number,
504
+ ) {
505
+ const newTransactions = msg.new[sessionID]?.newTransactions || [];
506
+
507
+ const actuallyNewOffset =
508
+ (sessionRow?.lastIdx || 0) - (msg.new[sessionID]?.after || 0);
509
+
510
+ const actuallyNewTransactions = newTransactions.slice(actuallyNewOffset);
511
+
512
+ let newBytesSinceLastSignature =
513
+ (sessionRow?.bytesSinceLastSignature || 0) +
514
+ actuallyNewTransactions.reduce(
515
+ (sum, tx) =>
516
+ sum +
517
+ (tx.privacy === "private"
518
+ ? tx.encryptedChanges.length
519
+ : tx.changes.length),
520
+ 0,
521
+ );
522
+
523
+ const newLastIdx =
524
+ (sessionRow?.lastIdx || 0) + actuallyNewTransactions.length;
525
+
526
+ let shouldWriteSignature = false;
527
+
528
+ if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
529
+ shouldWriteSignature = true;
530
+ newBytesSinceLastSignature = 0;
634
531
  }
635
532
 
636
- handleKnown(msg: CojsonInternalTypes.KnownStateMessage) {
637
- return this.sendNewContentAfter(msg);
638
- }
639
-
640
- handleDone(_msg: CojsonInternalTypes.DoneMessage) {}
641
-
642
- // inTransaction(mode: "readwrite" | "readonly"): {
643
- // coValues: IDBObjectStore;
644
- // sessions: IDBObjectStore;
645
- // transactions: IDBObjectStore;
646
- // signatureAfter: IDBObjectStore;
647
- // } {
648
- // const tx = this.db.transaction(
649
- // ["coValues", "sessions", "transactions", "signatureAfter"],
650
- // mode
651
- // );
652
-
653
- // const txID = lastTx;
654
- // lastTx++;
655
- // console.time("IndexedDB TX" + txID);
656
-
657
- // tx.onerror = (event) => {
658
- // const target = event.target as unknown as {
659
- // error: DOMException;
660
- // source?: { name: string };
661
- // } | null;
662
- // throw new Error(
663
- // `Error in transaction (${target?.source?.name}): ${target?.error}`,
664
- // { cause: target?.error }
665
- // );
666
- // };
667
- // tx.oncomplete = () => {
668
- // console.timeEnd("IndexedDB TX" + txID);
669
- // }
670
- // const coValues = tx.objectStore("coValues");
671
- // const sessions = tx.objectStore("sessions");
672
- // const transactions = tx.objectStore("transactions");
673
- // const signatureAfter = tx.objectStore("signatureAfter");
674
-
675
- // return { coValues, sessions, transactions, signatureAfter };
676
- // }
533
+ const nextIdx = sessionRow?.lastIdx || 0;
534
+
535
+ const sessionUpdate = {
536
+ coValue: storedCoValueRowID,
537
+ sessionID: sessionID,
538
+ lastIdx: newLastIdx,
539
+ lastSignature: msg.new[sessionID]!.lastSignature,
540
+ bytesSinceLastSignature: newBytesSinceLastSignature,
541
+ };
542
+
543
+ return this.makeRequest<number>(({ sessions }) =>
544
+ sessions.put(
545
+ sessionRow?.rowID
546
+ ? {
547
+ rowID: sessionRow.rowID,
548
+ ...sessionUpdate,
549
+ }
550
+ : sessionUpdate,
551
+ ),
552
+ ).then((sessionRowID) => {
553
+ let maybePutRequest;
554
+ if (shouldWriteSignature) {
555
+ maybePutRequest = this.makeRequest(({ signatureAfter }) =>
556
+ signatureAfter.put({
557
+ ses: sessionRowID,
558
+ // TODO: newLastIdx is a misnomer, it's actually more like nextIdx or length
559
+ idx: newLastIdx - 1,
560
+ signature: msg.new[sessionID]!.lastSignature,
561
+ } satisfies SignatureAfterRow),
562
+ );
563
+ } else {
564
+ maybePutRequest = SyncPromise.resolve();
565
+ }
566
+
567
+ return maybePutRequest.then(() =>
568
+ Promise.all(
569
+ actuallyNewTransactions.map((newTransaction, i) => {
570
+ return this.makeRequest(({ transactions }) =>
571
+ transactions.add({
572
+ ses: sessionRowID,
573
+ idx: nextIdx + i,
574
+ tx: newTransaction,
575
+ } satisfies TransactionRow),
576
+ );
577
+ }),
578
+ ),
579
+ );
580
+ });
581
+ }
582
+
583
+ handleKnown(msg: CojsonInternalTypes.KnownStateMessage) {
584
+ return this.sendNewContentAfter(msg);
585
+ }
586
+
587
+ handleDone(_msg: CojsonInternalTypes.DoneMessage) {}
588
+
589
+ // inTransaction(mode: "readwrite" | "readonly"): {
590
+ // coValues: IDBObjectStore;
591
+ // sessions: IDBObjectStore;
592
+ // transactions: IDBObjectStore;
593
+ // signatureAfter: IDBObjectStore;
594
+ // } {
595
+ // const tx = this.db.transaction(
596
+ // ["coValues", "sessions", "transactions", "signatureAfter"],
597
+ // mode
598
+ // );
599
+
600
+ // const txID = lastTx;
601
+ // lastTx++;
602
+ // console.time("IndexedDB TX" + txID);
603
+
604
+ // tx.onerror = (event) => {
605
+ // const target = event.target as unknown as {
606
+ // error: DOMException;
607
+ // source?: { name: string };
608
+ // } | null;
609
+ // throw new Error(
610
+ // `Error in transaction (${target?.source?.name}): ${target?.error}`,
611
+ // { cause: target?.error }
612
+ // );
613
+ // };
614
+ // tx.oncomplete = () => {
615
+ // console.timeEnd("IndexedDB TX" + txID);
616
+ // }
617
+ // const coValues = tx.objectStore("coValues");
618
+ // const sessions = tx.objectStore("sessions");
619
+ // const transactions = tx.objectStore("transactions");
620
+ // const signatureAfter = tx.objectStore("signatureAfter");
621
+
622
+ // return { coValues, sessions, transactions, signatureAfter };
623
+ // }
677
624
  }
678
625
 
679
626
  function collectNewTxs(
680
- newTxsInSession: TransactionRow[],
681
- newContentPieces: CojsonInternalTypes.NewContentMessage[],
682
- sessionRow: StoredSessionRow,
683
- signaturesAndIdxs: SignatureAfterRow[],
684
- theirKnown: CojsonInternalTypes.CoValueKnownState,
685
- firstNewTxIdx: number,
627
+ newTxsInSession: TransactionRow[],
628
+ newContentPieces: CojsonInternalTypes.NewContentMessage[],
629
+ sessionRow: StoredSessionRow,
630
+ signaturesAndIdxs: SignatureAfterRow[],
631
+ theirKnown: CojsonInternalTypes.CoValueKnownState,
632
+ firstNewTxIdx: number,
686
633
  ) {
687
- let idx = firstNewTxIdx;
688
-
689
- // console.log(
690
- // theirKnown.id,
691
- // "newTxInSession",
692
- // newTxInSession.length
693
- // );
694
- for (const tx of newTxsInSession) {
695
- let sessionEntry =
696
- newContentPieces[newContentPieces.length - 1]!.new[
697
- sessionRow.sessionID
698
- ];
699
- if (!sessionEntry) {
700
- sessionEntry = {
701
- after: idx,
702
- lastSignature:
703
- "WILL_BE_REPLACED" as CojsonInternalTypes.Signature,
704
- newTransactions: [],
705
- };
706
- newContentPieces[newContentPieces.length - 1]!.new[
707
- sessionRow.sessionID
708
- ] = sessionEntry;
709
- }
710
-
711
- sessionEntry.newTransactions.push(tx.tx);
634
+ let idx = firstNewTxIdx;
635
+
636
+ // console.log(
637
+ // theirKnown.id,
638
+ // "newTxInSession",
639
+ // newTxInSession.length
640
+ // );
641
+ for (const tx of newTxsInSession) {
642
+ let sessionEntry =
643
+ newContentPieces[newContentPieces.length - 1]!.new[sessionRow.sessionID];
644
+ if (!sessionEntry) {
645
+ sessionEntry = {
646
+ after: idx,
647
+ lastSignature: "WILL_BE_REPLACED" as CojsonInternalTypes.Signature,
648
+ newTransactions: [],
649
+ };
650
+ newContentPieces[newContentPieces.length - 1]!.new[sessionRow.sessionID] =
651
+ sessionEntry;
652
+ }
712
653
 
713
- if (signaturesAndIdxs[0] && idx === signaturesAndIdxs[0].idx) {
714
- sessionEntry.lastSignature = signaturesAndIdxs[0].signature;
715
- signaturesAndIdxs.shift();
716
- newContentPieces.push({
717
- action: "content",
718
- id: theirKnown.id,
719
- new: {},
720
- priority: cojsonInternals.getPriorityFromHeader(undefined),
721
- });
722
- } else if (idx === firstNewTxIdx + newTxsInSession.length - 1) {
723
- sessionEntry.lastSignature = sessionRow.lastSignature;
724
- }
725
- idx += 1;
654
+ sessionEntry.newTransactions.push(tx.tx);
655
+
656
+ if (signaturesAndIdxs[0] && idx === signaturesAndIdxs[0].idx) {
657
+ sessionEntry.lastSignature = signaturesAndIdxs[0].signature;
658
+ signaturesAndIdxs.shift();
659
+ newContentPieces.push({
660
+ action: "content",
661
+ id: theirKnown.id,
662
+ new: {},
663
+ priority: cojsonInternals.getPriorityFromHeader(undefined),
664
+ });
665
+ } else if (idx === firstNewTxIdx + newTxsInSession.length - 1) {
666
+ sessionEntry.lastSignature = sessionRow.lastSignature;
726
667
  }
668
+ idx += 1;
669
+ }
727
670
  }
728
671
 
729
672
  function getDependedOnCoValues(
730
- coValueRow: StoredCoValueRow | undefined,
731
- newContentPieces: CojsonInternalTypes.NewContentMessage[],
732
- theirKnown: CojsonInternalTypes.CoValueKnownState,
673
+ coValueRow: StoredCoValueRow | undefined,
674
+ newContentPieces: CojsonInternalTypes.NewContentMessage[],
675
+ theirKnown: CojsonInternalTypes.CoValueKnownState,
733
676
  ) {
734
- return coValueRow?.header.ruleset.type === "group"
735
- ? newContentPieces
736
- .flatMap((piece) => Object.values(piece.new))
737
- .flatMap((sessionEntry) =>
738
- sessionEntry.newTransactions.flatMap((tx) => {
739
- if (tx.privacy !== "trusting") return [];
740
- // TODO: avoid parse here?
741
- return cojsonInternals
742
- .parseJSON(tx.changes)
743
- .map(
744
- (change) =>
745
- change &&
746
- typeof change === "object" &&
747
- "op" in change &&
748
- change.op === "set" &&
749
- "key" in change &&
750
- change.key,
751
- )
752
- .filter(
753
- (key): key is CojsonInternalTypes.RawCoID =>
754
- typeof key === "string" &&
755
- key.startsWith("co_"),
756
- );
757
- }),
677
+ return coValueRow?.header.ruleset.type === "group"
678
+ ? newContentPieces
679
+ .flatMap((piece) => Object.values(piece.new))
680
+ .flatMap((sessionEntry) =>
681
+ sessionEntry.newTransactions.flatMap((tx) => {
682
+ if (tx.privacy !== "trusting") return [];
683
+ // TODO: avoid parse here?
684
+ return cojsonInternals
685
+ .parseJSON(tx.changes)
686
+ .map(
687
+ (change) =>
688
+ change &&
689
+ typeof change === "object" &&
690
+ "op" in change &&
691
+ change.op === "set" &&
692
+ "key" in change &&
693
+ change.key,
758
694
  )
759
- : coValueRow?.header.ruleset.type === "ownedByGroup"
760
- ? [
761
- coValueRow?.header.ruleset.group,
762
- ...new Set(
763
- newContentPieces.flatMap((piece) =>
764
- Object.keys(piece)
765
- .map((sessionID) =>
766
- cojsonInternals.accountOrAgentIDfromSessionID(
767
- sessionID as SessionID,
768
- ),
769
- )
770
- .filter(
771
- (accountID): accountID is RawAccountID =>
772
- cojsonInternals.isAccountID(accountID) &&
773
- accountID !== theirKnown.id,
774
- ),
775
- ),
695
+ .filter(
696
+ (key): key is CojsonInternalTypes.RawCoID =>
697
+ typeof key === "string" && key.startsWith("co_"),
698
+ );
699
+ }),
700
+ )
701
+ : coValueRow?.header.ruleset.type === "ownedByGroup"
702
+ ? [
703
+ coValueRow?.header.ruleset.group,
704
+ ...new Set(
705
+ newContentPieces.flatMap((piece) =>
706
+ Object.keys(piece)
707
+ .map((sessionID) =>
708
+ cojsonInternals.accountOrAgentIDfromSessionID(
709
+ sessionID as SessionID,
710
+ ),
711
+ )
712
+ .filter(
713
+ (accountID): accountID is RawAccountID =>
714
+ cojsonInternals.isAccountID(accountID) &&
715
+ accountID !== theirKnown.id,
776
716
  ),
777
- ]
778
- : [];
717
+ ),
718
+ ),
719
+ ]
720
+ : [];
779
721
  }
780
722
  // let lastTx = 0;
781
723