cojson-storage-sqlite 0.8.12 → 0.8.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/CHANGELOG.md +151 -137
  2. package/dist/index.js +13 -22
  3. package/dist/index.js.map +1 -1
  4. package/package.json +5 -9
  5. package/src/index.ts +532 -565
  6. package/tsconfig.json +2 -2
  7. package/.eslintrc.cjs +0 -24
  8. package/.prettierrc.js +0 -9
package/src/index.ts CHANGED
@@ -1,138 +1,138 @@
1
1
  import {
2
- cojsonInternals,
3
- SyncMessage,
4
- Peer,
5
- CojsonInternalTypes,
6
- SessionID,
7
- MAX_RECOMMENDED_TX_SIZE,
8
- RawAccountID,
9
- IncomingSyncStream,
10
- OutgoingSyncQueue,
2
+ CojsonInternalTypes,
3
+ IncomingSyncStream,
4
+ MAX_RECOMMENDED_TX_SIZE,
5
+ OutgoingSyncQueue,
6
+ Peer,
7
+ RawAccountID,
8
+ SessionID,
9
+ SyncMessage,
10
+ cojsonInternals,
11
11
  } from "cojson";
12
12
 
13
13
  import Database, { Database as DatabaseT } from "better-sqlite3";
14
14
 
15
15
  type CoValueRow = {
16
- id: CojsonInternalTypes.RawCoID;
17
- header: string;
16
+ id: CojsonInternalTypes.RawCoID;
17
+ header: string;
18
18
  };
19
19
 
20
20
  type StoredCoValueRow = CoValueRow & { rowID: number };
21
21
 
22
22
  type SessionRow = {
23
- coValue: number;
24
- sessionID: SessionID;
25
- lastIdx: number;
26
- lastSignature: CojsonInternalTypes.Signature;
27
- bytesSinceLastSignature?: number;
23
+ coValue: number;
24
+ sessionID: SessionID;
25
+ lastIdx: number;
26
+ lastSignature: CojsonInternalTypes.Signature;
27
+ bytesSinceLastSignature?: number;
28
28
  };
29
29
 
30
30
  type StoredSessionRow = SessionRow & { rowID: number };
31
31
 
32
32
  type TransactionRow = {
33
- ses: number;
34
- idx: number;
35
- tx: string;
33
+ ses: number;
34
+ idx: number;
35
+ tx: string;
36
36
  };
37
37
 
38
38
  type SignatureAfterRow = {
39
- ses: number;
40
- idx: number;
41
- signature: CojsonInternalTypes.Signature;
39
+ ses: number;
40
+ idx: number;
41
+ signature: CojsonInternalTypes.Signature;
42
42
  };
43
43
 
44
44
  export class SQLiteStorage {
45
- toLocalNode: OutgoingSyncQueue;
46
- db: DatabaseT;
47
-
48
- constructor(
49
- db: DatabaseT,
50
- fromLocalNode: IncomingSyncStream,
51
- toLocalNode: OutgoingSyncQueue,
52
- ) {
53
- this.db = db;
54
- this.toLocalNode = toLocalNode;
55
-
56
- const processMessages = async () => {
57
- for await (const msg of fromLocalNode) {
58
- try {
59
- if (msg === "Disconnected" || msg === "PingTimeout") {
60
- throw new Error("Unexpected Disconnected message");
61
- }
62
- await this.handleSyncMessage(msg);
63
- } catch (e) {
64
- console.error(
65
- new Error(
66
- `Error reading from localNode, handling msg\n\n${JSON.stringify(
67
- msg,
68
- (k, v) =>
69
- k === "changes" || k === "encryptedChanges"
70
- ? v.slice(0, 20) + "..."
71
- : v,
72
- )}`,
73
- { cause: e },
74
- ),
75
- );
76
- }
77
- }
78
- };
79
-
80
- processMessages().catch((e) =>
81
- console.error("Error in processMessages in sqlite", e),
82
- );
83
- }
84
-
85
- static async asPeer({
86
- filename,
87
- trace,
88
- localNodeName = "local",
89
- }: {
90
- filename: string;
91
- trace?: boolean;
92
- localNodeName?: string;
93
- }): Promise<Peer> {
94
- const [localNodeAsPeer, storageAsPeer] = cojsonInternals.connectedPeers(
95
- localNodeName,
96
- "storage",
97
- { peer1role: "client", peer2role: "storage", trace, crashOnClose: true },
98
- );
99
-
100
- await SQLiteStorage.open(
101
- filename,
102
- localNodeAsPeer.incoming,
103
- localNodeAsPeer.outgoing,
104
- );
105
-
106
- return { ...storageAsPeer, priority: 100 };
107
- }
108
-
109
- static async open(
110
- filename: string,
111
- fromLocalNode: IncomingSyncStream,
112
- toLocalNode: OutgoingSyncQueue,
113
- ) {
114
- const db = Database(filename);
115
- db.pragma("journal_mode = WAL");
116
-
117
- const oldVersion = (
118
- db.pragma("user_version") as [{ user_version: number }]
119
- )[0].user_version as number;
120
-
121
- console.log("DB version", oldVersion);
122
-
123
- if (oldVersion === 0) {
124
- console.log("Migration 0 -> 1: Basic schema");
125
- db.prepare(
126
- `CREATE TABLE IF NOT EXISTS transactions (
45
+ toLocalNode: OutgoingSyncQueue;
46
+ db: DatabaseT;
47
+
48
+ constructor(
49
+ db: DatabaseT,
50
+ fromLocalNode: IncomingSyncStream,
51
+ toLocalNode: OutgoingSyncQueue,
52
+ ) {
53
+ this.db = db;
54
+ this.toLocalNode = toLocalNode;
55
+
56
+ const processMessages = async () => {
57
+ for await (const msg of fromLocalNode) {
58
+ try {
59
+ if (msg === "Disconnected" || msg === "PingTimeout") {
60
+ throw new Error("Unexpected Disconnected message");
61
+ }
62
+ await this.handleSyncMessage(msg);
63
+ } catch (e) {
64
+ console.error(
65
+ new Error(
66
+ `Error reading from localNode, handling msg\n\n${JSON.stringify(
67
+ msg,
68
+ (k, v) =>
69
+ k === "changes" || k === "encryptedChanges"
70
+ ? v.slice(0, 20) + "..."
71
+ : v,
72
+ )}`,
73
+ { cause: e },
74
+ ),
75
+ );
76
+ }
77
+ }
78
+ };
79
+
80
+ processMessages().catch((e) =>
81
+ console.error("Error in processMessages in sqlite", e),
82
+ );
83
+ }
84
+
85
+ static async asPeer({
86
+ filename,
87
+ trace,
88
+ localNodeName = "local",
89
+ }: {
90
+ filename: string;
91
+ trace?: boolean;
92
+ localNodeName?: string;
93
+ }): Promise<Peer> {
94
+ const [localNodeAsPeer, storageAsPeer] = cojsonInternals.connectedPeers(
95
+ localNodeName,
96
+ "storage",
97
+ { peer1role: "client", peer2role: "storage", trace, crashOnClose: true },
98
+ );
99
+
100
+ await SQLiteStorage.open(
101
+ filename,
102
+ localNodeAsPeer.incoming,
103
+ localNodeAsPeer.outgoing,
104
+ );
105
+
106
+ return { ...storageAsPeer, priority: 100 };
107
+ }
108
+
109
+ static async open(
110
+ filename: string,
111
+ fromLocalNode: IncomingSyncStream,
112
+ toLocalNode: OutgoingSyncQueue,
113
+ ) {
114
+ const db = Database(filename);
115
+ db.pragma("journal_mode = WAL");
116
+
117
+ const oldVersion = (
118
+ db.pragma("user_version") as [{ user_version: number }]
119
+ )[0].user_version as number;
120
+
121
+ console.log("DB version", oldVersion);
122
+
123
+ if (oldVersion === 0) {
124
+ console.log("Migration 0 -> 1: Basic schema");
125
+ db.prepare(
126
+ `CREATE TABLE IF NOT EXISTS transactions (
127
127
  ses INTEGER,
128
128
  idx INTEGER,
129
129
  tx TEXT NOT NULL,
130
130
  PRIMARY KEY (ses, idx)
131
131
  ) WITHOUT ROWID;`,
132
- ).run();
132
+ ).run();
133
133
 
134
- db.prepare(
135
- `CREATE TABLE IF NOT EXISTS sessions (
134
+ db.prepare(
135
+ `CREATE TABLE IF NOT EXISTS sessions (
136
136
  rowID INTEGER PRIMARY KEY,
137
137
  coValue INTEGER NOT NULL,
138
138
  sessionID TEXT NOT NULL,
@@ -140,506 +140,473 @@ export class SQLiteStorage {
140
140
  lastSignature TEXT,
141
141
  UNIQUE (sessionID, coValue)
142
142
  );`,
143
- ).run();
143
+ ).run();
144
144
 
145
- db.prepare(
146
- `CREATE INDEX IF NOT EXISTS sessionsByCoValue ON sessions (coValue);`,
147
- ).run();
145
+ db.prepare(
146
+ `CREATE INDEX IF NOT EXISTS sessionsByCoValue ON sessions (coValue);`,
147
+ ).run();
148
148
 
149
- db.prepare(
150
- `CREATE TABLE IF NOT EXISTS coValues (
149
+ db.prepare(
150
+ `CREATE TABLE IF NOT EXISTS coValues (
151
151
  rowID INTEGER PRIMARY KEY,
152
152
  id TEXT NOT NULL UNIQUE,
153
153
  header TEXT NOT NULL UNIQUE
154
154
  );`,
155
- ).run();
155
+ ).run();
156
156
 
157
- db.prepare(
158
- `CREATE INDEX IF NOT EXISTS coValuesByID ON coValues (id);`,
159
- ).run();
157
+ db.prepare(
158
+ `CREATE INDEX IF NOT EXISTS coValuesByID ON coValues (id);`,
159
+ ).run();
160
160
 
161
- db.pragma("user_version = 1");
162
- console.log("Migration 0 -> 1: Basic schema - done");
163
- }
161
+ db.pragma("user_version = 1");
162
+ console.log("Migration 0 -> 1: Basic schema - done");
163
+ }
164
164
 
165
- if (oldVersion <= 1) {
166
- // fix embarrassing off-by-one error for transaction indices
167
- console.log(
168
- "Migration 1 -> 2: Fix off-by-one error for transaction indices",
169
- );
165
+ if (oldVersion <= 1) {
166
+ // fix embarrassing off-by-one error for transaction indices
167
+ console.log(
168
+ "Migration 1 -> 2: Fix off-by-one error for transaction indices",
169
+ );
170
170
 
171
- const txs = db
172
- .prepare(`SELECT * FROM transactions`)
173
- .all() as TransactionRow[];
174
-
175
- for (const tx of txs) {
176
- db.prepare(
177
- `DELETE FROM transactions WHERE ses = ? AND idx = ?`,
178
- ).run(tx.ses, tx.idx);
179
- tx.idx -= 1;
180
- db.prepare(
181
- `INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)`,
182
- ).run(tx.ses, tx.idx, tx.tx);
183
- }
184
-
185
- db.pragma("user_version = 2");
186
- console.log(
187
- "Migration 1 -> 2: Fix off-by-one error for transaction indices - done",
188
- );
189
- }
171
+ const txs = db
172
+ .prepare(`SELECT * FROM transactions`)
173
+ .all() as TransactionRow[];
190
174
 
191
- if (oldVersion <= 2) {
192
- console.log("Migration 2 -> 3: Add signatureAfter");
175
+ for (const tx of txs) {
176
+ db.prepare(`DELETE FROM transactions WHERE ses = ? AND idx = ?`).run(
177
+ tx.ses,
178
+ tx.idx,
179
+ );
180
+ tx.idx -= 1;
181
+ db.prepare(
182
+ `INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)`,
183
+ ).run(tx.ses, tx.idx, tx.tx);
184
+ }
185
+
186
+ db.pragma("user_version = 2");
187
+ console.log(
188
+ "Migration 1 -> 2: Fix off-by-one error for transaction indices - done",
189
+ );
190
+ }
193
191
 
194
- db.prepare(
195
- `CREATE TABLE IF NOT EXISTS signatureAfter (
192
+ if (oldVersion <= 2) {
193
+ console.log("Migration 2 -> 3: Add signatureAfter");
194
+
195
+ db.prepare(
196
+ `CREATE TABLE IF NOT EXISTS signatureAfter (
196
197
  ses INTEGER,
197
198
  idx INTEGER,
198
199
  signature TEXT NOT NULL,
199
200
  PRIMARY KEY (ses, idx)
200
201
  ) WITHOUT ROWID;`,
201
- ).run();
202
-
203
- db.prepare(
204
- `ALTER TABLE sessions ADD COLUMN bytesSinceLastSignature INTEGER;`,
205
- ).run();
202
+ ).run();
206
203
 
207
- db.pragma("user_version = 3");
208
- console.log("Migration 2 -> 3: Add signatureAfter - done");
209
- }
204
+ db.prepare(
205
+ `ALTER TABLE sessions ADD COLUMN bytesSinceLastSignature INTEGER;`,
206
+ ).run();
210
207
 
211
- return new SQLiteStorage(db, fromLocalNode, toLocalNode);
208
+ db.pragma("user_version = 3");
209
+ console.log("Migration 2 -> 3: Add signatureAfter - done");
212
210
  }
213
211
 
214
- async handleSyncMessage(msg: SyncMessage) {
215
- switch (msg.action) {
216
- case "load":
217
- await this.handleLoad(msg);
218
- break;
219
- case "content":
220
- await this.handleContent(msg);
221
- break;
222
- case "known":
223
- await this.handleKnown(msg);
224
- break;
225
- case "done":
226
- await this.handleDone(msg);
227
- break;
228
- }
212
+ return new SQLiteStorage(db, fromLocalNode, toLocalNode);
213
+ }
214
+
215
+ async handleSyncMessage(msg: SyncMessage) {
216
+ switch (msg.action) {
217
+ case "load":
218
+ await this.handleLoad(msg);
219
+ break;
220
+ case "content":
221
+ await this.handleContent(msg);
222
+ break;
223
+ case "known":
224
+ await this.handleKnown(msg);
225
+ break;
226
+ case "done":
227
+ await this.handleDone(msg);
228
+ break;
229
+ }
230
+ }
231
+
232
+ async sendNewContentAfter(
233
+ theirKnown: CojsonInternalTypes.CoValueKnownState,
234
+ asDependencyOf?: CojsonInternalTypes.RawCoID,
235
+ ) {
236
+ const coValueRow = (await this.db
237
+ .prepare(`SELECT * FROM coValues WHERE id = ?`)
238
+ .get(theirKnown.id)) as StoredCoValueRow | undefined;
239
+
240
+ const allOurSessions = coValueRow
241
+ ? (this.db
242
+ .prepare<number>(`SELECT * FROM sessions WHERE coValue = ?`)
243
+ .all(coValueRow.rowID) as StoredSessionRow[])
244
+ : [];
245
+
246
+ const ourKnown: CojsonInternalTypes.CoValueKnownState = {
247
+ id: theirKnown.id,
248
+ header: !!coValueRow,
249
+ sessions: {},
250
+ };
251
+
252
+ let parsedHeader;
253
+
254
+ try {
255
+ parsedHeader = (coValueRow?.header && JSON.parse(coValueRow.header)) as
256
+ | CojsonInternalTypes.CoValueHeader
257
+ | undefined;
258
+ } catch (e) {
259
+ console.warn(
260
+ theirKnown.id,
261
+ "Invalid JSON in header",
262
+ e,
263
+ coValueRow?.header,
264
+ );
265
+ return;
229
266
  }
230
267
 
231
- async sendNewContentAfter(
232
- theirKnown: CojsonInternalTypes.CoValueKnownState,
233
- asDependencyOf?: CojsonInternalTypes.RawCoID,
234
- ) {
235
- const coValueRow = (await this.db
236
- .prepare(`SELECT * FROM coValues WHERE id = ?`)
237
- .get(theirKnown.id)) as StoredCoValueRow | undefined;
238
-
239
- const allOurSessions = coValueRow
240
- ? (this.db
241
- .prepare<number>(`SELECT * FROM sessions WHERE coValue = ?`)
242
- .all(coValueRow.rowID) as StoredSessionRow[])
243
- : [];
244
-
245
- const ourKnown: CojsonInternalTypes.CoValueKnownState = {
246
- id: theirKnown.id,
247
- header: !!coValueRow,
248
- sessions: {},
249
- };
250
-
251
- let parsedHeader;
252
-
253
- try {
254
- parsedHeader = (coValueRow?.header &&
255
- JSON.parse(coValueRow.header)) as
256
- | CojsonInternalTypes.CoValueHeader
257
- | undefined;
258
- } catch (e) {
268
+ const priority = cojsonInternals.getPriorityFromHeader(parsedHeader);
269
+ const newContentPieces: CojsonInternalTypes.NewContentMessage[] = [
270
+ {
271
+ action: "content",
272
+ id: theirKnown.id,
273
+ header: theirKnown.header ? undefined : parsedHeader,
274
+ new: {},
275
+ priority,
276
+ },
277
+ ];
278
+
279
+ for (const sessionRow of allOurSessions) {
280
+ ourKnown.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
281
+
282
+ if (
283
+ sessionRow.lastIdx > (theirKnown.sessions[sessionRow.sessionID] || 0)
284
+ ) {
285
+ const firstNewTxIdx = theirKnown.sessions[sessionRow.sessionID] || 0;
286
+
287
+ const signaturesAndIdxs = this.db
288
+ .prepare<[number, number]>(
289
+ `SELECT * FROM signatureAfter WHERE ses = ? AND idx >= ?`,
290
+ )
291
+ .all(sessionRow.rowID, firstNewTxIdx) as SignatureAfterRow[];
292
+
293
+ // console.log(
294
+ // theirKnown.id,
295
+ // "signaturesAndIdxs",
296
+ // JSON.stringify(signaturesAndIdxs)
297
+ // );
298
+
299
+ const newTxInSession = this.db
300
+ .prepare<[number, number]>(
301
+ `SELECT * FROM transactions WHERE ses = ? AND idx >= ?`,
302
+ )
303
+ .all(sessionRow.rowID, firstNewTxIdx) as TransactionRow[];
304
+
305
+ let idx = firstNewTxIdx;
306
+
307
+ // console.log(
308
+ // theirKnown.id,
309
+ // "newTxInSession",
310
+ // newTxInSession.length
311
+ // );
312
+
313
+ for (const tx of newTxInSession) {
314
+ let sessionEntry =
315
+ newContentPieces[newContentPieces.length - 1]!.new[
316
+ sessionRow.sessionID
317
+ ];
318
+ if (!sessionEntry) {
319
+ sessionEntry = {
320
+ after: idx,
321
+ lastSignature:
322
+ "WILL_BE_REPLACED" as CojsonInternalTypes.Signature,
323
+ newTransactions: [],
324
+ };
325
+ newContentPieces[newContentPieces.length - 1]!.new[
326
+ sessionRow.sessionID
327
+ ] = sessionEntry;
328
+ }
329
+
330
+ let parsedTx;
331
+
332
+ try {
333
+ parsedTx = JSON.parse(tx.tx);
334
+ } catch (e) {
259
335
  console.warn(
260
- theirKnown.id,
261
- "Invalid JSON in header",
262
- e,
263
- coValueRow?.header,
264
- );
265
- return;
266
- }
267
-
268
- const priority = cojsonInternals.getPriorityFromHeader(parsedHeader);
269
- const newContentPieces: CojsonInternalTypes.NewContentMessage[] = [
270
- {
271
- action: "content",
272
- id: theirKnown.id,
273
- header: theirKnown.header ? undefined : parsedHeader,
274
- new: {},
275
- priority,
276
- },
277
- ];
278
-
279
- for (const sessionRow of allOurSessions) {
280
- ourKnown.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
281
-
282
- if (
283
- sessionRow.lastIdx >
284
- (theirKnown.sessions[sessionRow.sessionID] || 0)
285
- ) {
286
- const firstNewTxIdx =
287
- theirKnown.sessions[sessionRow.sessionID] || 0;
288
-
289
- const signaturesAndIdxs = this.db
290
- .prepare<[number, number]>(
291
- `SELECT * FROM signatureAfter WHERE ses = ? AND idx >= ?`,
292
- )
293
- .all(
294
- sessionRow.rowID,
295
- firstNewTxIdx,
296
- ) as SignatureAfterRow[];
297
-
298
- // console.log(
299
- // theirKnown.id,
300
- // "signaturesAndIdxs",
301
- // JSON.stringify(signaturesAndIdxs)
302
- // );
303
-
304
- const newTxInSession = this.db
305
- .prepare<[number, number]>(
306
- `SELECT * FROM transactions WHERE ses = ? AND idx >= ?`,
307
- )
308
- .all(sessionRow.rowID, firstNewTxIdx) as TransactionRow[];
309
-
310
- let idx = firstNewTxIdx;
311
-
312
- // console.log(
313
- // theirKnown.id,
314
- // "newTxInSession",
315
- // newTxInSession.length
316
- // );
317
-
318
- for (const tx of newTxInSession) {
319
- let sessionEntry =
320
- newContentPieces[newContentPieces.length - 1]!.new[
321
- sessionRow.sessionID
322
- ];
323
- if (!sessionEntry) {
324
- sessionEntry = {
325
- after: idx,
326
- lastSignature:
327
- "WILL_BE_REPLACED" as CojsonInternalTypes.Signature,
328
- newTransactions: [],
329
- };
330
- newContentPieces[newContentPieces.length - 1]!.new[
331
- sessionRow.sessionID
332
- ] = sessionEntry;
333
- }
334
-
335
- let parsedTx;
336
-
337
- try {
338
- parsedTx = JSON.parse(tx.tx);
339
- } catch (e) {
340
- console.warn(
341
- theirKnown.id,
342
- "Invalid JSON in transaction",
343
- e,
344
- tx.tx,
345
- );
346
- break;
347
- }
348
-
349
- sessionEntry.newTransactions.push(parsedTx);
350
-
351
- if (
352
- signaturesAndIdxs[0] &&
353
- idx === signaturesAndIdxs[0].idx
354
- ) {
355
- sessionEntry.lastSignature =
356
- signaturesAndIdxs[0].signature;
357
- signaturesAndIdxs.shift();
358
- newContentPieces.push({
359
- action: "content",
360
- id: theirKnown.id,
361
- new: {},
362
- priority,
363
- });
364
- } else if (
365
- idx ===
366
- firstNewTxIdx + newTxInSession.length - 1
367
- ) {
368
- sessionEntry.lastSignature = sessionRow.lastSignature;
369
- }
370
- idx += 1;
371
- }
372
- }
373
- }
374
-
375
- const dependedOnCoValues =
376
- parsedHeader?.ruleset.type === "group"
377
- ? newContentPieces
378
- .flatMap((piece) => Object.values(piece.new))
379
- .flatMap((sessionEntry) =>
380
- sessionEntry.newTransactions.flatMap((tx) => {
381
- if (tx.privacy !== "trusting") return [];
382
- // TODO: avoid parsing here?
383
- let parsedChanges;
384
-
385
- try {
386
- parsedChanges = cojsonInternals.parseJSON(
387
- tx.changes,
388
- );
389
- } catch (e) {
390
- console.warn(
391
- theirKnown.id,
392
- "Invalid JSON in transaction",
393
- e,
394
- tx.changes,
395
- );
396
- return [];
397
- }
398
-
399
- return parsedChanges
400
- .map(
401
- (change) =>
402
- change &&
403
- typeof change === "object" &&
404
- "op" in change &&
405
- change.op === "set" &&
406
- "key" in change &&
407
- change.key,
408
- )
409
- .filter(
410
- (
411
- key,
412
- ): key is CojsonInternalTypes.RawCoID =>
413
- typeof key === "string" &&
414
- key.startsWith("co_"),
415
- );
416
- }),
417
- )
418
- : parsedHeader?.ruleset.type === "ownedByGroup"
419
- ? [
420
- parsedHeader?.ruleset.group,
421
- ...new Set(
422
- newContentPieces.flatMap((piece) =>
423
- Object.keys(piece)
424
- .map((sessionID) =>
425
- cojsonInternals.accountOrAgentIDfromSessionID(
426
- sessionID as SessionID,
427
- ),
428
- )
429
- .filter(
430
- (accountID): accountID is RawAccountID =>
431
- cojsonInternals.isAccountID(
432
- accountID,
433
- ) && accountID !== theirKnown.id,
434
- ),
435
- ),
436
- ),
437
- ]
438
- : [];
439
-
440
- for (const dependedOnCoValue of dependedOnCoValues) {
441
- await this.sendNewContentAfter(
442
- { id: dependedOnCoValue, header: false, sessions: {} },
443
- asDependencyOf || theirKnown.id,
336
+ theirKnown.id,
337
+ "Invalid JSON in transaction",
338
+ e,
339
+ tx.tx,
444
340
  );
341
+ break;
342
+ }
343
+
344
+ sessionEntry.newTransactions.push(parsedTx);
345
+
346
+ if (signaturesAndIdxs[0] && idx === signaturesAndIdxs[0].idx) {
347
+ sessionEntry.lastSignature = signaturesAndIdxs[0].signature;
348
+ signaturesAndIdxs.shift();
349
+ newContentPieces.push({
350
+ action: "content",
351
+ id: theirKnown.id,
352
+ new: {},
353
+ priority,
354
+ });
355
+ } else if (idx === firstNewTxIdx + newTxInSession.length - 1) {
356
+ sessionEntry.lastSignature = sessionRow.lastSignature;
357
+ }
358
+ idx += 1;
445
359
  }
360
+ }
361
+ }
446
362
 
447
- this.toLocalNode
448
- .push({
449
- action: "known",
450
- ...ourKnown,
451
- asDependencyOf,
452
- })
453
- .catch((e) => console.error("Error while pushing known", e));
454
-
455
- const nonEmptyNewContentPieces = newContentPieces.filter(
456
- (piece) => piece.header || Object.keys(piece.new).length > 0,
457
- );
363
+ const dependedOnCoValues =
364
+ parsedHeader?.ruleset.type === "group"
365
+ ? newContentPieces
366
+ .flatMap((piece) => Object.values(piece.new))
367
+ .flatMap((sessionEntry) =>
368
+ sessionEntry.newTransactions.flatMap((tx) => {
369
+ if (tx.privacy !== "trusting") return [];
370
+ // TODO: avoid parsing here?
371
+ let parsedChanges;
458
372
 
459
- // console.log(theirKnown.id, nonEmptyNewContentPieces);
373
+ try {
374
+ parsedChanges = cojsonInternals.parseJSON(tx.changes);
375
+ } catch (e) {
376
+ console.warn(
377
+ theirKnown.id,
378
+ "Invalid JSON in transaction",
379
+ e,
380
+ tx.changes,
381
+ );
382
+ return [];
383
+ }
460
384
 
461
- for (const piece of nonEmptyNewContentPieces) {
462
- this.toLocalNode
463
- .push(piece)
464
- .catch((e) =>
465
- console.error("Error while pushing content piece", e),
466
- );
467
- await new Promise((resolve) => setTimeout(resolve, 0));
468
- }
385
+ return parsedChanges
386
+ .map(
387
+ (change) =>
388
+ change &&
389
+ typeof change === "object" &&
390
+ "op" in change &&
391
+ change.op === "set" &&
392
+ "key" in change &&
393
+ change.key,
394
+ )
395
+ .filter(
396
+ (key): key is CojsonInternalTypes.RawCoID =>
397
+ typeof key === "string" && key.startsWith("co_"),
398
+ );
399
+ }),
400
+ )
401
+ : parsedHeader?.ruleset.type === "ownedByGroup"
402
+ ? [
403
+ parsedHeader?.ruleset.group,
404
+ ...new Set(
405
+ newContentPieces.flatMap((piece) =>
406
+ Object.keys(piece)
407
+ .map((sessionID) =>
408
+ cojsonInternals.accountOrAgentIDfromSessionID(
409
+ sessionID as SessionID,
410
+ ),
411
+ )
412
+ .filter(
413
+ (accountID): accountID is RawAccountID =>
414
+ cojsonInternals.isAccountID(accountID) &&
415
+ accountID !== theirKnown.id,
416
+ ),
417
+ ),
418
+ ),
419
+ ]
420
+ : [];
421
+
422
+ for (const dependedOnCoValue of dependedOnCoValues) {
423
+ await this.sendNewContentAfter(
424
+ { id: dependedOnCoValue, header: false, sessions: {} },
425
+ asDependencyOf || theirKnown.id,
426
+ );
469
427
  }
470
428
 
471
- handleLoad(msg: CojsonInternalTypes.LoadMessage) {
472
- return this.sendNewContentAfter(msg);
429
+ this.toLocalNode
430
+ .push({
431
+ action: "known",
432
+ ...ourKnown,
433
+ asDependencyOf,
434
+ })
435
+ .catch((e) => console.error("Error while pushing known", e));
436
+
437
+ const nonEmptyNewContentPieces = newContentPieces.filter(
438
+ (piece) => piece.header || Object.keys(piece.new).length > 0,
439
+ );
440
+
441
+ // console.log(theirKnown.id, nonEmptyNewContentPieces);
442
+
443
+ for (const piece of nonEmptyNewContentPieces) {
444
+ this.toLocalNode
445
+ .push(piece)
446
+ .catch((e) => console.error("Error while pushing content piece", e));
447
+ await new Promise((resolve) => setTimeout(resolve, 0));
448
+ }
449
+ }
450
+
451
+ handleLoad(msg: CojsonInternalTypes.LoadMessage) {
452
+ return this.sendNewContentAfter(msg);
453
+ }
454
+
455
+ async handleContent(msg: CojsonInternalTypes.NewContentMessage) {
456
+ let storedCoValueRowID = (
457
+ this.db
458
+ .prepare<CojsonInternalTypes.RawCoID>(
459
+ `SELECT rowID FROM coValues WHERE id = ?`,
460
+ )
461
+ .get(msg.id) as StoredCoValueRow | undefined
462
+ )?.rowID;
463
+
464
+ if (storedCoValueRowID === undefined) {
465
+ const header = msg.header;
466
+ if (!header) {
467
+ console.error("Expected to be sent header first");
468
+ this.toLocalNode
469
+ .push({
470
+ action: "known",
471
+ id: msg.id,
472
+ header: false,
473
+ sessions: {},
474
+ isCorrection: true,
475
+ })
476
+ .catch((e) => console.error("Error while pushing known", e));
477
+ return;
478
+ }
479
+
480
+ storedCoValueRowID = this.db
481
+ .prepare<[CojsonInternalTypes.RawCoID, string]>(
482
+ `INSERT INTO coValues (id, header) VALUES (?, ?)`,
483
+ )
484
+ .run(msg.id, JSON.stringify(header)).lastInsertRowid as number;
473
485
  }
474
486
 
475
- async handleContent(msg: CojsonInternalTypes.NewContentMessage) {
476
- let storedCoValueRowID = (
477
- this.db
478
- .prepare<CojsonInternalTypes.RawCoID>(
479
- `SELECT rowID FROM coValues WHERE id = ?`,
480
- )
481
- .get(msg.id) as StoredCoValueRow | undefined
482
- )?.rowID;
483
-
484
- if (storedCoValueRowID === undefined) {
485
- const header = msg.header;
486
- if (!header) {
487
- console.error("Expected to be sent header first");
488
- this.toLocalNode
489
- .push({
490
- action: "known",
491
- id: msg.id,
492
- header: false,
493
- sessions: {},
494
- isCorrection: true,
495
- })
496
- .catch((e) =>
497
- console.error("Error while pushing known", e),
498
- );
499
- return;
500
- }
501
-
502
- storedCoValueRowID = this.db
503
- .prepare<[CojsonInternalTypes.RawCoID, string]>(
504
- `INSERT INTO coValues (id, header) VALUES (?, ?)`,
505
- )
506
- .run(msg.id, JSON.stringify(header)).lastInsertRowid as number;
487
+ const ourKnown: CojsonInternalTypes.CoValueKnownState = {
488
+ id: msg.id,
489
+ header: true,
490
+ sessions: {},
491
+ };
492
+ let invalidAssumptions = false;
493
+
494
+ this.db.transaction(() => {
495
+ const allOurSessions = (
496
+ this.db
497
+ .prepare<number>(`SELECT * FROM sessions WHERE coValue = ?`)
498
+ .all(storedCoValueRowID!) as StoredSessionRow[]
499
+ ).reduce(
500
+ (acc, row) => {
501
+ acc[row.sessionID] = row;
502
+ return acc;
503
+ },
504
+ {} as { [sessionID: string]: StoredSessionRow },
505
+ );
506
+
507
+ for (const sessionID of Object.keys(msg.new) as SessionID[]) {
508
+ const sessionRow = allOurSessions[sessionID];
509
+ if (sessionRow) {
510
+ ourKnown.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
507
511
  }
508
512
 
509
- const ourKnown: CojsonInternalTypes.CoValueKnownState = {
510
- id: msg.id,
511
- header: true,
512
- sessions: {},
513
- };
514
- let invalidAssumptions = false;
515
-
516
- this.db.transaction(() => {
517
- const allOurSessions = (
518
- this.db
519
- .prepare<number>(`SELECT * FROM sessions WHERE coValue = ?`)
520
- .all(storedCoValueRowID!) as StoredSessionRow[]
521
- ).reduce(
522
- (acc, row) => {
523
- acc[row.sessionID] = row;
524
- return acc;
525
- },
526
- {} as { [sessionID: string]: StoredSessionRow },
513
+ if ((sessionRow?.lastIdx || 0) < (msg.new[sessionID]?.after || 0)) {
514
+ invalidAssumptions = true;
515
+ } else {
516
+ const newTransactions = msg.new[sessionID]?.newTransactions || [];
517
+
518
+ const actuallyNewOffset =
519
+ (sessionRow?.lastIdx || 0) - (msg.new[sessionID]?.after || 0);
520
+
521
+ const actuallyNewTransactions =
522
+ newTransactions.slice(actuallyNewOffset);
523
+
524
+ let newBytesSinceLastSignature =
525
+ (sessionRow?.bytesSinceLastSignature || 0) +
526
+ actuallyNewTransactions.reduce(
527
+ (sum, tx) =>
528
+ sum +
529
+ (tx.privacy === "private"
530
+ ? tx.encryptedChanges.length
531
+ : tx.changes.length),
532
+ 0,
527
533
  );
528
534
 
529
- for (const sessionID of Object.keys(msg.new) as SessionID[]) {
530
- const sessionRow = allOurSessions[sessionID];
531
- if (sessionRow) {
532
- ourKnown.sessions[sessionRow.sessionID] =
533
- sessionRow.lastIdx;
534
- }
535
+ const newLastIdx =
536
+ (sessionRow?.lastIdx || 0) + actuallyNewTransactions.length;
535
537
 
536
- if (
537
- (sessionRow?.lastIdx || 0) <
538
- (msg.new[sessionID]?.after || 0)
539
- ) {
540
- invalidAssumptions = true;
541
- } else {
542
- const newTransactions =
543
- msg.new[sessionID]?.newTransactions || [];
544
-
545
- const actuallyNewOffset =
546
- (sessionRow?.lastIdx || 0) -
547
- (msg.new[sessionID]?.after || 0);
548
-
549
- const actuallyNewTransactions =
550
- newTransactions.slice(actuallyNewOffset);
551
-
552
- let newBytesSinceLastSignature =
553
- (sessionRow?.bytesSinceLastSignature || 0) +
554
- actuallyNewTransactions.reduce(
555
- (sum, tx) =>
556
- sum +
557
- (tx.privacy === "private"
558
- ? tx.encryptedChanges.length
559
- : tx.changes.length),
560
- 0,
561
- );
562
-
563
- const newLastIdx =
564
- (sessionRow?.lastIdx || 0) +
565
- actuallyNewTransactions.length;
566
-
567
- let shouldWriteSignature = false;
568
-
569
- if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
570
- shouldWriteSignature = true;
571
- newBytesSinceLastSignature = 0;
572
- }
573
-
574
- let nextIdx = sessionRow?.lastIdx || 0;
575
-
576
- const sessionUpdate = {
577
- coValue: storedCoValueRowID!,
578
- sessionID: sessionID,
579
- lastIdx: newLastIdx,
580
- lastSignature: msg.new[sessionID]!.lastSignature,
581
- bytesSinceLastSignature: newBytesSinceLastSignature,
582
- };
583
-
584
- const upsertedSession = this.db
585
- .prepare<[number, string, number, string, number]>(
586
- `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature, bytesSinceLastSignature) VALUES (?, ?, ?, ?, ?)
538
+ let shouldWriteSignature = false;
539
+
540
+ if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
541
+ shouldWriteSignature = true;
542
+ newBytesSinceLastSignature = 0;
543
+ }
544
+
545
+ let nextIdx = sessionRow?.lastIdx || 0;
546
+
547
+ const sessionUpdate = {
548
+ coValue: storedCoValueRowID!,
549
+ sessionID: sessionID,
550
+ lastIdx: newLastIdx,
551
+ lastSignature: msg.new[sessionID]!.lastSignature,
552
+ bytesSinceLastSignature: newBytesSinceLastSignature,
553
+ };
554
+
555
+ const upsertedSession = this.db
556
+ .prepare<[number, string, number, string, number]>(
557
+ `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature, bytesSinceLastSignature) VALUES (?, ?, ?, ?, ?)
587
558
  ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature, bytesSinceLastSignature=excluded.bytesSinceLastSignature
588
559
  RETURNING rowID`,
589
- )
590
- .get(
591
- sessionUpdate.coValue,
592
- sessionUpdate.sessionID,
593
- sessionUpdate.lastIdx,
594
- sessionUpdate.lastSignature,
595
- sessionUpdate.bytesSinceLastSignature,
596
- ) as { rowID: number };
597
-
598
- const sessionRowID = upsertedSession.rowID;
599
-
600
- if (shouldWriteSignature) {
601
- this.db
602
- .prepare<[number, number, string]>(
603
- `INSERT INTO signatureAfter (ses, idx, signature) VALUES (?, ?, ?)`,
604
- )
605
- .run(
606
- sessionRowID,
607
- // TODO: newLastIdx is a misnomer, it's actually more like nextIdx or length
608
- newLastIdx - 1,
609
- msg.new[sessionID]!.lastSignature,
610
- );
611
- }
612
-
613
- for (const newTransaction of actuallyNewTransactions) {
614
- this.db
615
- .prepare<[number, number, string]>(
616
- `INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)`,
617
- )
618
- .run(
619
- sessionRowID,
620
- nextIdx,
621
- JSON.stringify(newTransaction),
622
- );
623
- nextIdx++;
624
- }
625
- }
626
- }
627
- })();
628
-
629
- if (invalidAssumptions) {
630
- this.toLocalNode
631
- .push({
632
- action: "known",
633
- ...ourKnown,
634
- isCorrection: invalidAssumptions,
635
- })
636
- .catch((e) => console.error("Error while pushing known", e));
560
+ )
561
+ .get(
562
+ sessionUpdate.coValue,
563
+ sessionUpdate.sessionID,
564
+ sessionUpdate.lastIdx,
565
+ sessionUpdate.lastSignature,
566
+ sessionUpdate.bytesSinceLastSignature,
567
+ ) as { rowID: number };
568
+
569
+ const sessionRowID = upsertedSession.rowID;
570
+
571
+ if (shouldWriteSignature) {
572
+ this.db
573
+ .prepare<[number, number, string]>(
574
+ `INSERT INTO signatureAfter (ses, idx, signature) VALUES (?, ?, ?)`,
575
+ )
576
+ .run(
577
+ sessionRowID,
578
+ // TODO: newLastIdx is a misnomer, it's actually more like nextIdx or length
579
+ newLastIdx - 1,
580
+ msg.new[sessionID]!.lastSignature,
581
+ );
582
+ }
583
+
584
+ for (const newTransaction of actuallyNewTransactions) {
585
+ this.db
586
+ .prepare<[number, number, string]>(
587
+ `INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)`,
588
+ )
589
+ .run(sessionRowID, nextIdx, JSON.stringify(newTransaction));
590
+ nextIdx++;
591
+ }
637
592
  }
593
+ }
594
+ })();
595
+
596
+ if (invalidAssumptions) {
597
+ this.toLocalNode
598
+ .push({
599
+ action: "known",
600
+ ...ourKnown,
601
+ isCorrection: invalidAssumptions,
602
+ })
603
+ .catch((e) => console.error("Error while pushing known", e));
638
604
  }
605
+ }
639
606
 
640
- handleKnown(msg: CojsonInternalTypes.KnownStateMessage) {
641
- return this.sendNewContentAfter(msg);
642
- }
607
+ handleKnown(msg: CojsonInternalTypes.KnownStateMessage) {
608
+ return this.sendNewContentAfter(msg);
609
+ }
643
610
 
644
- handleDone(_msg: CojsonInternalTypes.DoneMessage) {}
611
+ handleDone(_msg: CojsonInternalTypes.DoneMessage) {}
645
612
  }