@powersync/service-module-mongodb-storage 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/migrations/db/migrations/1741697235857-bucket-state-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1741697235857-bucket-state-index.js +28 -0
- package/dist/migrations/db/migrations/1741697235857-bucket-state-index.js.map +1 -0
- package/dist/storage/implementation/MongoCompactor.js +3 -1
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +7 -4
- package/dist/storage/implementation/MongoSyncBucketStorage.js +128 -31
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +15 -2
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +193 -17
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/PersistedBatch.d.ts +8 -0
- package/dist/storage/implementation/PersistedBatch.js +44 -0
- package/dist/storage/implementation/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +2 -1
- package/dist/storage/implementation/db.js +4 -0
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +17 -0
- package/package.json +4 -4
- package/src/migrations/db/migrations/1741697235857-bucket-state-index.ts +40 -0
- package/src/storage/implementation/MongoCompactor.ts +3 -1
- package/src/storage/implementation/MongoSyncBucketStorage.ts +164 -35
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +262 -25
- package/src/storage/implementation/PersistedBatch.ts +52 -0
- package/src/storage/implementation/db.ts +5 -0
- package/src/storage/implementation/models.ts +18 -0
- package/test/src/__snapshots__/storage_sync.test.ts.snap +171 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import * as framework from '@powersync/lib-services-framework';
|
|
2
|
-
import { storage } from '@powersync/service-core';
|
|
2
|
+
import { Demultiplexer, storage } from '@powersync/service-core';
|
|
3
3
|
export class MongoWriteCheckpointAPI {
|
|
4
4
|
db;
|
|
5
5
|
_mode;
|
|
6
|
+
sync_rules_id;
|
|
6
7
|
constructor(options) {
|
|
7
8
|
this.db = options.db;
|
|
8
9
|
this._mode = options.mode;
|
|
10
|
+
this.sync_rules_id = options.sync_rules_id;
|
|
9
11
|
}
|
|
10
12
|
get writeCheckpointMode() {
|
|
11
13
|
return this._mode;
|
|
@@ -16,21 +18,6 @@ export class MongoWriteCheckpointAPI {
|
|
|
16
18
|
async batchCreateCustomWriteCheckpoints(checkpoints) {
|
|
17
19
|
return batchCreateCustomWriteCheckpoints(this.db, checkpoints);
|
|
18
20
|
}
|
|
19
|
-
async createCustomWriteCheckpoint(options) {
|
|
20
|
-
if (this.writeCheckpointMode !== storage.WriteCheckpointMode.CUSTOM) {
|
|
21
|
-
throw new framework.errors.ValidationError(`Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`);
|
|
22
|
-
}
|
|
23
|
-
const { checkpoint, user_id, sync_rules_id } = options;
|
|
24
|
-
const doc = await this.db.custom_write_checkpoints.findOneAndUpdate({
|
|
25
|
-
user_id: user_id,
|
|
26
|
-
sync_rules_id
|
|
27
|
-
}, {
|
|
28
|
-
$set: {
|
|
29
|
-
checkpoint
|
|
30
|
-
}
|
|
31
|
-
}, { upsert: true, returnDocument: 'after' });
|
|
32
|
-
return doc.checkpoint;
|
|
33
|
-
}
|
|
34
21
|
async createManagedWriteCheckpoint(checkpoint) {
|
|
35
22
|
if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) {
|
|
36
23
|
throw new framework.errors.ValidationError(`Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`);
|
|
@@ -62,6 +49,174 @@ export class MongoWriteCheckpointAPI {
|
|
|
62
49
|
return this.lastManagedWriteCheckpoint(filters);
|
|
63
50
|
}
|
|
64
51
|
}
|
|
52
|
+
watchUserWriteCheckpoint(options) {
|
|
53
|
+
switch (this.writeCheckpointMode) {
|
|
54
|
+
case storage.WriteCheckpointMode.CUSTOM:
|
|
55
|
+
return this.watchCustomWriteCheckpoint(options);
|
|
56
|
+
case storage.WriteCheckpointMode.MANAGED:
|
|
57
|
+
return this.watchManagedWriteCheckpoint(options);
|
|
58
|
+
default:
|
|
59
|
+
throw new Error('Invalid write checkpoint mode');
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
sharedManagedIter = new Demultiplexer((signal) => {
|
|
63
|
+
const clusterTimePromise = this.getClusterTime();
|
|
64
|
+
return {
|
|
65
|
+
iterator: this.watchAllManagedWriteCheckpoints(clusterTimePromise, signal),
|
|
66
|
+
getFirstValue: async (user_id) => {
|
|
67
|
+
// Potential race conditions we cater for:
|
|
68
|
+
// Case 1: changestream is behind.
|
|
69
|
+
// We get a doc now, then the same or older doc again later.
|
|
70
|
+
// No problem!
|
|
71
|
+
// Case 2: Query is behind. I.e. doc has been created, and emitted on the changestream, but the query doesn't see it yet.
|
|
72
|
+
// Not possible luckily, but can we make sure?
|
|
73
|
+
// Case 3: changestream delays openeing. A doc is created after our query here, but before the changestream is opened.
|
|
74
|
+
// Awaiting clusterTimePromise should be sufficient here, but as a sanity check we also confirm that our query
|
|
75
|
+
// timestamp is > the startClusterTime.
|
|
76
|
+
const changeStreamStart = await clusterTimePromise;
|
|
77
|
+
let doc = null;
|
|
78
|
+
let clusterTime = null;
|
|
79
|
+
await this.db.client.withSession(async (session) => {
|
|
80
|
+
doc = await this.db.write_checkpoints.findOne({
|
|
81
|
+
user_id: user_id
|
|
82
|
+
}, {
|
|
83
|
+
session
|
|
84
|
+
});
|
|
85
|
+
const time = session.clusterTime?.clusterTime ?? null;
|
|
86
|
+
clusterTime = time;
|
|
87
|
+
});
|
|
88
|
+
if (clusterTime == null) {
|
|
89
|
+
throw new framework.ServiceAssertionError('Could not get clusterTime for write checkpoint');
|
|
90
|
+
}
|
|
91
|
+
if (clusterTime.lessThan(changeStreamStart)) {
|
|
92
|
+
throw new framework.ServiceAssertionError('clusterTime for write checkpoint is older than changestream start');
|
|
93
|
+
}
|
|
94
|
+
if (doc == null) {
|
|
95
|
+
return {
|
|
96
|
+
id: null,
|
|
97
|
+
lsn: null
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
id: doc.client_id,
|
|
102
|
+
lsn: doc.lsns['1']
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
});
|
|
107
|
+
async *watchAllManagedWriteCheckpoints(clusterTimePromise, signal) {
|
|
108
|
+
const clusterTime = await clusterTimePromise;
|
|
109
|
+
const stream = this.db.write_checkpoints.watch([{ $match: { operationType: { $in: ['insert', 'update', 'replace'] } } }], {
|
|
110
|
+
fullDocument: 'updateLookup',
|
|
111
|
+
startAtOperationTime: clusterTime
|
|
112
|
+
});
|
|
113
|
+
signal.onabort = () => {
|
|
114
|
+
stream.close();
|
|
115
|
+
};
|
|
116
|
+
if (signal.aborted) {
|
|
117
|
+
stream.close();
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
for await (let event of stream) {
|
|
121
|
+
if (!('fullDocument' in event) || event.fullDocument == null) {
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
const user_id = event.fullDocument.user_id;
|
|
125
|
+
yield {
|
|
126
|
+
key: user_id,
|
|
127
|
+
value: {
|
|
128
|
+
id: event.fullDocument.client_id,
|
|
129
|
+
lsn: event.fullDocument.lsns['1']
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
watchManagedWriteCheckpoint(options) {
|
|
135
|
+
const stream = this.sharedManagedIter.subscribe(options.user_id, options.signal);
|
|
136
|
+
return this.orderedStream(stream);
|
|
137
|
+
}
|
|
138
|
+
sharedCustomIter = new Demultiplexer((signal) => {
|
|
139
|
+
const clusterTimePromise = this.getClusterTime();
|
|
140
|
+
return {
|
|
141
|
+
iterator: this.watchAllCustomWriteCheckpoints(clusterTimePromise, signal),
|
|
142
|
+
getFirstValue: async (user_id) => {
|
|
143
|
+
// We cater for the same potential race conditions as for managed write checkpoints.
|
|
144
|
+
const changeStreamStart = await clusterTimePromise;
|
|
145
|
+
let doc = null;
|
|
146
|
+
let clusterTime = null;
|
|
147
|
+
await this.db.client.withSession(async (session) => {
|
|
148
|
+
doc = await this.db.custom_write_checkpoints.findOne({
|
|
149
|
+
user_id: user_id,
|
|
150
|
+
sync_rules_id: this.sync_rules_id
|
|
151
|
+
}, {
|
|
152
|
+
session
|
|
153
|
+
});
|
|
154
|
+
const time = session.clusterTime?.clusterTime ?? null;
|
|
155
|
+
clusterTime = time;
|
|
156
|
+
});
|
|
157
|
+
if (clusterTime == null) {
|
|
158
|
+
throw new framework.ServiceAssertionError('Could not get clusterTime for write checkpoint');
|
|
159
|
+
}
|
|
160
|
+
if (clusterTime.lessThan(changeStreamStart)) {
|
|
161
|
+
throw new framework.ServiceAssertionError('clusterTime for write checkpoint is older than changestream start');
|
|
162
|
+
}
|
|
163
|
+
if (doc == null) {
|
|
164
|
+
// No write checkpoint, but we still need to return a result
|
|
165
|
+
return {
|
|
166
|
+
id: null,
|
|
167
|
+
lsn: null
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
return {
|
|
171
|
+
id: doc.checkpoint,
|
|
172
|
+
// custom write checkpoints are not tied to a LSN
|
|
173
|
+
lsn: null
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
});
|
|
178
|
+
async *watchAllCustomWriteCheckpoints(clusterTimePromise, signal) {
|
|
179
|
+
const clusterTime = await clusterTimePromise;
|
|
180
|
+
const stream = this.db.custom_write_checkpoints.watch([
|
|
181
|
+
{
|
|
182
|
+
$match: {
|
|
183
|
+
'fullDocument.sync_rules_id': this.sync_rules_id,
|
|
184
|
+
operationType: { $in: ['insert', 'update', 'replace'] }
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
], {
|
|
188
|
+
fullDocument: 'updateLookup',
|
|
189
|
+
startAtOperationTime: clusterTime
|
|
190
|
+
});
|
|
191
|
+
signal.onabort = () => {
|
|
192
|
+
stream.close();
|
|
193
|
+
};
|
|
194
|
+
if (signal.aborted) {
|
|
195
|
+
stream.close();
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
for await (let event of stream) {
|
|
199
|
+
if (!('fullDocument' in event) || event.fullDocument == null) {
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
const user_id = event.fullDocument.user_id;
|
|
203
|
+
yield {
|
|
204
|
+
key: user_id,
|
|
205
|
+
value: {
|
|
206
|
+
id: event.fullDocument.checkpoint,
|
|
207
|
+
// Custom write checkpoints are not tied to a specific LSN
|
|
208
|
+
lsn: null
|
|
209
|
+
}
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
watchCustomWriteCheckpoint(options) {
|
|
214
|
+
if (options.sync_rules_id != this.sync_rules_id) {
|
|
215
|
+
throw new framework.ServiceAssertionError('sync_rules_id does not match');
|
|
216
|
+
}
|
|
217
|
+
const stream = this.sharedCustomIter.subscribe(options.user_id, options.signal);
|
|
218
|
+
return this.orderedStream(stream);
|
|
219
|
+
}
|
|
65
220
|
async lastCustomWriteCheckpoint(filters) {
|
|
66
221
|
const { user_id, sync_rules_id } = filters;
|
|
67
222
|
const lastWriteCheckpoint = await this.db.custom_write_checkpoints.findOne({
|
|
@@ -84,9 +239,30 @@ export class MongoWriteCheckpointAPI {
|
|
|
84
239
|
});
|
|
85
240
|
return lastWriteCheckpoint?.client_id ?? null;
|
|
86
241
|
}
|
|
242
|
+
async getClusterTime() {
|
|
243
|
+
const hello = await this.db.db.command({ hello: 1 });
|
|
244
|
+
// Note: This is not valid on sharded clusters.
|
|
245
|
+
const startClusterTime = hello.lastWrite?.majorityOpTime?.ts;
|
|
246
|
+
return startClusterTime;
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Makes a write checkpoint stream an orderered one - any out-of-order events are discarded.
|
|
250
|
+
*/
|
|
251
|
+
async *orderedStream(stream) {
|
|
252
|
+
let lastId = -1n;
|
|
253
|
+
for await (let event of stream) {
|
|
254
|
+
// Guard against out-of-order events
|
|
255
|
+
if (lastId == -1n || (event.id != null && event.id > lastId)) {
|
|
256
|
+
yield event;
|
|
257
|
+
if (event.id != null) {
|
|
258
|
+
lastId = event.id;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
87
263
|
}
|
|
88
264
|
export async function batchCreateCustomWriteCheckpoints(db, checkpoints) {
|
|
89
|
-
if (
|
|
265
|
+
if (checkpoints.length == 0) {
|
|
90
266
|
return;
|
|
91
267
|
}
|
|
92
268
|
await db.custom_write_checkpoints.bulkWrite(checkpoints.map((checkpointOptions) => ({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoWriteCheckpointAPI.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoWriteCheckpointAPI.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"MongoWriteCheckpointAPI.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoWriteCheckpointAPI.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,mCAAmC,CAAC;AAC/D,OAAO,EACL,aAAa,EAEb,OAAO,EAGR,MAAM,yBAAyB,CAAC;AAUjC,MAAM,OAAO,uBAAuB;IACzB,EAAE,CAAiB;IACpB,KAAK,CAA8B;IACnC,aAAa,CAAS;IAE9B,YAAY,OAAkC;QAC5C,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;QAC1B,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAC7C,CAAC;IAED,IAAI,mBAAmB;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,sBAAsB,CAAC,IAAiC;QACtD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,iCAAiC,CAAC,WAAmD;QACzF,OAAO,iCAAiC,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,4BAA4B,CAAC,UAAiD;QAClF,IAAI,IAAI,CAAC,mBAAmB,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;YACrE,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,CACxC,qGAAqG,IAAI,CAAC,mBAAmB,GAAG,CACjI,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC;QAC5C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,gBAAgB,CAC1D;YACE,OAAO,EAAE,OAAO;SACjB,EACD;YACE,IAAI,EAAE;gBACJ,IAAI;aACL;YACD,IAAI,EAAE;gBACJ,SAAS,EAAE,EAAE;aACd;SACF,EACD,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,CAC1C,CAAC;QACF,OAAO,GAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,OAA2C;QACnE,QAAQ,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACjC,KAAK,OAAO,CAAC,mBAAmB,CAAC,MAAM;gBACrC,IAAI,KAAK,IAAI,eAAe,IAAI,OAAO,EAAE,CAAC;oBACxC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,iEAAiE,CAAC,CAAC;gBAChH,CAAC;gBACD,OAAO,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC;YACjD,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO;gBACtC,IAAI,KAAK,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;oBAChC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,CACxC,qEAAqE,CACtE,CAAC;gBACJ,CAAC;gBACD,OAAO,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,wBAAwB,CAAC,OAAwC;QAC/D,QAAQ,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACjC,KAAK,OAAO,CAAC,mBAAmB,CAAC,MAAM;gBACrC,OAAO,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,CAAC;YAClD,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO;gBACtC,OAAO,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;YACnD;gBACE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAEO,iBAAiB,GAAG,IAAI,aAAa,CAAwB,CAAC,MAAM,EAAE,EAAE;QAC9E,MAAM,kBAAkB,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAEjD,OAAO;YACL,QAAQ,EAAE,IAAI,CAAC,+BAA+B,CAAC,kBAAkB,EAAE,MAAM,CAAC;YAC1E,aAAa,EAAE,KAAK,EAAE,OAAe,EAAE,EAAE;gBACvC,0CAA0C;gBAE1C,kCAAkC;gBAClC,4DAA4D;gBAC5D,cAAc;gBAEd,yHAAyH;gBACzH,8CAA8C;gBAE9C,sHAAsH;gBACtH,8GAA8G;gBAC9G,uCAAuC;gBAEvC,MAAM,iBAAiB,GAAG,MAAM,kBAAkB,CAAC;gBAEnD,IAAI,GAAG,GAAG,IAAsC,CAAC;gBACjD,IAAI,WAAW,GAAG,IAA8B,CAAC;gBAEjD,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;oBACjD,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAC3C;wBACE,OAAO,EAAE,OAAO;qBACjB,EACD;wBACE,OAAO;qBACR,CACF,CAAC;oBACF,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,WAAW,IAAI,IAAI,CAAC;oBACtD,WAAW,GAAG,IAAI,CAAC;gBACrB,CAAC,CAAC,CAAC;gBACH,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;oBACxB,MAAM,IAAI,SAAS,CAAC,qBAAqB,CAAC,gDAAgD,CAAC,CAAC;gBAC9F,CAAC;gBAED,IAAI,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;oBAC5C,MAAM,IAAI,SAAS,CAAC,qBAAqB,CACvC,mEAAmE,CACpE,CAAC;gBACJ,CAAC;gBAED,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;oBAChB,OAAO;wBACL,EAAE,EAAE,IAAI;wBACR,GAAG,EAAE,IAAI;qBACV,CAAC;gBACJ,CAAC;gBAED,OAAO;oBACL,EAAE,EAAE,GAAG,CAAC,SAAS;oBACjB,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;iBACnB,CAAC;YACJ,CAAC;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEK,KAAK,CAAC,CAAC,+BAA+B,CAC5C,kBAAiD,EACjD,MAAmB;QAEnB,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAAC;QAE7C,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,KAAK,CAC5C,CAAC,EAAE,MAAM,EAAE,EAAE,aAAa,EAAE,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,EACzE;YACE,YAAY,EAAE,cAAc;YAC5B,oBAAoB,EAAE,WAAW;SAClC,CACF,CAAC;QAEF,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE;YACpB,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,CAAC,CAAC;QAEF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QAED,IAAI,KAAK,EAAE,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;YAC/B,IAAI,CAAC,CAAC,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;gBAC7D,SAAS;YACX,CAAC;YAED,MAAM,OAAO,GAAG,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC;YAC3C,MAAM;gBACJ,GAAG,EAAE,OAAO;gBACZ,KAAK,EAAE;oBACL,EAAE,EAAE,KAAK,CAAC,YAAY,CAAC,SAAS;oBAChC,GAAG,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC;iBAClC;aACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,2BAA2B,CAAC,OAAwC;QAClE,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACjF,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAEO,gBAAgB,GAAG,IAAI,aAAa,CAAwB,CAAC,MAAM,EAAE,EAAE;QAC7E,MAAM,kBAAkB,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAEjD,OAAO;YACL,QAAQ,EAAE,IAAI,CAAC,8BAA8B,CAAC,kBAAkB,EAAE,MAAM,CAAC;YACzE,aAAa,EAAE,KAAK,EAAE,OAAe,EAAE,EAAE;gBACvC,oFAAoF;gBAEpF,MAAM,iBAAiB,GAAG,MAAM,kBAAkB,CAAC;gBAEnD,IAAI,GAAG,GAAG,IAA4C,CAAC;gBACvD,IAAI,WAAW,GAAG,IAA8B,CAAC;gBAEjD,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;oBACjD,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAClD;wBACE,OAAO,EAAE,OAAO;wBAChB,aAAa,EAAE,IAAI,CAAC,aAAa;qBAClC,EACD;wBACE,OAAO;qBACR,CACF,CAAC;oBACF,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,WAAW,IAAI,IAAI,CAAC;oBACtD,WAAW,GAAG,IAAI,CAAC;gBACrB,CAAC,CAAC,CAAC;gBACH,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;oBACxB,MAAM,IAAI,SAAS,CAAC,qBAAqB,CAAC,gDAAgD,CAAC,CAAC;gBAC9F,CAAC;gBAED,IAAI,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;oBAC5C,MAAM,IAAI,SAAS,CAAC,qBAAqB,CACvC,mEAAmE,CACpE,CAAC;gBACJ,CAAC;gBAED,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;oBAChB,4DAA4D;oBAC5D,OAAO;wBACL,EAAE,EAAE,IAAI;wBACR,GAAG,EAAE,IAAI;qBACV,CAAC;gBACJ,CAAC;gBAED,OAAO;oBACL,EAAE,EAAE,GAAG,CAAC,UAAU;oBAClB,iDAAiD;oBACjD,GAAG,EAAE,IAAI;iBACV,CAAC;YACJ,CAAC;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEK,KAAK,CAAC,CAAC,8BAA8B,CAC3C,kBAAiD,EACjD,MAAmB;QAEnB,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAAC;QAE7C,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,wBAAwB,CAAC,KAAK,CACnD;YACE;gBACE,MAAM,EAAE;oBACN,4BAA4B,EAAE,IAAI,CAAC,aAAa;oBAChD,aAAa,EAAE,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE;iBACxD;aACF;SACF,EACD;YACE,YAAY,EAAE,cAAc;YAC5B,oBAAoB,EAAE,WAAW;SAClC,CACF,CAAC;QAEF,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE;YACpB,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,CAAC,CAAC;QAEF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QAED,IAAI,KAAK,EAAE,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;YAC/B,IAAI,CAAC,CAAC,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;gBAC7D,SAAS;YACX,CAAC;YAED,MAAM,OAAO,GAAG,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC;YAC3C,MAAM;gBACJ,GAAG,EAAE,OAAO;gBACZ,KAAK,EAAE;oBACL,EAAE,EAAE,KAAK,CAAC,YAAY,CAAC,UAAU;oBACjC,0DAA0D;oBAC1D,GAAG,EAAE,IAAI;iBACV;aACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,0BAA0B,CAAC,OAAwC;QACjE,IAAI,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAChD,MAAM,IAAI,SAAS,CAAC,qBAAqB,CAAC,8BAA8B,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAChF,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAES,KAAK,CAAC,yBAAyB,CAAC,OAA6C;QACrF,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAC3C,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC;YACzE,OAAO;YACP,aAAa;SACd,CAAC,CAAC;QACH,OAAO,mBAAmB,EAAE,UAAU,IAAI,IAAI,CAAC;IACjD,CAAC;IAES,KAAK,CAAC,0BAA0B,CAAC,OAA8C;QACvF,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QACnC,4EAA4E;QAC5E,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,oDAAoD;YACpD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC;YAClE,OAAO,EAAE,OAAO;YAChB,QAAQ,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;SACxB,CAAC,CAAC;QACH,OAAO,mBAAmB,EAAE,SAAS,IAAI,IAAI,CAAC;IAChD,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACrD,+CAA+C;QAC/C,MAAM,gBAAgB,GAAG,KAAK,CAAC,SAAS,EAAE,cAAc,EAAE,EAAqB,CAAC;QAChF,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,CAAC,aAAa,CAAC,MAAoD;QAC/E,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAEjB,IAAI,KAAK,EAAE,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;YAC/B,oCAAoC;YACpC,IAAI,MAAM,IAAI,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,IAAI,IAAI,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC;gBAC7D,MAAM,KAAK,CAAC;gBACZ,IAAI,KAAK,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;oBACrB,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC;gBACpB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,iCAAiC,CACrD,EAAkB,EAClB,WAAmD;IAEnD,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC5B,OAAO;IACT,CAAC;IAED,MAAM,EAAE,CAAC,wBAAwB,CAAC,SAAS,CACzC,WAAW,CAAC,GAAG,CAAC,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;QACtC,SAAS,EAAE;YACT,MAAM,EAAE,EAAE,OAAO,EAAE,iBAAiB,CAAC,OAAO,EAAE,aAAa,EAAE,iBAAiB,CAAC,aAAa,EAAE;YAC9F,MAAM,EAAE;gBACN,IAAI,EAAE;oBACJ,UAAU,EAAE,iBAAiB,CAAC,UAAU;oBACxC,aAAa,EAAE,iBAAiB,CAAC,aAAa;iBAC/C;aACF;YACD,MAAM,EAAE,IAAI;SACb;KACF,CAAC,CAAC,EACH,EAAE,CACH,CAAC;AACJ,CAAC"}
|
|
@@ -16,6 +16,7 @@ export declare class PersistedBatch {
|
|
|
16
16
|
bucketData: mongo.AnyBulkWriteOperation<BucketDataDocument>[];
|
|
17
17
|
bucketParameters: mongo.AnyBulkWriteOperation<BucketParameterDocument>[];
|
|
18
18
|
currentData: mongo.AnyBulkWriteOperation<CurrentDataDocument>[];
|
|
19
|
+
bucketStates: Map<string, BucketStateUpdate>;
|
|
19
20
|
/**
|
|
20
21
|
* For debug logging only.
|
|
21
22
|
*/
|
|
@@ -25,6 +26,7 @@ export declare class PersistedBatch {
|
|
|
25
26
|
*/
|
|
26
27
|
currentSize: number;
|
|
27
28
|
constructor(group_id: number, writtenSize: number);
|
|
29
|
+
private incrementBucket;
|
|
28
30
|
saveBucketData(options: {
|
|
29
31
|
op_seq: MongoIdSequence;
|
|
30
32
|
sourceKey: storage.ReplicaId;
|
|
@@ -43,4 +45,10 @@ export declare class PersistedBatch {
|
|
|
43
45
|
upsertCurrentData(id: SourceKey, values: Partial<CurrentDataDocument>): void;
|
|
44
46
|
shouldFlushTransaction(): boolean;
|
|
45
47
|
flush(db: PowerSyncMongo, session: mongo.ClientSession): Promise<void>;
|
|
48
|
+
private getBucketStateUpdates;
|
|
46
49
|
}
|
|
50
|
+
interface BucketStateUpdate {
|
|
51
|
+
lastOp: InternalOpId;
|
|
52
|
+
incrementCount: number;
|
|
53
|
+
}
|
|
54
|
+
export {};
|
|
@@ -33,6 +33,7 @@ export class PersistedBatch {
|
|
|
33
33
|
bucketData = [];
|
|
34
34
|
bucketParameters = [];
|
|
35
35
|
currentData = [];
|
|
36
|
+
bucketStates = new Map();
|
|
36
37
|
/**
|
|
37
38
|
* For debug logging only.
|
|
38
39
|
*/
|
|
@@ -45,6 +46,19 @@ export class PersistedBatch {
|
|
|
45
46
|
this.group_id = group_id;
|
|
46
47
|
this.currentSize = writtenSize;
|
|
47
48
|
}
|
|
49
|
+
incrementBucket(bucket, op_id) {
|
|
50
|
+
let existingState = this.bucketStates.get(bucket);
|
|
51
|
+
if (existingState) {
|
|
52
|
+
existingState.lastOp = op_id;
|
|
53
|
+
existingState.incrementCount += 1;
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
this.bucketStates.set(bucket, {
|
|
57
|
+
lastOp: op_id,
|
|
58
|
+
incrementCount: 1
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
}
|
|
48
62
|
saveBucketData(options) {
|
|
49
63
|
const remaining_buckets = new Map();
|
|
50
64
|
for (let b of options.before_buckets) {
|
|
@@ -87,6 +101,7 @@ export class PersistedBatch {
|
|
|
87
101
|
}
|
|
88
102
|
}
|
|
89
103
|
});
|
|
104
|
+
this.incrementBucket(k.bucket, op_id);
|
|
90
105
|
}
|
|
91
106
|
for (let bd of remaining_buckets.values()) {
|
|
92
107
|
// REMOVE
|
|
@@ -111,6 +126,7 @@ export class PersistedBatch {
|
|
|
111
126
|
}
|
|
112
127
|
});
|
|
113
128
|
this.currentSize += 200;
|
|
129
|
+
this.incrementBucket(bd.bucket, op_id);
|
|
114
130
|
}
|
|
115
131
|
}
|
|
116
132
|
saveParameterData(data) {
|
|
@@ -221,13 +237,41 @@ export class PersistedBatch {
|
|
|
221
237
|
ordered: true
|
|
222
238
|
});
|
|
223
239
|
}
|
|
240
|
+
if (this.bucketStates.size > 0) {
|
|
241
|
+
await db.bucket_state.bulkWrite(this.getBucketStateUpdates(), {
|
|
242
|
+
session,
|
|
243
|
+
// Per-bucket operation - order doesn't matter
|
|
244
|
+
ordered: false
|
|
245
|
+
});
|
|
246
|
+
}
|
|
224
247
|
const duration = performance.now() - startAt;
|
|
225
248
|
logger.info(`powersync_${this.group_id} Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${this.currentData.length} updates, ${Math.round(this.currentSize / 1024)}kb in ${duration.toFixed(0)}ms. Last op_id: ${this.debugLastOpId}`);
|
|
226
249
|
this.bucketData = [];
|
|
227
250
|
this.bucketParameters = [];
|
|
228
251
|
this.currentData = [];
|
|
252
|
+
this.bucketStates.clear();
|
|
229
253
|
this.currentSize = 0;
|
|
230
254
|
this.debugLastOpId = null;
|
|
231
255
|
}
|
|
256
|
+
getBucketStateUpdates() {
|
|
257
|
+
return Array.from(this.bucketStates.entries()).map(([bucket, state]) => {
|
|
258
|
+
return {
|
|
259
|
+
updateOne: {
|
|
260
|
+
filter: {
|
|
261
|
+
_id: {
|
|
262
|
+
g: this.group_id,
|
|
263
|
+
b: bucket
|
|
264
|
+
}
|
|
265
|
+
},
|
|
266
|
+
update: {
|
|
267
|
+
$set: {
|
|
268
|
+
last_op: state.lastOp
|
|
269
|
+
}
|
|
270
|
+
},
|
|
271
|
+
upsert: true
|
|
272
|
+
}
|
|
273
|
+
};
|
|
274
|
+
});
|
|
275
|
+
}
|
|
232
276
|
}
|
|
233
277
|
//# sourceMappingURL=PersistedBatch.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../src/storage/implementation/PersistedBatch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAIrD,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAgB,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../src/storage/implementation/PersistedBatch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAIrD,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAgB,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAWvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,UAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IAiBf;IAhBV,UAAU,GAAsD,EAAE,CAAC;IACnE,gBAAgB,GAA2D,EAAE,CAAC;IAC9E,WAAW,GAAuD,EAAE,CAAC;IACrE,YAAY,GAAmC,IAAI,GAAG,EAAE,CAAC;IAEzD;;OAEG;IACH,aAAa,GAAwB,IAAI,CAAC;IAE1C;;OAEG;IACH,WAAW,GAAG,CAAC,CAAC;IAEhB,YACU,QAAgB,EACxB,WAAmB;QADX,aAAQ,GAAR,QAAQ,CAAQ;QAGxB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAEO,eAAe,CAAC,MAAc,EAAE,KAAmB;QACzD,IAAI,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAClD,IAAI,aAAa,EAAE,CAAC;YAClB,aAAa,CAAC,MAAM,GAAG,KAAK,CAAC;YAC7B,aAAa,CAAC,cAAc,IAAI,CAAC,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE;gBAC5B,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,CAAC;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,cAAc,CAAC,OAMd;QACC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAyB,CAAC;QAC3D,KAAK,IAAI,CAAC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QAE3F,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAEhC,SAAS;YACT,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC3D,IAAI,UAAU,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;gBACrC,gGAAgG;gBAChG,iEAAiE;gBACjE,oGAAoG;gBACpG,yBAAyB;gBACzB,MAAM,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,QAAQ,QAAQ,GAAG,eAAe,UAAU,CAAC,MAAM,mBAAmB,CAAC,CAAC;gBACvG,SAAS;YACX,CAAC;YAED,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAE5C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,CAAC,CAAC,MAAM;4BACX,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,KAAK;wBACT,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,MAAM,EAAE,CAAC,CAAC,EAAE;wBACZ,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACxC,CAAC;QAED,KAAK,IAAI,EAAE,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,SAAS;YAET,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,EAAE,CAAC,MAAM;4BACZ,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,QAAQ;wBACZ,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,EAAE,CAAC,KAAK;wBACf,MAAM,EAAE,EAAE,CAAC,EAAE;wBACb,QAAQ,EAAE,SAAS;wBACnB,IAAI,EAAE,IAAI;qBACX;iBACF;aACF,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;YACxB,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,IAMjB;QACC,yCAAyC;QACzC,qEAAqE;QACrE,0HAA0H;QAC1H,2DAA2D;QAC3D,8GAA8G;QAC9G,+BAA+B;QAC/B,6CAA6C;QAC7C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEnD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAuB,CAAC;QACzD,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,wBAAwB;QACxB,KAAK,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzD,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACzC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,SAAS;wBACjB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;qBAC5C;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;QAED,kDAAkD;QAClD,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,MAAM;wBACd,iBAAiB,EAAE,EAAE;qBACtB;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,EAAa;QAC7B,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;aACpB;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,iBAAiB,CAAC,EAAa,EAAE,MAAoC;QACnE,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;gBACnB,MAAM,EAAE;oBACN,IAAI,EAAE,MAAM;iBACb;gBACD,MAAM,EAAE,IAAI;aACb;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;IACzD,CAAC;IAED,sBAAsB;QACpB,OAAO,CACL,IAAI,CAAC,WAAW,IAAI,0BAA0B;YAC9C,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,yBAAyB;YACnD,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,yBAAyB;YACpD,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,yBAAyB,CAC1D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAkB,EAAE,OAA4B;QAC1D,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC9C,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,EAAE;gBAC1D,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE;gBAChD,OAAO;gBACP,mEAAmE;gBACnE,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE;gBAC5D,OAAO;gBACP,8CAA8C;gBAC9C,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAC7C,MAAM,CAAC,IAAI,CACT,aAAa,IAAI,CAAC,QAAQ,YAAY,IAAI,CAAC,UAAU,CAAC,MAAM,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,MAC5F,IAAI,CAAC,WAAW,CAAC,MACnB,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,IAAI,CAAC,aAAa,EAAE,CACpH,CAAC;QAEF,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC5B,CAAC;IAEO,qBAAqB;QAC3B,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE;YACrE,OAAO;gBACL,SAAS,EAAE;oBACT,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,MAAM;yBACV;qBACF;oBACD,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,OAAO,EAAE,KAAK,CAAC,MAAM;yBACtB;qBACF;oBACD,MAAM,EAAE,IAAI;iBACb;aACyD,CAAC;QAC/D,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
3
3
|
import { MongoStorageConfig } from '../../types/types.js';
|
|
4
|
-
import { BucketDataDocument, BucketParameterDocument, CurrentDataDocument, CustomWriteCheckpointDocument, IdSequenceDocument, InstanceDocument, SourceTableDocument, SyncRuleDocument, WriteCheckpointDocument } from './models.js';
|
|
4
|
+
import { BucketDataDocument, BucketParameterDocument, BucketStateDocument, CurrentDataDocument, CustomWriteCheckpointDocument, IdSequenceDocument, InstanceDocument, SourceTableDocument, SyncRuleDocument, WriteCheckpointDocument } from './models.js';
|
|
5
5
|
export interface PowerSyncMongoOptions {
|
|
6
6
|
/**
|
|
7
7
|
* Optional - uses the database from the MongoClient connection URI if not specified.
|
|
@@ -19,6 +19,7 @@ export declare class PowerSyncMongo {
|
|
|
19
19
|
readonly write_checkpoints: mongo.Collection<WriteCheckpointDocument>;
|
|
20
20
|
readonly instance: mongo.Collection<InstanceDocument>;
|
|
21
21
|
readonly locks: mongo.Collection<lib_mongo.locks.Lock>;
|
|
22
|
+
readonly bucket_state: mongo.Collection<BucketStateDocument>;
|
|
22
23
|
readonly client: mongo.MongoClient;
|
|
23
24
|
readonly db: mongo.Db;
|
|
24
25
|
constructor(client: mongo.MongoClient, options?: PowerSyncMongoOptions);
|
|
@@ -11,6 +11,7 @@ export class PowerSyncMongo {
|
|
|
11
11
|
write_checkpoints;
|
|
12
12
|
instance;
|
|
13
13
|
locks;
|
|
14
|
+
bucket_state;
|
|
14
15
|
client;
|
|
15
16
|
db;
|
|
16
17
|
constructor(client, options) {
|
|
@@ -29,6 +30,7 @@ export class PowerSyncMongo {
|
|
|
29
30
|
this.write_checkpoints = db.collection('write_checkpoints');
|
|
30
31
|
this.instance = db.collection('instance');
|
|
31
32
|
this.locks = this.db.collection('locks');
|
|
33
|
+
this.bucket_state = this.db.collection('bucket_state');
|
|
32
34
|
}
|
|
33
35
|
/**
|
|
34
36
|
* Clear all collections.
|
|
@@ -43,6 +45,8 @@ export class PowerSyncMongo {
|
|
|
43
45
|
await this.write_checkpoints.deleteMany({});
|
|
44
46
|
await this.instance.deleteOne({});
|
|
45
47
|
await this.locks.deleteMany({});
|
|
48
|
+
await this.bucket_state.deleteMany({});
|
|
49
|
+
await this.custom_write_checkpoints.deleteMany({});
|
|
46
50
|
}
|
|
47
51
|
/**
|
|
48
52
|
* Drop the entire database.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAuBlD,MAAM,OAAO,cAAc;IAChB,YAAY,CAAwC;IACpD,WAAW,CAAuC;IAClD,iBAAiB,CAA4C;IAC7D,cAAc,CAAuC;IACrD,UAAU,CAAqC;IAC/C,aAAa,CAAwC;IACrD,wBAAwB,CAAkD;IAC1E,iBAAiB,CAA4C;IAC7D,QAAQ,CAAqC;IAC7C,KAAK,CAAyC;IAC9C,YAAY,CAAwC;IAEpD,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAEtB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,iCAAiC;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAsB,cAAc,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,UAAU,oBAAoB,CAAC,MAA0B,EAAE,OAA0C;IACzG,OAAO,IAAI,cAAc,CAAC,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;AACzG,CAAC"}
|
|
@@ -69,6 +69,23 @@ export interface SourceTableDocument {
|
|
|
69
69
|
}[] | undefined;
|
|
70
70
|
snapshot_done: boolean | undefined;
|
|
71
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* Record the state of each bucket.
|
|
74
|
+
*
|
|
75
|
+
* Right now, this is just used to track when buckets are updated, for efficient incremental sync.
|
|
76
|
+
* In the future, this could be used to track operation counts, both for diagnostic purposes, and for
|
|
77
|
+
* determining when a compact and/or defragment could be beneficial.
|
|
78
|
+
*
|
|
79
|
+
* Note: There is currently no migration to populate this collection from existing data - it is only
|
|
80
|
+
* populated by new updates.
|
|
81
|
+
*/
|
|
82
|
+
export interface BucketStateDocument {
|
|
83
|
+
_id: {
|
|
84
|
+
g: number;
|
|
85
|
+
b: string;
|
|
86
|
+
};
|
|
87
|
+
last_op: bigint;
|
|
88
|
+
}
|
|
72
89
|
export interface IdSequenceDocument {
|
|
73
90
|
_id: string;
|
|
74
91
|
op_id: bigint;
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@powersync/service-module-mongodb-storage",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
4
|
"types": "dist/index.d.ts",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.8.0",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"license": "FSL-1.1-Apache-2.0",
|
|
8
8
|
"type": "module",
|
|
@@ -28,15 +28,15 @@
|
|
|
28
28
|
"lru-cache": "^10.2.2",
|
|
29
29
|
"uuid": "^9.0.1",
|
|
30
30
|
"@powersync/lib-services-framework": "0.5.3",
|
|
31
|
-
"@powersync/service-core": "1.9.0",
|
|
32
31
|
"@powersync/service-jsonbig": "0.17.10",
|
|
33
|
-
"@powersync/service-sync-rules": "0.
|
|
32
|
+
"@powersync/service-sync-rules": "0.25.0",
|
|
34
33
|
"@powersync/service-types": "0.9.0",
|
|
34
|
+
"@powersync/service-core": "1.10.0",
|
|
35
35
|
"@powersync/lib-service-mongodb": "0.5.1"
|
|
36
36
|
},
|
|
37
37
|
"devDependencies": {
|
|
38
38
|
"@types/uuid": "^9.0.4",
|
|
39
|
-
"@powersync/service-core-tests": "0.
|
|
39
|
+
"@powersync/service-core-tests": "0.8.0"
|
|
40
40
|
},
|
|
41
41
|
"scripts": {
|
|
42
42
|
"build": "tsc -b",
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { migrations } from '@powersync/service-core';
|
|
2
|
+
import * as storage from '../../../storage/storage-index.js';
|
|
3
|
+
import { MongoStorageConfig } from '../../../types/types.js';
|
|
4
|
+
|
|
5
|
+
const INDEX_NAME = 'bucket_updates';
|
|
6
|
+
|
|
7
|
+
export const up: migrations.PowerSyncMigrationFunction = async (context) => {
|
|
8
|
+
const {
|
|
9
|
+
service_context: { configuration }
|
|
10
|
+
} = context;
|
|
11
|
+
const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
|
|
12
|
+
|
|
13
|
+
try {
|
|
14
|
+
await db.bucket_state.createIndex(
|
|
15
|
+
{
|
|
16
|
+
'_id.g': 1,
|
|
17
|
+
last_op: 1
|
|
18
|
+
},
|
|
19
|
+
{ name: INDEX_NAME, unique: true }
|
|
20
|
+
);
|
|
21
|
+
} finally {
|
|
22
|
+
await db.client.close();
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export const down: migrations.PowerSyncMigrationFunction = async (context) => {
|
|
27
|
+
const {
|
|
28
|
+
service_context: { configuration }
|
|
29
|
+
} = context;
|
|
30
|
+
|
|
31
|
+
const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
|
|
32
|
+
|
|
33
|
+
try {
|
|
34
|
+
if (await db.bucket_state.indexExists(INDEX_NAME)) {
|
|
35
|
+
await db.bucket_state.dropIndex(INDEX_NAME);
|
|
36
|
+
}
|
|
37
|
+
} finally {
|
|
38
|
+
await db.client.close();
|
|
39
|
+
}
|
|
40
|
+
};
|
|
@@ -314,10 +314,12 @@ export class MongoCompactor {
|
|
|
314
314
|
let lastOpId: BucketDataKey | null = null;
|
|
315
315
|
let targetOp: bigint | null = null;
|
|
316
316
|
let gotAnOp = false;
|
|
317
|
+
let numberOfOpsToClear = 0;
|
|
317
318
|
for await (let op of query.stream()) {
|
|
318
319
|
if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') {
|
|
319
320
|
checksum = utils.addChecksums(checksum, op.checksum);
|
|
320
321
|
lastOpId = op._id;
|
|
322
|
+
numberOfOpsToClear += 1;
|
|
321
323
|
if (op.op != 'CLEAR') {
|
|
322
324
|
gotAnOp = true;
|
|
323
325
|
}
|
|
@@ -337,7 +339,7 @@ export class MongoCompactor {
|
|
|
337
339
|
return;
|
|
338
340
|
}
|
|
339
341
|
|
|
340
|
-
logger.info(`Flushing CLEAR at ${lastOpId?.o}`);
|
|
342
|
+
logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOpId?.o}`);
|
|
341
343
|
await this.db.bucket_data.deleteMany(
|
|
342
344
|
{
|
|
343
345
|
_id: {
|