@powersync/service-module-mongodb-storage 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/dist/storage/MongoBucketStorage.d.ts +6 -4
- package/dist/storage/MongoBucketStorage.js +110 -36
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +4 -6
- package/dist/storage/implementation/BucketDefinitionMapping.js +3 -3
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -1
- package/dist/storage/implementation/CheckpointState.d.ts +20 -0
- package/dist/storage/implementation/CheckpointState.js +31 -0
- package/dist/storage/implementation/CheckpointState.js.map +1 -0
- package/dist/storage/implementation/MongoBucketBatch.d.ts +33 -22
- package/dist/storage/implementation/MongoBucketBatch.js +45 -271
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +2 -1
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +1 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +4 -4
- package/dist/storage/implementation/MongoPersistedSyncRules.js +11 -8
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +19 -5
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +53 -19
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +21 -10
- package/dist/storage/implementation/MongoSyncBucketStorage.js +18 -163
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +5 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +7 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +14 -0
- package/dist/storage/implementation/SyncRuleStateUpdate.js +36 -0
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -0
- package/dist/storage/implementation/common/BucketDataDoc.d.ts +1 -1
- package/dist/storage/implementation/common/PersistedBatch.d.ts +2 -2
- package/dist/storage/implementation/common/SourceRecordStore.d.ts +1 -2
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +10 -2
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +31 -47
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +385 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +16 -7
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +77 -6
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +1 -2
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +12 -1
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +17 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +429 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +1 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +1 -2
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +29 -7
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +117 -16
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +1 -2
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +3 -2
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +3 -0
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/models.d.ts +61 -3
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/package.json +6 -6
- package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +1 -1
- package/src/storage/MongoBucketStorage.ts +166 -44
- package/src/storage/implementation/BucketDefinitionMapping.ts +12 -9
- package/src/storage/implementation/CheckpointState.ts +59 -0
- package/src/storage/implementation/MongoBucketBatch.ts +81 -355
- package/src/storage/implementation/MongoChecksums.ts +2 -1
- package/src/storage/implementation/MongoCompactor.ts +1 -1
- package/src/storage/implementation/MongoPersistedSyncRules.ts +13 -7
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +69 -24
- package/src/storage/implementation/MongoSyncBucketStorage.ts +40 -215
- package/src/storage/implementation/MongoSyncRulesLock.ts +9 -3
- package/src/storage/implementation/SyncRuleStateUpdate.ts +38 -0
- package/src/storage/implementation/common/BucketDataDoc.ts +1 -1
- package/src/storage/implementation/common/PersistedBatch.ts +2 -2
- package/src/storage/implementation/common/SourceRecordStore.ts +1 -2
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +2 -2
- package/src/storage/implementation/db.ts +5 -2
- package/src/storage/implementation/models.ts +35 -58
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +478 -1
- package/src/storage/implementation/v1/MongoCompactorV1.ts +1 -1
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +111 -16
- package/src/storage/implementation/v1/PersistedBatchV1.ts +1 -2
- package/src/storage/implementation/v1/models.ts +15 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +564 -1
- package/src/storage/implementation/v3/MongoCompactorV3.ts +1 -1
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +1 -2
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +150 -22
- package/src/storage/implementation/v3/PersistedBatchV3.ts +1 -2
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +7 -2
- package/src/storage/implementation/v3/models.ts +70 -2
- package/test/src/storage_sync.test.ts +422 -6
- package/test/src/storeCurrentData.test.ts +211 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,16 +1,64 @@
|
|
|
1
1
|
import { deserializeParameterLookup, JwtPayload, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
|
|
2
2
|
import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
|
|
3
|
-
import { RequestParameters } from '@powersync/service-sync-rules';
|
|
3
|
+
import { DEFAULT_HYDRATION_STATE, nodeSqlite, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
|
|
4
4
|
import * as bson from 'bson';
|
|
5
|
+
import * as sqlite from 'node:sqlite';
|
|
5
6
|
import { describe, expect, test } from 'vitest';
|
|
6
7
|
import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
|
|
7
8
|
import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
|
|
8
|
-
import { SyncRuleDocument } from '../../src/storage/implementation/models.js';
|
|
9
9
|
import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js';
|
|
10
10
|
import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
CurrentBucketV3,
|
|
13
|
+
ReplicationStreamDocumentV3,
|
|
14
|
+
SyncConfigDefinition
|
|
15
|
+
} from '../../src/storage/implementation/v3/models.js';
|
|
12
16
|
import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
|
|
13
17
|
|
|
18
|
+
const MINIMAL_SYNC_RULES = `
|
|
19
|
+
bucket_definitions:
|
|
20
|
+
global:
|
|
21
|
+
data:
|
|
22
|
+
- SELECT id FROM test
|
|
23
|
+
`;
|
|
24
|
+
|
|
25
|
+
function sourceDescriptor(
|
|
26
|
+
name: string,
|
|
27
|
+
options: {
|
|
28
|
+
objectId?: string;
|
|
29
|
+
replicaIdColumns?: string[];
|
|
30
|
+
} = {}
|
|
31
|
+
): storage.SourceEntityDescriptor {
|
|
32
|
+
return {
|
|
33
|
+
connectionTag: storage.SourceTable.DEFAULT_TAG,
|
|
34
|
+
objectId: options.objectId ?? name,
|
|
35
|
+
schema: 'public',
|
|
36
|
+
name,
|
|
37
|
+
replicaIdColumns: (options.replicaIdColumns ?? ['id']).map((column) => ({
|
|
38
|
+
name: column,
|
|
39
|
+
type: 'VARCHAR',
|
|
40
|
+
typeId: 25
|
|
41
|
+
}))
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function objectIdGenerator(id: string) {
|
|
46
|
+
let used = false;
|
|
47
|
+
return () => {
|
|
48
|
+
if (used) {
|
|
49
|
+
throw new Error(`Can only generate a single id using ${id}`);
|
|
50
|
+
}
|
|
51
|
+
used = true;
|
|
52
|
+
return new bson.ObjectId(id);
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function hydratedRulesFor(yaml: string) {
|
|
57
|
+
const parsed = SqlSyncRules.fromYaml(yaml, test_utils.PARSE_OPTIONS);
|
|
58
|
+
expect(parsed.errors).toEqual([]);
|
|
59
|
+
return parsed.config.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE, sqlite: nodeSqlite(sqlite) });
|
|
60
|
+
}
|
|
61
|
+
|
|
14
62
|
function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) {
|
|
15
63
|
register.registerSyncTests(storageConfig.factory, {
|
|
16
64
|
storageVersion,
|
|
@@ -157,6 +205,316 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
157
205
|
expect(checksumTypes).toEqual([{ _id: 'long', count: 4 }]);
|
|
158
206
|
});
|
|
159
207
|
|
|
208
|
+
test('resolveTables populates matching data and parameter sources', async () => {
|
|
209
|
+
await using factory = await storageConfig.factory();
|
|
210
|
+
const syncRules = await factory.updateSyncRules(
|
|
211
|
+
updateSyncRulesFromYaml(
|
|
212
|
+
`
|
|
213
|
+
bucket_definitions:
|
|
214
|
+
by_owner:
|
|
215
|
+
parameters:
|
|
216
|
+
- SELECT owner_id FROM test WHERE id = token_parameters.test_id
|
|
217
|
+
data:
|
|
218
|
+
- SELECT id, owner_id FROM test WHERE owner_id = bucket.owner_id
|
|
219
|
+
`,
|
|
220
|
+
{ storageVersion }
|
|
221
|
+
)
|
|
222
|
+
);
|
|
223
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
224
|
+
|
|
225
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
226
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
227
|
+
|
|
228
|
+
expect(sourceTable.bucketDataSources).toHaveLength(1);
|
|
229
|
+
expect(sourceTable.parameterLookupSources).toHaveLength(1);
|
|
230
|
+
expect(sourceTable.syncData).toBe(true);
|
|
231
|
+
expect(sourceTable.syncParameters).toBe(true);
|
|
232
|
+
expect(sourceTable.syncEvent).toBe(false);
|
|
233
|
+
});
|
|
234
|
+
|
|
235
|
+
test('resolveTables drops old table when table name changes for the same objectId', async () => {
|
|
236
|
+
await using factory = await storageConfig.factory();
|
|
237
|
+
const syncRules = await factory.updateSyncRules(
|
|
238
|
+
updateSyncRulesFromYaml(
|
|
239
|
+
`
|
|
240
|
+
bucket_definitions:
|
|
241
|
+
global:
|
|
242
|
+
data:
|
|
243
|
+
- SELECT id FROM "%"
|
|
244
|
+
`,
|
|
245
|
+
{ storageVersion }
|
|
246
|
+
)
|
|
247
|
+
);
|
|
248
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
249
|
+
|
|
250
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
251
|
+
const before = await writer.resolveTables({
|
|
252
|
+
connection_id: 1,
|
|
253
|
+
source: sourceDescriptor('orders', { objectId: 'orders-relation' }),
|
|
254
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b38342')
|
|
255
|
+
});
|
|
256
|
+
const after = await writer.resolveTables({
|
|
257
|
+
connection_id: 1,
|
|
258
|
+
source: sourceDescriptor('renamed_orders', { objectId: 'orders-relation' }),
|
|
259
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b38343')
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
expect(after.tables).toHaveLength(1);
|
|
263
|
+
expect(after.tables[0].id).not.toEqual(before.tables[0].id);
|
|
264
|
+
expect(after.tables[0].bucketDataSources).toHaveLength(1);
|
|
265
|
+
expect(after.tables[0].parameterLookupSources).toHaveLength(0);
|
|
266
|
+
expect(after.dropTables.map((table) => ({ id: table.id, name: table.name }))).toEqual([
|
|
267
|
+
{ id: before.tables[0].id, name: 'orders' }
|
|
268
|
+
]);
|
|
269
|
+
expect(after.dropTables[0].bucketDataSources).toHaveLength(1);
|
|
270
|
+
expect(after.dropTables[0].parameterLookupSources).toHaveLength(0);
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
test('resolveTables drops old table when objectId changes for the same table name', async () => {
|
|
274
|
+
await using factory = await storageConfig.factory();
|
|
275
|
+
const syncRules = await factory.updateSyncRules(
|
|
276
|
+
updateSyncRulesFromYaml(
|
|
277
|
+
`
|
|
278
|
+
bucket_definitions:
|
|
279
|
+
global:
|
|
280
|
+
data:
|
|
281
|
+
- SELECT id FROM "%"
|
|
282
|
+
`,
|
|
283
|
+
{ storageVersion }
|
|
284
|
+
)
|
|
285
|
+
);
|
|
286
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
287
|
+
|
|
288
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
289
|
+
const before = await writer.resolveTables({
|
|
290
|
+
connection_id: 1,
|
|
291
|
+
source: sourceDescriptor('accounts', { objectId: 'accounts-relation-old' }),
|
|
292
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b38344')
|
|
293
|
+
});
|
|
294
|
+
const after = await writer.resolveTables({
|
|
295
|
+
connection_id: 1,
|
|
296
|
+
source: sourceDescriptor('accounts', { objectId: 'accounts-relation-new' }),
|
|
297
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b38345')
|
|
298
|
+
});
|
|
299
|
+
|
|
300
|
+
expect(after.tables).toHaveLength(1);
|
|
301
|
+
expect(after.tables[0].id).not.toEqual(before.tables[0].id);
|
|
302
|
+
expect(after.tables[0].bucketDataSources).toHaveLength(1);
|
|
303
|
+
expect(after.dropTables.map((table) => ({ id: table.id, objectId: table.objectId }))).toEqual([
|
|
304
|
+
{ id: before.tables[0].id, objectId: 'accounts-relation-old' }
|
|
305
|
+
]);
|
|
306
|
+
expect(after.dropTables[0].bucketDataSources).toHaveLength(1);
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
test('resolveTables drops old table when replica id columns change', async () => {
|
|
310
|
+
await using factory = await storageConfig.factory();
|
|
311
|
+
const syncRules = await factory.updateSyncRules(
|
|
312
|
+
updateSyncRulesFromYaml(
|
|
313
|
+
`
|
|
314
|
+
bucket_definitions:
|
|
315
|
+
global:
|
|
316
|
+
data:
|
|
317
|
+
- SELECT id FROM "%"
|
|
318
|
+
`,
|
|
319
|
+
{ storageVersion }
|
|
320
|
+
)
|
|
321
|
+
);
|
|
322
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
323
|
+
|
|
324
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
325
|
+
const before = await writer.resolveTables({
|
|
326
|
+
connection_id: 1,
|
|
327
|
+
source: sourceDescriptor('items', { objectId: 'items-relation', replicaIdColumns: ['id'] }),
|
|
328
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b38346')
|
|
329
|
+
});
|
|
330
|
+
const after = await writer.resolveTables({
|
|
331
|
+
connection_id: 1,
|
|
332
|
+
source: sourceDescriptor('items', { objectId: 'items-relation', replicaIdColumns: ['tenant_id', 'id'] }),
|
|
333
|
+
idGenerator: objectIdGenerator('6544e3899293153fa7b38347')
|
|
334
|
+
});
|
|
335
|
+
|
|
336
|
+
expect(after.tables).toHaveLength(1);
|
|
337
|
+
expect(after.tables[0].id).not.toEqual(before.tables[0].id);
|
|
338
|
+
expect(after.tables[0].replicaIdColumns.map((column) => column.name)).toEqual(['tenant_id', 'id']);
|
|
339
|
+
expect(after.tables[0].bucketDataSources).toHaveLength(1);
|
|
340
|
+
expect(
|
|
341
|
+
after.dropTables.map((table) => ({ id: table.id, columns: table.replicaIdColumns.map((c) => c.name) }))
|
|
342
|
+
).toEqual([{ id: before.tables[0].id, columns: ['id'] }]);
|
|
343
|
+
expect(after.dropTables[0].bucketDataSources).toHaveLength(1);
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
test.runIf(storageVersion >= 3)(
|
|
347
|
+
'resolveTables resolves v3 event-only tables without source memberships',
|
|
348
|
+
async () => {
|
|
349
|
+
await using factory = await storageConfig.factory();
|
|
350
|
+
const syncRules = await factory.updateSyncRules(
|
|
351
|
+
updateSyncRulesFromYaml(
|
|
352
|
+
`
|
|
353
|
+
bucket_definitions:
|
|
354
|
+
by_owner:
|
|
355
|
+
data:
|
|
356
|
+
- SELECT id FROM users
|
|
357
|
+
|
|
358
|
+
event_definitions:
|
|
359
|
+
write_checkpoints:
|
|
360
|
+
payloads:
|
|
361
|
+
- SELECT user_id, checkpoint FROM checkpoints
|
|
362
|
+
`,
|
|
363
|
+
{ storageVersion }
|
|
364
|
+
)
|
|
365
|
+
);
|
|
366
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
367
|
+
|
|
368
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
369
|
+
const resolved = await writer.resolveTables({
|
|
370
|
+
connection_id: 1,
|
|
371
|
+
source: {
|
|
372
|
+
connectionTag: storage.SourceTable.DEFAULT_TAG,
|
|
373
|
+
objectId: 'checkpoints',
|
|
374
|
+
schema: 'public',
|
|
375
|
+
name: 'checkpoints',
|
|
376
|
+
replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }]
|
|
377
|
+
},
|
|
378
|
+
idGenerator: () => new bson.ObjectId('6544e3899293153fa7b38341')
|
|
379
|
+
});
|
|
380
|
+
|
|
381
|
+
expect(resolved.tables).toHaveLength(1);
|
|
382
|
+
expect(resolved.dropTables).toHaveLength(0);
|
|
383
|
+
expect(resolved.tables[0].bucketDataSources).toEqual([]);
|
|
384
|
+
expect(resolved.tables[0].parameterLookupSources).toEqual([]);
|
|
385
|
+
expect(resolved.tables[0].syncData).toBe(false);
|
|
386
|
+
expect(resolved.tables[0].syncParameters).toBe(false);
|
|
387
|
+
expect(resolved.tables[0].syncEvent).toBe(true);
|
|
388
|
+
}
|
|
389
|
+
);
|
|
390
|
+
|
|
391
|
+
test.runIf(storageVersion >= 3)('resolveTables handles v3 source membership additions and removals', async () => {
|
|
392
|
+
// Tests the behavior of resolveTables when bucket data sources and parameter index creators are added or removed.
|
|
393
|
+
// These are not end-to-end tests yet, since we don't have a full incremental reprocessing implementation.
|
|
394
|
+
// This just tests the specific resolveTables behavior.
|
|
395
|
+
|
|
396
|
+
// The same tests should work with sync streams, but legacy bucket_definitions make it easy
|
|
397
|
+
// to see the distinction between the parameter index queries and the data sources.
|
|
398
|
+
const fullRulesYaml = `
|
|
399
|
+
bucket_definitions:
|
|
400
|
+
by_owner:
|
|
401
|
+
parameters:
|
|
402
|
+
- SELECT owner_id FROM memberships WHERE id = token_parameters.test_id
|
|
403
|
+
data:
|
|
404
|
+
- SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
|
|
405
|
+
`;
|
|
406
|
+
const dataOnlyRulesYaml = `
|
|
407
|
+
bucket_definitions:
|
|
408
|
+
by_owner:
|
|
409
|
+
parameters:
|
|
410
|
+
- SELECT token_parameters.owner_id as owner_id
|
|
411
|
+
data:
|
|
412
|
+
- SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
|
|
413
|
+
`;
|
|
414
|
+
const parameterOnlyRulesYaml = `
|
|
415
|
+
bucket_definitions:
|
|
416
|
+
by_owner:
|
|
417
|
+
parameters:
|
|
418
|
+
- SELECT owner_id FROM memberships WHERE id = token_parameters.test_id
|
|
419
|
+
data: []
|
|
420
|
+
`;
|
|
421
|
+
const eventOnlyRulesYaml = `
|
|
422
|
+
bucket_definitions: {}
|
|
423
|
+
|
|
424
|
+
event_definitions:
|
|
425
|
+
write_checkpoints:
|
|
426
|
+
payloads:
|
|
427
|
+
- SELECT id, owner_id FROM memberships
|
|
428
|
+
`;
|
|
429
|
+
|
|
430
|
+
await using factory = await storageConfig.factory();
|
|
431
|
+
// This does not quite match what actual API usage would look like.
|
|
432
|
+
// Here we're persisting one sync config, then resolving tables with others.
|
|
433
|
+
// We're also using the default hydration state for them all.
|
|
434
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(fullRulesYaml, { storageVersion }));
|
|
435
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
436
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
437
|
+
const fullRules = hydratedRulesFor(fullRulesYaml);
|
|
438
|
+
const dataOnlyRules = hydratedRulesFor(dataOnlyRulesYaml);
|
|
439
|
+
const parameterOnlyRules = hydratedRulesFor(parameterOnlyRulesYaml);
|
|
440
|
+
const eventOnlyRules = hydratedRulesFor(eventOnlyRulesYaml);
|
|
441
|
+
const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
|
|
442
|
+
const dataOnlyTableId = new bson.ObjectId('6544e3899293153fa7b38348');
|
|
443
|
+
const addedParameterTableId = new bson.ObjectId('6544e3899293153fa7b38349');
|
|
444
|
+
const removedDataTableId = new bson.ObjectId('6544e3899293153fa7b3834a');
|
|
445
|
+
|
|
446
|
+
const dataOnly = await writer.resolveTables({
|
|
447
|
+
connection_id: 1,
|
|
448
|
+
source,
|
|
449
|
+
idGenerator: () => dataOnlyTableId,
|
|
450
|
+
syncRules: dataOnlyRules
|
|
451
|
+
});
|
|
452
|
+
expect(dataOnly.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
|
|
453
|
+
expect(dataOnly.dropTables.map((table) => table.id)).toEqual([]);
|
|
454
|
+
expect(dataOnly.tables[0].bucketDataSources).toHaveLength(1);
|
|
455
|
+
expect(dataOnly.tables[0].parameterLookupSources).toHaveLength(0);
|
|
456
|
+
|
|
457
|
+
const addedParameter = await writer.resolveTables({
|
|
458
|
+
connection_id: 1,
|
|
459
|
+
source,
|
|
460
|
+
idGenerator: () => addedParameterTableId,
|
|
461
|
+
syncRules: fullRules
|
|
462
|
+
});
|
|
463
|
+
// Adding a definition always creates a new SourceTable
|
|
464
|
+
expect(addedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId, addedParameterTableId]);
|
|
465
|
+
expect(addedParameter.tables.map((table) => table.bucketDataSources.length).sort()).toEqual([0, 1]);
|
|
466
|
+
expect(addedParameter.tables.map((table) => table.parameterLookupSources.length).sort()).toEqual([0, 1]);
|
|
467
|
+
expect(addedParameter.dropTables.map((table) => table.id)).toEqual([]);
|
|
468
|
+
|
|
469
|
+
const removedParameter = await writer.resolveTables({
|
|
470
|
+
connection_id: 1,
|
|
471
|
+
source,
|
|
472
|
+
idGenerator: () => {
|
|
473
|
+
throw new Error('data-only resolve should reuse existing v3 source table');
|
|
474
|
+
},
|
|
475
|
+
syncRules: dataOnlyRules
|
|
476
|
+
});
|
|
477
|
+
expect(removedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
|
|
478
|
+
// Now this sourceTable is unused & dropped
|
|
479
|
+
expect(removedParameter.dropTables.map((table) => table.id)).toEqual([addedParameterTableId]);
|
|
480
|
+
expect(removedParameter.tables[0].bucketDataSources).toHaveLength(1);
|
|
481
|
+
expect(removedParameter.tables[0].parameterLookupSources).toHaveLength(0);
|
|
482
|
+
await writer.drop(removedParameter.dropTables);
|
|
483
|
+
|
|
484
|
+
const removedData = await writer.resolveTables({
|
|
485
|
+
connection_id: 1,
|
|
486
|
+
source,
|
|
487
|
+
idGenerator: () => removedDataTableId,
|
|
488
|
+
syncRules: parameterOnlyRules
|
|
489
|
+
});
|
|
490
|
+
|
|
491
|
+
// This goes from dataOnlyRules -> parameterOnlyRules, which adds one definition and removes another.
|
|
492
|
+
// This generates a new SourceTable again, and removes all others.
|
|
493
|
+
expect(removedData.tables.map((table) => table.id)).toEqual([removedDataTableId]);
|
|
494
|
+
expect(removedData.dropTables.map((table) => table.id)).toEqual([dataOnlyTableId]);
|
|
495
|
+
expect(removedData.tables[0].bucketDataSources).toHaveLength(0);
|
|
496
|
+
expect(removedData.tables[0].parameterLookupSources).toHaveLength(1);
|
|
497
|
+
await writer.drop(removedData.dropTables);
|
|
498
|
+
|
|
499
|
+
const eventOnly = await writer.resolveTables({
|
|
500
|
+
connection_id: 1,
|
|
501
|
+
source,
|
|
502
|
+
idGenerator: () => {
|
|
503
|
+
throw new Error('resolve should reuse existing v3 source table');
|
|
504
|
+
},
|
|
505
|
+
syncRules: eventOnlyRules
|
|
506
|
+
});
|
|
507
|
+
|
|
508
|
+
// Event-only table can re-use any existing table.
|
|
509
|
+
expect(eventOnly.tables.map((table) => table.id)).toEqual([removedDataTableId]);
|
|
510
|
+
expect(eventOnly.dropTables.map((table) => table.id)).toEqual([]);
|
|
511
|
+
expect(eventOnly.tables[0].bucketDataSources).toHaveLength(0);
|
|
512
|
+
expect(eventOnly.tables[0].parameterLookupSources).toHaveLength(0);
|
|
513
|
+
expect(eventOnly.tables[0].syncData).toBe(false);
|
|
514
|
+
expect(eventOnly.tables[0].syncParameters).toBe(false);
|
|
515
|
+
expect(eventOnly.tables[0].syncEvent).toBe(true);
|
|
516
|
+
});
|
|
517
|
+
|
|
160
518
|
test.runIf(storageVersion >= 3)('uses v3 mongodb model shapes', async () => {
|
|
161
519
|
await using factory = await storageConfig.factory();
|
|
162
520
|
const syncRules = await factory.updateSyncRules(
|
|
@@ -173,7 +531,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
173
531
|
)
|
|
174
532
|
);
|
|
175
533
|
const bucketStorage = factory.getInstance(syncRules);
|
|
176
|
-
const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).
|
|
534
|
+
const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
|
|
177
535
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
178
536
|
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
179
537
|
|
|
@@ -219,8 +577,9 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
219
577
|
bucketCollections.some((collection) => collection.name === `bucket_data_${syncRules.id}_${firstBucket?.def}`)
|
|
220
578
|
).toBe(true);
|
|
221
579
|
|
|
222
|
-
const syncRule = await mongoFactory.db.sync_rules.findOne({ _id: syncRules.id });
|
|
223
|
-
const
|
|
580
|
+
const syncRule = (await mongoFactory.db.sync_rules.findOne({ _id: syncRules.id })) as ReplicationStreamDocumentV3;
|
|
581
|
+
const syncConfig = await db.syncConfigDefinitions.findOne({ _id: syncRule.sync_configs[0]._id });
|
|
582
|
+
const ruleMapping: SyncConfigDefinition['rule_mapping'] | undefined = syncConfig?.rule_mapping;
|
|
224
583
|
expect(Object.keys(ruleMapping?.definitions ?? {})).not.toHaveLength(0);
|
|
225
584
|
|
|
226
585
|
const parameterIndexId = Object.values(ruleMapping?.parameter_indexes ?? {})[0] as string | undefined;
|
|
@@ -229,6 +588,34 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
229
588
|
expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']);
|
|
230
589
|
});
|
|
231
590
|
|
|
591
|
+
test.runIf(storageVersion < 3)('can replace processing legacy sync rules', async () => {
|
|
592
|
+
await using factory = await storageConfig.factory();
|
|
593
|
+
|
|
594
|
+
const firstSyncRules = await factory.updateSyncRules(
|
|
595
|
+
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion })
|
|
596
|
+
);
|
|
597
|
+
|
|
598
|
+
await expect(
|
|
599
|
+
factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }))
|
|
600
|
+
).resolves.toBeDefined();
|
|
601
|
+
|
|
602
|
+
const mongoFactory = factory as MongoBucketStorage;
|
|
603
|
+
expect((await mongoFactory.db.sync_rules.findOne({ _id: firstSyncRules.id }))?.state).toBe(
|
|
604
|
+
storage.SyncRuleState.STOP
|
|
605
|
+
);
|
|
606
|
+
});
|
|
607
|
+
|
|
608
|
+
test('can lock newly-created sync rules', async () => {
|
|
609
|
+
await using factory = await storageConfig.factory();
|
|
610
|
+
|
|
611
|
+
const syncRules = await factory.updateSyncRules(
|
|
612
|
+
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion, lock: true })
|
|
613
|
+
);
|
|
614
|
+
|
|
615
|
+
expect(syncRules.current_lock?.sync_rules_id).toBe(syncRules.id);
|
|
616
|
+
await syncRules.current_lock?.release();
|
|
617
|
+
});
|
|
618
|
+
|
|
232
619
|
test.runIf(storageVersion < 3)('uses a single current_data collection for v1 source records', async () => {
|
|
233
620
|
await using factory = await storageConfig.factory();
|
|
234
621
|
const syncRules = await factory.updateSyncRules(
|
|
@@ -481,6 +868,35 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
481
868
|
}
|
|
482
869
|
|
|
483
870
|
describe('sync - mongodb', () => {
|
|
871
|
+
test('v3 activation stops legacy active sync rules', async () => {
|
|
872
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
873
|
+
const mongoFactory = factory as MongoBucketStorage;
|
|
874
|
+
|
|
875
|
+
const legacySyncRules = await factory.updateSyncRules(
|
|
876
|
+
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion: storage.LEGACY_STORAGE_VERSION })
|
|
877
|
+
);
|
|
878
|
+
const legacyStorage = factory.getInstance(legacySyncRules);
|
|
879
|
+
await using legacyWriter = await legacyStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
880
|
+
await legacyWriter.markAllSnapshotDone('1/1');
|
|
881
|
+
await legacyWriter.commit('1/1');
|
|
882
|
+
|
|
883
|
+
expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.id }))?.state).toBe(
|
|
884
|
+
storage.SyncRuleState.ACTIVE
|
|
885
|
+
);
|
|
886
|
+
|
|
887
|
+
const v3SyncRules = await factory.updateSyncRules(
|
|
888
|
+
updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion: storage.STORAGE_VERSION_3 })
|
|
889
|
+
);
|
|
890
|
+
const v3Storage = factory.getInstance(v3SyncRules);
|
|
891
|
+
await using v3Writer = await v3Storage.createWriter(test_utils.BATCH_OPTIONS);
|
|
892
|
+
await v3Writer.markAllSnapshotDone('2/1');
|
|
893
|
+
await v3Writer.commit('2/1');
|
|
894
|
+
|
|
895
|
+
expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.id }))?.state).toBe(
|
|
896
|
+
storage.SyncRuleState.STOP
|
|
897
|
+
);
|
|
898
|
+
});
|
|
899
|
+
|
|
484
900
|
for (const storageVersion of TEST_STORAGE_VERSIONS) {
|
|
485
901
|
describe(`storage v${storageVersion}`, () => {
|
|
486
902
|
registerSyncStorageTests(INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion);
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
|
|
2
|
+
import { bucketRequest, test_utils } from '@powersync/service-core-tests';
|
|
3
|
+
import * as bson from 'bson';
|
|
4
|
+
import { describe, expect, test } from 'vitest';
|
|
5
|
+
import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
|
|
6
|
+
import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Tests for the per-table `storeCurrentData` flag in MongoDB storage (both v1 and v3).
|
|
10
|
+
*
|
|
11
|
+
* Two things are exercised:
|
|
12
|
+
* 1. `resolveTables` derives `storeCurrentData = !sendsCompleteRows` in memory on every call
|
|
13
|
+
* (it is not persisted; an unreported source defaults to keeping a copy).
|
|
14
|
+
* 2. A batch honours the flag: when `storeCurrentData` is false, the row payload is NOT kept in
|
|
15
|
+
* the current_data collection, while the bucket data is still produced as usual.
|
|
16
|
+
*
|
|
17
|
+
* The mapping from a source's replica identity to `sendsCompleteRows` is a Postgres concern and is
|
|
18
|
+
* covered in module-postgres; here we only exercise the storage layer's boolean handling.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
const SYNC_RULES = `
|
|
22
|
+
bucket_definitions:
|
|
23
|
+
global:
|
|
24
|
+
data:
|
|
25
|
+
- SELECT id, description FROM "%"
|
|
26
|
+
`;
|
|
27
|
+
|
|
28
|
+
function descriptor(name: string, sendsCompleteRows?: boolean): storage.SourceEntityDescriptor {
|
|
29
|
+
return {
|
|
30
|
+
connectionTag: storage.SourceTable.DEFAULT_TAG,
|
|
31
|
+
objectId: name,
|
|
32
|
+
schema: 'public',
|
|
33
|
+
name,
|
|
34
|
+
replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }],
|
|
35
|
+
sendsCompleteRows
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function singleUseIdGenerator(hex: string) {
|
|
40
|
+
let used = false;
|
|
41
|
+
return () => {
|
|
42
|
+
if (used) {
|
|
43
|
+
throw new Error(`Can only generate a single id using ${hex}`);
|
|
44
|
+
}
|
|
45
|
+
used = true;
|
|
46
|
+
return new bson.ObjectId(hex);
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Returns the row payload stored in current_data for the (single) record of `table`, or null if no
|
|
52
|
+
* payload is stored. v1 stores an empty BSON object ({}) when storeCurrentData is false; v3 stores
|
|
53
|
+
* null. Both are normalised to null here so the assertions are version-agnostic.
|
|
54
|
+
*/
|
|
55
|
+
async function storedRowPayload(
|
|
56
|
+
storageVersion: number,
|
|
57
|
+
factory: storage.BucketStorageFactory,
|
|
58
|
+
bucketStorage: storage.SyncRulesBucketStorage,
|
|
59
|
+
syncRulesId: number,
|
|
60
|
+
table: storage.SourceTable
|
|
61
|
+
): Promise<Record<string, any> | null> {
|
|
62
|
+
let data: bson.Binary | null | undefined;
|
|
63
|
+
if (storageVersion < 3) {
|
|
64
|
+
const db = (factory as MongoBucketStorage).db;
|
|
65
|
+
const doc = await db.current_data.findOne({ '_id.g': syncRulesId });
|
|
66
|
+
data = doc?.data;
|
|
67
|
+
} else {
|
|
68
|
+
const db = (bucketStorage as any).db;
|
|
69
|
+
const doc = await db.sourceRecordsV3(syncRulesId, table.id).findOne({});
|
|
70
|
+
data = doc?.data;
|
|
71
|
+
}
|
|
72
|
+
if (data == null) {
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
const decoded = bson.deserialize(data.buffer);
|
|
76
|
+
return Object.keys(decoded).length === 0 ? null : decoded;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function registerStoreCurrentDataTests(storageVersion: number) {
|
|
80
|
+
test('resolveTables derives storeCurrentData fresh each call, with no persisted memory', async () => {
|
|
81
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
82
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
|
|
83
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
84
|
+
|
|
85
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
86
|
+
|
|
87
|
+
// Sources that always send complete rows (e.g. Postgres REPLICA IDENTITY FULL) don't need current_data.
|
|
88
|
+
const complete = await writer.resolveTables({
|
|
89
|
+
connection_id: 1,
|
|
90
|
+
source: descriptor('test_complete', true),
|
|
91
|
+
idGenerator: singleUseIdGenerator('6544e3899293153fa7b38301')
|
|
92
|
+
});
|
|
93
|
+
expect(complete.tables[0].storeCurrentData).toBe(false);
|
|
94
|
+
|
|
95
|
+
// Sources that may send partial/key data still need current_data.
|
|
96
|
+
const partial = await writer.resolveTables({
|
|
97
|
+
connection_id: 1,
|
|
98
|
+
source: descriptor('test_partial', false),
|
|
99
|
+
idGenerator: singleUseIdGenerator('6544e3899293153fa7b38302')
|
|
100
|
+
});
|
|
101
|
+
expect(partial.tables[0].storeCurrentData).toBe(true);
|
|
102
|
+
|
|
103
|
+
// Re-resolving an already-resolved table with no completeness info (sendsCompleteRows undefined)
|
|
104
|
+
// does not remember the earlier value: nothing is persisted, so it falls back to the default
|
|
105
|
+
// (keep a copy).
|
|
106
|
+
const reResolved = await writer.resolveTables({
|
|
107
|
+
connection_id: 1,
|
|
108
|
+
source: descriptor('test_complete')
|
|
109
|
+
});
|
|
110
|
+
expect(reResolved.tables[0].storeCurrentData).toBe(true);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
test('storeCurrentData=false omits the row payload from current_data, data still syncs', async () => {
|
|
114
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
115
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
|
|
116
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
117
|
+
|
|
118
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
119
|
+
const table = await test_utils.resolveTestTable(writer, 'test_data', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
120
|
+
table.storeCurrentData = false;
|
|
121
|
+
|
|
122
|
+
await writer.save({
|
|
123
|
+
sourceTable: table,
|
|
124
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
125
|
+
after: { id: 'test1', description: 'test data' },
|
|
126
|
+
afterReplicaId: test_utils.rid('test1')
|
|
127
|
+
});
|
|
128
|
+
const flushResult = await writer.flush();
|
|
129
|
+
const checkpoint = flushResult!.flushed_op;
|
|
130
|
+
|
|
131
|
+
const batch = await test_utils.fromAsync(
|
|
132
|
+
bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)])
|
|
133
|
+
);
|
|
134
|
+
expect(test_utils.getBatchData(batch)).toMatchObject([{ op: 'PUT', object_id: 'test1' }]);
|
|
135
|
+
|
|
136
|
+
// No row payload retained in current_data.
|
|
137
|
+
expect(await storedRowPayload(storageVersion, factory, bucketStorage, syncRules.id, table)).toBeNull();
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
test('storeCurrentData=true retains the row payload in current_data', async () => {
|
|
141
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
142
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
|
|
143
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
144
|
+
|
|
145
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
146
|
+
const table = await test_utils.resolveTestTable(writer, 'test_data', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
147
|
+
table.storeCurrentData = true;
|
|
148
|
+
|
|
149
|
+
await writer.save({
|
|
150
|
+
sourceTable: table,
|
|
151
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
152
|
+
after: { id: 'test1', description: 'test data' },
|
|
153
|
+
afterReplicaId: test_utils.rid('test1')
|
|
154
|
+
});
|
|
155
|
+
const flushResult = await writer.flush();
|
|
156
|
+
const checkpoint = flushResult!.flushed_op;
|
|
157
|
+
|
|
158
|
+
const batch = await test_utils.fromAsync(
|
|
159
|
+
bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)])
|
|
160
|
+
);
|
|
161
|
+
expect(test_utils.getBatchData(batch)).toMatchObject([{ op: 'PUT', object_id: 'test1' }]);
|
|
162
|
+
|
|
163
|
+
expect(await storedRowPayload(storageVersion, factory, bucketStorage, syncRules.id, table)).toMatchObject({
|
|
164
|
+
id: 'test1'
|
|
165
|
+
});
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
test('storeCurrentData=false processes UPDATE without a stored copy', async () => {
|
|
169
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
170
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
|
|
171
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
172
|
+
|
|
173
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
174
|
+
const table = await test_utils.resolveTestTable(writer, 'test_data', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
175
|
+
table.storeCurrentData = false;
|
|
176
|
+
|
|
177
|
+
// With REPLICA IDENTITY FULL the UPDATE carries the full row, so it is applied from `after`
|
|
178
|
+
// rather than a stored copy of the previous row.
|
|
179
|
+
await writer.save({
|
|
180
|
+
sourceTable: table,
|
|
181
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
182
|
+
after: { id: 'test1', description: 'original' },
|
|
183
|
+
afterReplicaId: test_utils.rid('test1')
|
|
184
|
+
});
|
|
185
|
+
await writer.save({
|
|
186
|
+
sourceTable: table,
|
|
187
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
188
|
+
after: { id: 'test1', description: 'updated' },
|
|
189
|
+
afterReplicaId: test_utils.rid('test1')
|
|
190
|
+
});
|
|
191
|
+
const flushResult = await writer.flush();
|
|
192
|
+
const checkpoint = flushResult!.flushed_op;
|
|
193
|
+
|
|
194
|
+
const batch = await test_utils.fromAsync(
|
|
195
|
+
bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)])
|
|
196
|
+
);
|
|
197
|
+
const data = test_utils.getBatchData(batch);
|
|
198
|
+
expect(data.length).toBeGreaterThan(0);
|
|
199
|
+
expect(data.at(-1)).toMatchObject({ op: 'PUT', object_id: 'test1' });
|
|
200
|
+
|
|
201
|
+
expect(await storedRowPayload(storageVersion, factory, bucketStorage, syncRules.id, table)).toBeNull();
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
describe('MongoDB storage - storeCurrentData', () => {
|
|
206
|
+
for (const storageVersion of TEST_STORAGE_VERSIONS) {
|
|
207
|
+
describe(`storage v${storageVersion}`, () => {
|
|
208
|
+
registerStoreCurrentDataTests(storageVersion);
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
});
|