@powersync/service-core-tests 0.16.0 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/dist/test-utils/AbstractStreamTestContext.d.ts +6 -5
- package/dist/test-utils/AbstractStreamTestContext.js +26 -22
- package/dist/test-utils/AbstractStreamTestContext.js.map +1 -1
- package/dist/test-utils/StorageDataHelpers.d.ts +5 -5
- package/dist/test-utils/StorageDataHelpers.js.map +1 -1
- package/dist/test-utils/bucket-validation.d.ts +3 -3
- package/dist/test-utils/bucket-validation.js +3 -3
- package/dist/test-utils/general-utils.d.ts +13 -2
- package/dist/test-utils/general-utils.js +24 -3
- package/dist/test-utils/general-utils.js.map +1 -1
- package/dist/tests/register-compacting-tests.js +28 -21
- package/dist/tests/register-compacting-tests.js.map +1 -1
- package/dist/tests/register-data-storage-checkpoint-tests.js +8 -2
- package/dist/tests/register-data-storage-checkpoint-tests.js.map +1 -1
- package/dist/tests/register-data-storage-data-tests.js +148 -101
- package/dist/tests/register-data-storage-data-tests.js.map +1 -1
- package/dist/tests/register-data-storage-parameter-tests.js +44 -42
- package/dist/tests/register-data-storage-parameter-tests.js.map +1 -1
- package/dist/tests/register-sync-tests.js +141 -8
- package/dist/tests/register-sync-tests.js.map +1 -1
- package/dist/tests/util.d.ts +2 -2
- package/dist/tests/util.js.map +1 -1
- package/package.json +6 -7
- package/src/test-utils/AbstractStreamTestContext.ts +29 -24
- package/src/test-utils/StorageDataHelpers.ts +7 -6
- package/src/test-utils/bucket-validation.ts +3 -3
- package/src/test-utils/general-utils.ts +35 -7
- package/src/tests/register-compacting-tests.ts +45 -22
- package/src/tests/register-data-storage-checkpoint-tests.ts +12 -2
- package/src/tests/register-data-storage-data-tests.ts +210 -107
- package/src/tests/register-data-storage-parameter-tests.ts +58 -42
- package/src/tests/register-sync-tests.ts +159 -8
- package/src/tests/util.ts +2 -2
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,39 @@
|
|
|
1
1
|
# @powersync/service-core-tests
|
|
2
2
|
|
|
3
|
+
## 0.17.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 71d4a0a: Add logs for checkpoint_complete to assist with debugging. Remove "sync lock" logs.
|
|
8
|
+
- e4f683d: [MongoDB Storage] Add experimental option to allow reading data from secondaries.
|
|
9
|
+
- 71d4a0a: Fix edge case where the service can return incomplete data. This could happen when using bucket priorities, and a checkpoint is invalidated by the compact process while syncing.
|
|
10
|
+
- df9ab1e: Batch concurrent write checkpoint requests and increase request limits to improve throughput.
|
|
11
|
+
- Updated dependencies [71d4a0a]
|
|
12
|
+
- Updated dependencies [e4f683d]
|
|
13
|
+
- Updated dependencies [71d4a0a]
|
|
14
|
+
- Updated dependencies [a6ae678]
|
|
15
|
+
- Updated dependencies [c2edf86]
|
|
16
|
+
- Updated dependencies [df9ab1e]
|
|
17
|
+
- @powersync/service-core@1.23.2
|
|
18
|
+
- @powersync/service-sync-rules@0.38.1
|
|
19
|
+
- @powersync/lib-services-framework@0.9.7
|
|
20
|
+
|
|
21
|
+
## 0.17.0
|
|
22
|
+
|
|
23
|
+
### Minor Changes
|
|
24
|
+
|
|
25
|
+
- a91a08f: [Experimental] Enable incremental reprocessing for MongoDB source + MongoDB storage. This includes significant changes to the v3 storage format.
|
|
26
|
+
|
|
27
|
+
### Patch Changes
|
|
28
|
+
|
|
29
|
+
- Updated dependencies [a91a08f]
|
|
30
|
+
- Updated dependencies [184c39f]
|
|
31
|
+
- Updated dependencies [c3f75df]
|
|
32
|
+
- Updated dependencies [4bd35ea]
|
|
33
|
+
- @powersync/service-core@1.23.0
|
|
34
|
+
- @powersync/service-sync-rules@0.38.0
|
|
35
|
+
- @powersync/lib-services-framework@0.9.6
|
|
36
|
+
|
|
3
37
|
## 0.16.0
|
|
4
38
|
|
|
5
39
|
### Minor Changes
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { BucketStorageFactory, InternalOpId, storage, SyncRulesBucketStorage } from '@powersync/service-core';
|
|
1
|
+
import { BucketStorageFactory, InternalOpId, ReplicationCheckpoint, storage, SyncRulesBucketStorage } from '@powersync/service-core';
|
|
2
2
|
export declare abstract class AbstractStreamTestContext implements AsyncDisposable {
|
|
3
3
|
protected abortController: AbortController;
|
|
4
|
-
protected syncRulesContent?: storage.
|
|
4
|
+
protected syncRulesContent?: storage.PersistedSyncConfigContent;
|
|
5
|
+
protected replicationStream?: storage.PersistedReplicationStream;
|
|
5
6
|
storage?: SyncRulesBucketStorage;
|
|
6
7
|
protected settledReplicationPromise?: Promise<PromiseSettledResult<void>>;
|
|
7
8
|
abstract get factory(): BucketStorageFactory;
|
|
@@ -12,7 +13,7 @@ export declare abstract class AbstractStreamTestContext implements AsyncDisposab
|
|
|
12
13
|
updateSyncRules(content: string): Promise<storage.SyncRulesBucketStorage>;
|
|
13
14
|
loadNextSyncRules(): Promise<storage.SyncRulesBucketStorage>;
|
|
14
15
|
loadActiveSyncRules(): Promise<storage.SyncRulesBucketStorage>;
|
|
15
|
-
private
|
|
16
|
+
private getSyncConfigContent;
|
|
16
17
|
/**
|
|
17
18
|
* Replicate a snapshot, start streaming, and wait for a consistent checkpoint.
|
|
18
19
|
*/
|
|
@@ -25,10 +26,10 @@ export declare abstract class AbstractStreamTestContext implements AsyncDisposab
|
|
|
25
26
|
replicateSnapshot(): Promise<void>;
|
|
26
27
|
abstract getClientCheckpoint(options?: {
|
|
27
28
|
timeout?: number;
|
|
28
|
-
}): Promise<
|
|
29
|
+
}): Promise<ReplicationCheckpoint>;
|
|
29
30
|
getCheckpoint(options?: {
|
|
30
31
|
timeout?: number;
|
|
31
|
-
}): Promise<
|
|
32
|
+
}): Promise<ReplicationCheckpoint>;
|
|
32
33
|
getBucketsDataBatch(buckets: Record<string, InternalOpId>, options?: {
|
|
33
34
|
timeout?: number;
|
|
34
35
|
}): Promise<storage.SyncBucketDataChunk[]>;
|
|
@@ -6,6 +6,7 @@ import { fromAsync } from './stream_utils.js';
|
|
|
6
6
|
export class AbstractStreamTestContext {
|
|
7
7
|
abortController = new AbortController();
|
|
8
8
|
syncRulesContent;
|
|
9
|
+
replicationStream;
|
|
9
10
|
storage;
|
|
10
11
|
settledReplicationPromise;
|
|
11
12
|
async [Symbol.asyncDispose]() {
|
|
@@ -28,32 +29,35 @@ export class AbstractStreamTestContext {
|
|
|
28
29
|
}
|
|
29
30
|
}
|
|
30
31
|
async updateSyncRules(content) {
|
|
31
|
-
const
|
|
32
|
-
this.
|
|
33
|
-
this.
|
|
32
|
+
const stream = await this.factory.updateSyncRules(updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion }));
|
|
33
|
+
this.replicationStream = stream;
|
|
34
|
+
this.syncRulesContent = stream.syncConfigContent[0];
|
|
35
|
+
this.storage = this.factory.getInstance(stream);
|
|
34
36
|
return this.storage;
|
|
35
37
|
}
|
|
36
38
|
async loadNextSyncRules() {
|
|
37
|
-
const
|
|
38
|
-
if (
|
|
39
|
-
throw new Error(`Next sync
|
|
39
|
+
const syncConfig = await this.factory.getDeployingSyncConfig();
|
|
40
|
+
if (syncConfig == null) {
|
|
41
|
+
throw new Error(`Next sync config not available`);
|
|
40
42
|
}
|
|
41
|
-
this.syncRulesContent =
|
|
42
|
-
this.
|
|
43
|
+
this.syncRulesContent = syncConfig.content;
|
|
44
|
+
this.replicationStream = syncConfig.replicationStream;
|
|
45
|
+
this.storage = syncConfig.storage;
|
|
43
46
|
return this.storage;
|
|
44
47
|
}
|
|
45
48
|
async loadActiveSyncRules() {
|
|
46
|
-
const
|
|
47
|
-
if (
|
|
48
|
-
throw new Error(`Active sync
|
|
49
|
+
const syncConfig = await this.factory.getActiveSyncConfig();
|
|
50
|
+
if (syncConfig == null) {
|
|
51
|
+
throw new Error(`Active sync config not available`);
|
|
49
52
|
}
|
|
50
|
-
this.syncRulesContent =
|
|
51
|
-
this.
|
|
53
|
+
this.syncRulesContent = syncConfig.content;
|
|
54
|
+
this.replicationStream = syncConfig.replicationStream;
|
|
55
|
+
this.storage = syncConfig.storage;
|
|
52
56
|
return this.storage;
|
|
53
57
|
}
|
|
54
|
-
|
|
58
|
+
getSyncConfigContent() {
|
|
55
59
|
if (this.syncRulesContent == null) {
|
|
56
|
-
throw new Error('Sync
|
|
60
|
+
throw new Error('Sync config not configured - call updateSyncRules() first');
|
|
57
61
|
}
|
|
58
62
|
return this.syncRulesContent;
|
|
59
63
|
}
|
|
@@ -97,7 +101,7 @@ export class AbstractStreamTestContext {
|
|
|
97
101
|
return checkpoint;
|
|
98
102
|
}
|
|
99
103
|
async getBucketsDataBatch(buckets, options) {
|
|
100
|
-
const helpers = new StorageDataHelpers(this.storage, this.
|
|
104
|
+
const helpers = new StorageDataHelpers(this.storage, this.getSyncConfigContent());
|
|
101
105
|
const checkpoint = await this.getCheckpoint(options);
|
|
102
106
|
return helpers.getBucketsDataBatch(buckets, checkpoint);
|
|
103
107
|
}
|
|
@@ -105,14 +109,14 @@ export class AbstractStreamTestContext {
|
|
|
105
109
|
* This waits for a client checkpoint.
|
|
106
110
|
*/
|
|
107
111
|
async getBucketData(bucket, start, options) {
|
|
108
|
-
const helpers = new StorageDataHelpers(this.storage, this.
|
|
112
|
+
const helpers = new StorageDataHelpers(this.storage, this.getSyncConfigContent());
|
|
109
113
|
const checkpoint = await this.getCheckpoint(options);
|
|
110
114
|
return helpers.getBucketData(bucket, checkpoint, start);
|
|
111
115
|
}
|
|
112
116
|
async getChecksums(buckets, options) {
|
|
113
117
|
const checkpoint = await this.getCheckpoint(options);
|
|
114
|
-
const
|
|
115
|
-
const versionedBuckets = buckets.map((bucket) => bucketRequest(
|
|
118
|
+
const syncConfigContent = this.getSyncConfigContent();
|
|
119
|
+
const versionedBuckets = buckets.map((bucket) => bucketRequest(syncConfigContent, bucket, 0n));
|
|
116
120
|
const checksums = await this.storage.getChecksums(checkpoint, versionedBuckets);
|
|
117
121
|
const unversioned = new Map();
|
|
118
122
|
for (let i = 0; i < buckets.length; i++) {
|
|
@@ -132,9 +136,9 @@ export class AbstractStreamTestContext {
|
|
|
132
136
|
if (typeof start == 'string') {
|
|
133
137
|
start = BigInt(start);
|
|
134
138
|
}
|
|
135
|
-
const
|
|
136
|
-
const
|
|
137
|
-
const map = [bucketRequest(
|
|
139
|
+
const syncConfigContent = this.getSyncConfigContent();
|
|
140
|
+
const checkpoint = await this.storage.getCheckpoint();
|
|
141
|
+
const map = [bucketRequest(syncConfigContent, bucket, start)];
|
|
138
142
|
const batch = this.storage.getBucketDataBatch(checkpoint, map);
|
|
139
143
|
const batches = await fromAsync(batch);
|
|
140
144
|
return batches[0]?.chunkData.data ?? [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractStreamTestContext.js","sourceRoot":"","sources":["../../src/test-utils/AbstractStreamTestContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,
|
|
1
|
+
{"version":3,"file":"AbstractStreamTestContext.js","sourceRoot":"","sources":["../../src/test-utils/AbstractStreamTestContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAIL,cAAc,EAGd,gBAAgB,EAChB,uBAAuB,EACxB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE9C,MAAM,OAAgB,yBAAyB;IACnC,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IACxC,gBAAgB,CAAsC;IACtD,iBAAiB,CAAsC;IAC1D,OAAO,CAA0B;IAC9B,yBAAyB,CAAuC;IAK1E,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACzB,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACvB,CAAC;IAID,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,yBAAyB,CAAC;YACrC,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yFAAyF;YACzF,yCAAyC;YACzC,yFAAyF;YACzF,qBAAqB;YACrB,OAAO,CAAC,KAAK,CAAC,8CAA8C,EAAE,CAAC,CAAC,CAAC;YACjE,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe;QACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAC/C,uBAAuB,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,CAC1F,CAAC;QACF,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC;QAChC,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;QACpD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAChD,OAAO,IAAI,CAAC,OAAQ,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;QAC/D,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,UAAU,CAAC,OAAO,CAAC;QAC3C,IAAI,CAAC,iBAAiB,GAAG,UAAU,CAAC,iBAAiB,CAAC;QACtD,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QAClC,OAAO,IAAI,CAAC,OAAQ,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAC5D,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACtD,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,UAAU,CAAC,OAAO,CAAC;QAC3C,IAAI,CAAC,iBAAiB,GAAG,UAAU,CAAC,iBAAiB,CAAC;QACtD,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QAClC,OAAO,IAAI,CAAC,OAAQ,CAAC;IACvB,CAAC;IAEO,oBAAoB;QAC1B,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC/E,CAAC;QACD,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACzB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,6BAA6B;QAC7B,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;IAC7B,CAAC;IAKD;;OAEG;IACH,KAAK,CAAC,iBAAiB;QACrB,qDAAqD;QACrD,IAAI,CAAC,yBAAyB,GAAG,cAAc,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;QAC3E,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,yBAAyB,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAC;QACxG,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,YAAY,uBAAuB,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBAC5D,yGAAyG;gBACzG,wGAAwG;gBACxG,iGAAiG;gBACjG,MAAM,CAAC,CAAC,KAAK,CAAC;YAChB,CAAC;YACD,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAID,KAAK,CAAC,aAAa,CAAC,OAA8B;QAChD,IAAI,UAAU,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YAClC,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC;YACjC,gBAAgB,CAAC,IAAI,CAAC,yBAA0B,CAAC;SAClD,CAAC,CAAC;QACH,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,qFAAqF;YACrF,2BAA2B;YAC3B,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,OAAqC,EAAE,OAA8B;QAC7F,MAAM,OAAO,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAQ,EAAE,IAAI,CAAC,oBAAoB,EAAE,CAAC,CAAC;QACnF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACrD,OAAO,OAAO,CAAC,mBAAmB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,MAAc,EAAE,KAAyC,EAAE,OAA8B;QAC3G,MAAM,OAAO,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAQ,EAAE,IAAI,CAAC,oBAAoB,EAAE,CAAC,CAAC;QACnF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACrD,OAAO,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,OAAiB,EAAE,OAA8B;QAClE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACrD,MAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACtD,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC,iBAAiB,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;QAC/F,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAQ,CAAC,YAAY,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC;QAEjF,MAAM,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAE,CAAC,CAAC;QAC1E,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,OAA8B;QAC9D,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;QAC7D,OAAO,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB,CAAC,MAAc,EAAE,KAAyC;QAClF,KAAK,KAAK,EAAE,CAAC;QACb,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;QACD,MAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACtD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAQ,CAAC,aAAa,EAAE,CAAC;QACvD,MAAM,GAAG,GAAG,CAAC,aAAa,CAAC,iBAAiB,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAQ,CAAC,kBAAkB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QAChE,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC;QACvC,OAAO,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC;IAC1C,CAAC;CACF"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { InternalOpId, OplogEntry,
|
|
1
|
+
import { InternalOpId, OplogEntry, ParsedSyncConfigSet, PersistedSyncConfigContent, ReplicationCheckpoint, SyncRulesBucketStorage } from '@powersync/service-core';
|
|
2
2
|
export declare class StorageDataHelpers {
|
|
3
3
|
storage: SyncRulesBucketStorage;
|
|
4
|
-
syncRules:
|
|
5
|
-
constructor(storage: SyncRulesBucketStorage, syncRules:
|
|
6
|
-
getBucketData(bucket: string, checkpoint:
|
|
7
|
-
getBucketsDataBatch(buckets: Record<string, InternalOpId>, checkpoint:
|
|
4
|
+
syncRules: PersistedSyncConfigContent | ParsedSyncConfigSet;
|
|
5
|
+
constructor(storage: SyncRulesBucketStorage, syncRules: PersistedSyncConfigContent | ParsedSyncConfigSet);
|
|
6
|
+
getBucketData(bucket: string, checkpoint: ReplicationCheckpoint, start?: InternalOpId | string | undefined): Promise<OplogEntry<import("@powersync/service-core").StoredOplogData>[]>;
|
|
7
|
+
getBucketsDataBatch(buckets: Record<string, InternalOpId>, checkpoint: ReplicationCheckpoint): Promise<import("@powersync/service-core").SyncBucketDataChunk[]>;
|
|
8
8
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"StorageDataHelpers.js","sourceRoot":"","sources":["../../src/test-utils/StorageDataHelpers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"StorageDataHelpers.js","sourceRoot":"","sources":["../../src/test-utils/StorageDataHelpers.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE9C,MAAM,OAAO,kBAAkB;IAC7B,OAAO,CAAyB;IAChC,SAAS,CAAmD;IAE5D,YAAY,OAA+B,EAAE,SAA2D;QACtG,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,MAAc,EAAE,UAAiC,EAAE,KAAyC;QAC9G,KAAK,KAAK,EAAE,CAAC;QACb,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;QACD,IAAI,GAAG,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QACzD,IAAI,IAAI,GAAiB,EAAE,CAAC;QAC5B,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,KAAK,GAAG,IAAI,CAAC,OAAQ,CAAC,kBAAkB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YAEhE,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC;YACvC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YACrD,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;gBAC3D,MAAM;YACR,CAAC;YACD,GAAG,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC1F,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,OAAqC,EAAE,UAAiC;QAChG,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QAC3G,OAAO,SAAS,CAAC,IAAI,CAAC,OAAQ,CAAC,kBAAkB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC;IACtE,CAAC;CACF"}
|
|
@@ -7,7 +7,7 @@ import { utils } from '@powersync/service-core';
|
|
|
7
7
|
* All other operations are replaced with a single CLEAR operation,
|
|
8
8
|
* summing their checksums, and using a 0 as an op_id.
|
|
9
9
|
*
|
|
10
|
-
* This is the function $r(B)$, as described in /docs/bucket-properties.md.
|
|
10
|
+
* This is the function $r(B)$, as described in /docs/storage/bucket-properties.md.
|
|
11
11
|
*/
|
|
12
12
|
export declare function reduceBucket(operations: utils.OplogEntry[]): utils.OplogEntry<utils.StoredOplogData>[];
|
|
13
13
|
/**
|
|
@@ -16,7 +16,7 @@ import { reduceBucket } from '@/util/utils.js';
|
|
|
16
16
|
import { expect } from 'vitest';
|
|
17
17
|
|
|
18
18
|
/**
|
|
19
|
-
* Validate this property, as described in /docs/bucket-properties.md:
|
|
19
|
+
* Validate this property, as described in /docs/storage/bucket-properties.md:
|
|
20
20
|
*
|
|
21
21
|
* $r(B_{[..id_n]}) = r(r(B_{[..id_i]}) \cup B_{[id_{i+1}..id_n]}) \;\forall\; i \in [1..n]$
|
|
22
22
|
*
|
|
@@ -27,7 +27,7 @@ import { expect } from 'vitest';
|
|
|
27
27
|
export declare function validateBucket(bucket: utils.OplogEntry[]): void;
|
|
28
28
|
/**
|
|
29
29
|
* Validate these properties for a bucket $B$ and its compacted version $B'$,:
|
|
30
|
-
* as described in /docs/bucket-properties.md:
|
|
30
|
+
* as described in /docs/storage/bucket-properties.md:
|
|
31
31
|
*
|
|
32
32
|
* 1. $r(B) = r(B')$
|
|
33
33
|
* 2. $r(B_{[..c]}) = r(r(B_{[..c_i]}) \cup B'_{[c_i+1..c]}) \;\forall\; c_i \in B$
|
|
@@ -8,7 +8,7 @@ import { expect } from 'vitest';
|
|
|
8
8
|
* All other operations are replaced with a single CLEAR operation,
|
|
9
9
|
* summing their checksums, and using a 0 as an op_id.
|
|
10
10
|
*
|
|
11
|
-
* This is the function $r(B)$, as described in /docs/bucket-properties.md.
|
|
11
|
+
* This is the function $r(B)$, as described in /docs/storage/bucket-properties.md.
|
|
12
12
|
*/
|
|
13
13
|
export function reduceBucket(operations) {
|
|
14
14
|
let rowState = new Map();
|
|
@@ -60,7 +60,7 @@ import { reduceBucket } from '@/util/utils.js';
|
|
|
60
60
|
import { expect } from 'vitest';
|
|
61
61
|
|
|
62
62
|
/**
|
|
63
|
-
* Validate this property, as described in /docs/bucket-properties.md:
|
|
63
|
+
* Validate this property, as described in /docs/storage/bucket-properties.md:
|
|
64
64
|
*
|
|
65
65
|
* $r(B_{[..id_n]}) = r(r(B_{[..id_i]}) \cup B_{[id_{i+1}..id_n]}) \;\forall\; i \in [1..n]$
|
|
66
66
|
*
|
|
@@ -82,7 +82,7 @@ export function validateBucket(bucket) {
|
|
|
82
82
|
}
|
|
83
83
|
/**
|
|
84
84
|
* Validate these properties for a bucket $B$ and its compacted version $B'$,:
|
|
85
|
-
* as described in /docs/bucket-properties.md:
|
|
85
|
+
* as described in /docs/storage/bucket-properties.md:
|
|
86
86
|
*
|
|
87
87
|
* 1. $r(B) = r(B')$
|
|
88
88
|
* 2. $r(B_{[..c]}) = r(r(B_{[..c_i]}) \cup B'_{[c_i+1..c]}) \;\forall\; c_i \in B$
|
|
@@ -2,8 +2,19 @@ import { BucketDataRequest, InternalOpId, storage, utils } from '@powersync/serv
|
|
|
2
2
|
import { GetQuerierOptions, RequestParameters } from '@powersync/service-sync-rules';
|
|
3
3
|
import * as bson from 'bson';
|
|
4
4
|
export declare const ZERO_LSN = "0/0";
|
|
5
|
-
export declare const PARSE_OPTIONS: storage.
|
|
5
|
+
export declare const PARSE_OPTIONS: storage.ParseSyncConfigOptions;
|
|
6
6
|
export declare const BATCH_OPTIONS: storage.CreateWriterOptions;
|
|
7
|
+
export declare function testCheckpoint(checkpoint: InternalOpId, lsn?: string | null): storage.ReplicationCheckpoint;
|
|
8
|
+
/**
|
|
9
|
+
* Deploy a sync config and return both the replication stream (for {@link storage.BucketStorageFactory.getInstance})
|
|
10
|
+
* and the deployed config content (for parsing / bucket requests).
|
|
11
|
+
*
|
|
12
|
+
* Replication streams and sync config content are separate concerns since one stream can hold multiple configs.
|
|
13
|
+
*/
|
|
14
|
+
export declare function deploySyncRules(factory: storage.BucketStorageFactory, options: storage.UpdateSyncRulesOptions): Promise<{
|
|
15
|
+
stream: storage.PersistedReplicationStream;
|
|
16
|
+
content: storage.PersistedSyncConfigContent;
|
|
17
|
+
}>;
|
|
7
18
|
/**
|
|
8
19
|
* With newer storage versions, we need actual test tables, resolved via the writer.
|
|
9
20
|
*/
|
|
@@ -20,7 +31,7 @@ export declare function getBatchData(batch: utils.SyncBucketData[] | storage.Syn
|
|
|
20
31
|
* Bucket names no longer purely depend on the sync config.
|
|
21
32
|
* This converts a bucket name like "global[]" into the actual bucket name, for use in tests.
|
|
22
33
|
*/
|
|
23
|
-
export declare function bucketRequest(syncRules: storage.
|
|
34
|
+
export declare function bucketRequest(syncRules: storage.PersistedSyncConfigContent | storage.ParsedSyncConfigSet, bucket: string, start?: InternalOpId | string | number): BucketDataRequest;
|
|
24
35
|
export declare function getBatchMeta(batch: utils.SyncBucketData[] | storage.SyncBucketDataChunk[] | storage.SyncBucketDataChunk): {
|
|
25
36
|
has_more: boolean;
|
|
26
37
|
after: string;
|
|
@@ -10,6 +10,26 @@ export const BATCH_OPTIONS = {
|
|
|
10
10
|
zeroLSN: ZERO_LSN,
|
|
11
11
|
storeCurrentData: true
|
|
12
12
|
};
|
|
13
|
+
export function testCheckpoint(checkpoint, lsn = null) {
|
|
14
|
+
return {
|
|
15
|
+
checkpoint,
|
|
16
|
+
lsn,
|
|
17
|
+
async getParameterSets() {
|
|
18
|
+
return [];
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Deploy a sync config and return both the replication stream (for {@link storage.BucketStorageFactory.getInstance})
|
|
24
|
+
* and the deployed config content (for parsing / bucket requests).
|
|
25
|
+
*
|
|
26
|
+
* Replication streams and sync config content are separate concerns since one stream can hold multiple configs.
|
|
27
|
+
*/
|
|
28
|
+
export async function deploySyncRules(factory, options) {
|
|
29
|
+
const stream = await factory.updateSyncRules(options);
|
|
30
|
+
const content = stream.syncConfigContent[0];
|
|
31
|
+
return { stream, content };
|
|
32
|
+
}
|
|
13
33
|
/**
|
|
14
34
|
* With newer storage versions, we need actual test tables, resolved via the writer.
|
|
15
35
|
*/
|
|
@@ -61,7 +81,7 @@ export function getBatchData(batch) {
|
|
|
61
81
|
});
|
|
62
82
|
}
|
|
63
83
|
function isParsedSyncRules(syncRules) {
|
|
64
|
-
return syncRules.
|
|
84
|
+
return syncRules.syncConfigs !== undefined;
|
|
65
85
|
}
|
|
66
86
|
/**
|
|
67
87
|
* Bucket names no longer purely depend on the sync config.
|
|
@@ -73,9 +93,10 @@ export function bucketRequest(syncRules, bucket, start) {
|
|
|
73
93
|
const parameterStart = bucket.indexOf('[');
|
|
74
94
|
const definitionName = bucket.substring(0, parameterStart);
|
|
75
95
|
const parameters = bucket.substring(parameterStart);
|
|
76
|
-
const
|
|
96
|
+
const availableSources = parsed.syncConfigs.flatMap((config) => config.config.bucketDataSources);
|
|
97
|
+
const source = availableSources.find((b) => b.uniqueName === definitionName);
|
|
77
98
|
if (source == null) {
|
|
78
|
-
throw new Error(`Failed to find global bucket ${bucket}`);
|
|
99
|
+
throw new Error(`Failed to find global bucket ${bucket}. Available: ${availableSources.map((s) => s.uniqueName).join(',')}`);
|
|
79
100
|
}
|
|
80
101
|
const bucketName = hydrationState.getBucketSourceScope(source).bucketPrefix + parameters;
|
|
81
102
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"general-utils.js","sourceRoot":"","sources":["../../src/test-utils/general-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmC,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACtG,OAAO,EAAqB,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACrF,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,CAAC;AAE9B,MAAM,CAAC,MAAM,aAAa,
|
|
1
|
+
{"version":3,"file":"general-utils.js","sourceRoot":"","sources":["../../src/test-utils/general-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmC,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACtG,OAAO,EAAqB,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACrF,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,CAAC;AAE9B,MAAM,CAAC,MAAM,aAAa,GAAmC;IAC3D,aAAa,EAAE,QAAQ;CACxB,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAgC;IACxD,GAAG,aAAa;IAChB,OAAO,EAAE,QAAQ;IACjB,gBAAgB,EAAE,IAAI;CACvB,CAAC;AAEF,MAAM,UAAU,cAAc,CAAC,UAAwB,EAAE,MAAqB,IAAI;IAChF,OAAO;QACL,UAAU;QACV,GAAG;QACH,KAAK,CAAC,gBAAgB;YACpB,OAAO,EAAE,CAAC;QACZ,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAqC,EACrC,OAAuC;IAEvC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;IAC5C,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAkC,EAClC,IAAY,EACZ,gBAAsC,EACtC,OAAoC,EACpC,UAAkB,CAAC;IAEnB,KAAK,OAAO,CAAC;IACb,KAAK,OAAO,CAAC;IAEb,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,gBAAgB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACpF,yDAAyD;IACzD,4EAA4E;IAC5E,MAAM,QAAQ,GAAG,wBAAwB,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACvF,MAAM,EAAE,GAAG,OAAO,CAAC,cAAc,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IACpF,IAAI,aAAa,GAAG,KAAK,CAAC;IAE1B,MAAM,MAAM,GAAmC;QAC7C,aAAa,EAAE,OAAO,CAAC,WAAW,CAAC,WAAW;QAC9C,QAAQ,EAAE,KAAK;QACf,MAAM,EAAE,QAAQ;QAChB,IAAI,EAAE,IAAI;QACV,gBAAgB,EAAE,CAAC,gBAAgB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;KAChH,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;QAC1C,aAAa,EAAE,CAAC;QAChB,MAAM;QACN,WAAW,EAAE,GAAG,EAAE;YAChB,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;YAChF,CAAC;YACD,aAAa,GAAG,IAAI,CAAC;YACrB,OAAO,EAAE,CAAC;QACZ,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACjC,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,gCAAgC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAClF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,KAA2F;IAE3F,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1B,OAAO;YACL,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,QAAQ,EAAE,CAAC,CAAC,QAAQ;SACrB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CACxB,SAA2E;IAE3E,OAAQ,SAAyC,CAAC,WAAW,KAAK,SAAS,CAAC;AAC9E,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAC3B,SAA2E,EAC3E,MAAc,EACd,KAAsC;IAEtC,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAC1F,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;IAC7C,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;IAC3D,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACpD,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACjG,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,cAAc,CAAC,CAAC;IAE7E,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CACb,gCAAgC,MAAM,gBAAgB,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAC5G,CAAC;IACJ,CAAC;IACD,MAAM,UAAU,GAAG,cAAc,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,YAAY,GAAG,UAAU,CAAC;IACzF,OAAO;QACL,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;QAC1B,MAAM,EAAE,MAAM;KACf,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,KAA2F;IAE3F,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,UAAU,EAAE,KAAK,CAAC,UAAU;KAC7B,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CACf,KAA2F;IAE3F,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC,SAAS,CAAC;IACzB,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACrB,IAAK,KAAqC,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;QAC7D,OAAQ,KAAqC,CAAC,SAAS,CAAC;IAC1D,CAAC;SAAM,CAAC;QACN,OAAO,KAA6B,CAAC;IACvC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,GAAG,CAAC,EAAU;IAC5B,OAAO,KAAK,CAAC,0BAA0B,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;AACrG,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,gBAAmC;IAChE,OAAO;QACL,gBAAgB;QAChB,iBAAiB,EAAE,IAAI;QACvB,OAAO,EAAE,EAAE;KACZ,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,UAA+B,EAC/B,gBAAsC;IAEtC,OAAO,IAAI,iBAAiB,CAAC,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,gBAAgB,IAAI,EAAE,CAAC,CAAC;AACnF,CAAC;AAED,MAAM,UAAU,YAAY,CAA6B,GAAM;IAC7D,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,GAAG,CAAC;IAChC,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -13,6 +13,7 @@ bucket_definitions:
|
|
|
13
13
|
data: [select * from test]
|
|
14
14
|
`));
|
|
15
15
|
const bucketStorage = factory.getInstance(syncRules);
|
|
16
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
16
17
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
17
18
|
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
18
19
|
await writer.markAllSnapshotDone('1/1');
|
|
@@ -43,10 +44,10 @@ bucket_definitions:
|
|
|
43
44
|
await writer.commit('1/1');
|
|
44
45
|
await writer.flush();
|
|
45
46
|
const checkpoint = writer.last_flushed_op;
|
|
46
|
-
const request = bucketRequest(
|
|
47
|
-
const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
47
|
+
const request = bucketRequest(syncRulesContent, 'global[]');
|
|
48
|
+
const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request]));
|
|
48
49
|
const dataBefore = batchBefore.chunkData.data;
|
|
49
|
-
const checksumBefore = await bucketStorage.getChecksums(checkpoint, [request]);
|
|
50
|
+
const checksumBefore = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request]);
|
|
50
51
|
expect(dataBefore).toMatchObject([
|
|
51
52
|
{
|
|
52
53
|
object_id: 't1',
|
|
@@ -72,11 +73,11 @@ bucket_definitions:
|
|
|
72
73
|
minBucketChanges: 1,
|
|
73
74
|
minChangeRatio: 0
|
|
74
75
|
});
|
|
75
|
-
const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
76
|
+
const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request]));
|
|
76
77
|
const dataAfter = batchAfter.chunkData.data;
|
|
77
|
-
const checksumAfter = await bucketStorage.getChecksums(checkpoint, [request]);
|
|
78
|
+
const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request]);
|
|
78
79
|
bucketStorage.clearChecksumCache();
|
|
79
|
-
const checksumAfter2 = await bucketStorage.getChecksums(checkpoint, [request]);
|
|
80
|
+
const checksumAfter2 = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request]);
|
|
80
81
|
expect(batchAfter.targetOp).toEqual(3n);
|
|
81
82
|
expect(dataAfter).toMatchObject([
|
|
82
83
|
dataBefore[0],
|
|
@@ -104,6 +105,7 @@ bucket_definitions:
|
|
|
104
105
|
data: [select * from test]
|
|
105
106
|
`));
|
|
106
107
|
const bucketStorage = factory.getInstance(syncRules);
|
|
108
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
107
109
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
108
110
|
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
109
111
|
await writer.markAllSnapshotDone('1/1');
|
|
@@ -142,10 +144,10 @@ bucket_definitions:
|
|
|
142
144
|
await writer.commit('1/1');
|
|
143
145
|
await writer.flush();
|
|
144
146
|
const checkpoint = writer.last_flushed_op;
|
|
145
|
-
const request = bucketRequest(
|
|
146
|
-
const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
147
|
+
const request = bucketRequest(syncRulesContent, 'global[]');
|
|
148
|
+
const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request]));
|
|
147
149
|
const dataBefore = batchBefore.chunkData.data;
|
|
148
|
-
const checksumBefore = await bucketStorage.getChecksums(checkpoint, [request]);
|
|
150
|
+
const checksumBefore = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request]);
|
|
149
151
|
// op_id sequence depends on the storage implementation
|
|
150
152
|
expect(dataBefore).toMatchObject([
|
|
151
153
|
{
|
|
@@ -172,10 +174,10 @@ bucket_definitions:
|
|
|
172
174
|
minBucketChanges: 1,
|
|
173
175
|
minChangeRatio: 0
|
|
174
176
|
});
|
|
175
|
-
const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
177
|
+
const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request]));
|
|
176
178
|
const dataAfter = batchAfter.chunkData.data;
|
|
177
179
|
bucketStorage.clearChecksumCache();
|
|
178
|
-
const checksumAfter = await bucketStorage.getChecksums(checkpoint, [request]);
|
|
180
|
+
const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request]);
|
|
179
181
|
expect(batchAfter.targetOp).toBeLessThanOrEqual(checkpoint);
|
|
180
182
|
expect(dataAfter).toMatchObject([
|
|
181
183
|
{
|
|
@@ -202,6 +204,7 @@ bucket_definitions:
|
|
|
202
204
|
data: [select * from test]
|
|
203
205
|
`));
|
|
204
206
|
const bucketStorage = factory.getInstance(syncRules);
|
|
207
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
205
208
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
206
209
|
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
207
210
|
await writer.markAllSnapshotDone('1/1');
|
|
@@ -232,7 +235,7 @@ bucket_definitions:
|
|
|
232
235
|
await writer.commit('1/1');
|
|
233
236
|
await writer.flush();
|
|
234
237
|
const checkpoint1 = writer.last_flushed_op;
|
|
235
|
-
const request = bucketRequest(
|
|
238
|
+
const request = bucketRequest(syncRulesContent, 'global[]');
|
|
236
239
|
await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
237
240
|
const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
|
|
238
241
|
await writer2.save({
|
|
@@ -253,10 +256,10 @@ bucket_definitions:
|
|
|
253
256
|
minBucketChanges: 1,
|
|
254
257
|
minChangeRatio: 0
|
|
255
258
|
});
|
|
256
|
-
const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request]));
|
|
259
|
+
const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint2), [request]));
|
|
257
260
|
const dataAfter = batchAfter.chunkData.data;
|
|
258
261
|
await bucketStorage.clearChecksumCache();
|
|
259
|
-
const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
|
|
262
|
+
const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint2), [request]);
|
|
260
263
|
expect(dataAfter).toMatchObject([
|
|
261
264
|
{
|
|
262
265
|
op: 'CLEAR'
|
|
@@ -279,6 +282,7 @@ bucket_definitions:
|
|
|
279
282
|
data:
|
|
280
283
|
- select * from test where b = bucket.b`));
|
|
281
284
|
const bucketStorage = factory.getInstance(syncRules);
|
|
285
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
282
286
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
283
287
|
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
284
288
|
await writer.markAllSnapshotDone('1/1');
|
|
@@ -351,7 +355,7 @@ bucket_definitions:
|
|
|
351
355
|
minBucketChanges: 1,
|
|
352
356
|
minChangeRatio: 0
|
|
353
357
|
});
|
|
354
|
-
const batchAfter = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(
|
|
358
|
+
const batchAfter = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), bucketRequestMap(syncRulesContent, [
|
|
355
359
|
['grouped["b1"]', 0n],
|
|
356
360
|
['grouped["b2"]', 0n]
|
|
357
361
|
])));
|
|
@@ -388,6 +392,7 @@ bucket_definitions:
|
|
|
388
392
|
data: [select * from test]
|
|
389
393
|
`));
|
|
390
394
|
const bucketStorage = factory.getInstance(syncRules);
|
|
395
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
391
396
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
392
397
|
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
393
398
|
await writer.markAllSnapshotDone('1/1');
|
|
@@ -437,9 +442,9 @@ bucket_definitions:
|
|
|
437
442
|
await writer2.commit('2/1');
|
|
438
443
|
await writer2.flush();
|
|
439
444
|
const checkpoint2 = writer2.last_flushed_op;
|
|
440
|
-
const request = bucketRequest(
|
|
445
|
+
const request = bucketRequest(syncRulesContent, 'global[]');
|
|
441
446
|
await bucketStorage.clearChecksumCache();
|
|
442
|
-
const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
|
|
447
|
+
const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint2), [request]);
|
|
443
448
|
const globalChecksum = checksumAfter.get(request.bucket);
|
|
444
449
|
expect(globalChecksum).toMatchObject({
|
|
445
450
|
bucket: request.bucket,
|
|
@@ -456,6 +461,7 @@ bucket_definitions:
|
|
|
456
461
|
data: [select * from test]
|
|
457
462
|
`));
|
|
458
463
|
const bucketStorage = factory.getInstance(syncRules);
|
|
464
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
459
465
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
460
466
|
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
461
467
|
await writer.markAllSnapshotDone('1/1');
|
|
@@ -478,7 +484,7 @@ bucket_definitions:
|
|
|
478
484
|
await writer.commit('1/1');
|
|
479
485
|
await writer.flush();
|
|
480
486
|
// Get checksums here just to populate the cache
|
|
481
|
-
await bucketStorage.getChecksums(writer.last_flushed_op, bucketRequests(
|
|
487
|
+
await bucketStorage.getChecksums(test_utils.testCheckpoint(writer.last_flushed_op), bucketRequests(syncRulesContent, ['global[]']));
|
|
482
488
|
await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
483
489
|
const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
|
|
484
490
|
await writer2.save({
|
|
@@ -499,9 +505,9 @@ bucket_definitions:
|
|
|
499
505
|
minChangeRatio: 0
|
|
500
506
|
});
|
|
501
507
|
const checkpoint2 = writer2.last_flushed_op;
|
|
502
|
-
const request = bucketRequest(
|
|
508
|
+
const request = bucketRequest(syncRulesContent, 'global[]');
|
|
503
509
|
// Check that the checksum was correctly updated with the clear operation after having a cached checksum
|
|
504
|
-
const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
|
|
510
|
+
const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint2), [request]);
|
|
505
511
|
const globalChecksum = checksumAfter.get(request.bucket);
|
|
506
512
|
expect(globalChecksum).toMatchObject({
|
|
507
513
|
bucket: request.bucket,
|
|
@@ -518,6 +524,7 @@ bucket_definitions:
|
|
|
518
524
|
data: [select * from test]
|
|
519
525
|
`));
|
|
520
526
|
const bucketStorage = factory.getInstance(syncRules);
|
|
527
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
521
528
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
522
529
|
const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
|
|
523
530
|
await writer.markAllSnapshotDone('1/1');
|
|
@@ -550,7 +557,7 @@ bucket_definitions:
|
|
|
550
557
|
minBucketChanges: 1,
|
|
551
558
|
minChangeRatio: 0
|
|
552
559
|
});
|
|
553
|
-
const batchAfterDefaultCompact = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(
|
|
560
|
+
const batchAfterDefaultCompact = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint2), bucketRequestMap(syncRulesContent, [['global[]', 0n]])));
|
|
554
561
|
// Operation 1 should remain a PUT because op_id=2 is above the default maxOpId checkpoint.
|
|
555
562
|
expect(batchAfterDefaultCompact.chunkData.data).toMatchObject([
|
|
556
563
|
{ op_id: '1', op: 'PUT', object_id: 't1' },
|