@powersync/service-core 0.18.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/dist/api/RouteAPI.d.ts +9 -1
- package/dist/api/diagnostics.js +107 -169
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/entry/commands/compact-action.js +10 -73
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +1 -1
- package/dist/replication/AbstractReplicator.js +2 -65
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +3 -3
- package/dist/routes/endpoints/admin.d.ts +6 -6
- package/dist/routes/endpoints/checkpointing.js +16 -85
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +5 -5
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +5 -5
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/runner/teardown.js +3 -65
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +1 -442
- package/dist/storage/BucketStorage.js +0 -9
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +130 -0
- package/dist/storage/BucketStorageBatch.js +10 -0
- package/dist/storage/BucketStorageBatch.js.map +1 -0
- package/dist/storage/BucketStorageFactory.d.ts +136 -0
- package/dist/storage/BucketStorageFactory.js +2 -0
- package/dist/storage/BucketStorageFactory.js.map +1 -0
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +20 -0
- package/dist/storage/PersistedSyncRulesContent.js +2 -0
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -0
- package/dist/storage/ReplicationEventPayload.d.ts +1 -1
- package/dist/storage/ReplicationLock.d.ts +4 -0
- package/dist/storage/ReplicationLock.js +2 -0
- package/dist/storage/ReplicationLock.js.map +1 -0
- package/dist/storage/StorageEngine.d.ts +4 -4
- package/dist/storage/StorageEngine.js +2 -2
- package/dist/storage/StorageEngine.js.map +1 -1
- package/dist/storage/StorageProvider.d.ts +4 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +201 -0
- package/dist/storage/SyncRulesBucketStorage.js +7 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -0
- package/dist/storage/bson.d.ts +11 -3
- package/dist/storage/bson.js +24 -2
- package/dist/storage/bson.js.map +1 -1
- package/dist/storage/storage-index.d.ts +5 -0
- package/dist/storage/storage-index.js +5 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +91 -0
- package/dist/sync/BucketChecksumState.js +313 -0
- package/dist/sync/BucketChecksumState.js.map +1 -0
- package/dist/sync/sync-index.d.ts +1 -0
- package/dist/sync/sync-index.js +1 -0
- package/dist/sync/sync-index.js.map +1 -1
- package/dist/sync/sync.d.ts +7 -3
- package/dist/sync/sync.js +131 -135
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +9 -0
- package/dist/sync/util.js +44 -0
- package/dist/sync/util.js.map +1 -1
- package/dist/util/checkpointing.d.ts +13 -0
- package/dist/util/checkpointing.js +29 -0
- package/dist/util/checkpointing.js.map +1 -0
- package/dist/util/protocol-types.d.ts +13 -4
- package/dist/util/util-index.d.ts +1 -0
- package/dist/util/util-index.js +1 -0
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +0 -1
- package/dist/util/utils.js +0 -9
- package/dist/util/utils.js.map +1 -1
- package/package.json +4 -4
- package/src/api/RouteAPI.ts +11 -1
- package/src/api/diagnostics.ts +1 -1
- package/src/entry/commands/compact-action.ts +2 -3
- package/src/modules/AbstractModule.ts +1 -1
- package/src/replication/AbstractReplicator.ts +1 -2
- package/src/routes/endpoints/checkpointing.ts +11 -22
- package/src/routes/endpoints/socket-route.ts +7 -5
- package/src/routes/endpoints/sync-stream.ts +8 -5
- package/src/runner/teardown.ts +1 -1
- package/src/storage/BucketStorage.ts +1 -552
- package/src/storage/BucketStorageBatch.ts +158 -0
- package/src/storage/BucketStorageFactory.ts +156 -0
- package/src/storage/ChecksumCache.ts +1 -0
- package/src/storage/PersistedSyncRulesContent.ts +26 -0
- package/src/storage/ReplicationEventPayload.ts +1 -1
- package/src/storage/ReplicationLock.ts +5 -0
- package/src/storage/StorageEngine.ts +4 -4
- package/src/storage/StorageProvider.ts +4 -1
- package/src/storage/SyncRulesBucketStorage.ts +256 -0
- package/src/storage/bson.ts +28 -4
- package/src/storage/storage-index.ts +5 -0
- package/src/sync/BucketChecksumState.ts +392 -0
- package/src/sync/sync-index.ts +1 -0
- package/src/sync/sync.ts +173 -157
- package/src/sync/util.ts +54 -0
- package/src/util/checkpointing.ts +41 -0
- package/src/util/protocol-types.ts +16 -4
- package/src/util/util-index.ts +1 -0
- package/src/util/utils.ts +0 -10
- package/test/src/auth.test.ts +5 -5
- package/test/src/sync/BucketChecksumState.test.ts +565 -0
- package/test/src/sync/util.test.ts +34 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { ObserverClient } from '@powersync/lib-services-framework';
|
|
2
|
+
import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
|
|
3
|
+
import { ReplicationEventPayload } from './ReplicationEventPayload.js';
|
|
4
|
+
import { ReplicationLock } from './ReplicationLock.js';
|
|
5
|
+
import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
|
|
6
|
+
/**
|
|
7
|
+
* Represents a configured storage provider.
|
|
8
|
+
*
|
|
9
|
+
* The provider can handle multiple copies of sync rules concurrently, each with their own storage.
|
|
10
|
+
* This is to handle replication of a new version of sync rules, while the old version is still active.
|
|
11
|
+
*
|
|
12
|
+
* Storage APIs for a specific copy of sync rules are provided by the `SyncRulesBucketStorage` instances.
|
|
13
|
+
*/
|
|
14
|
+
export interface BucketStorageFactory extends ObserverClient<BucketStorageFactoryListener>, AsyncDisposable {
|
|
15
|
+
/**
|
|
16
|
+
* Update sync rules from configuration, if changed.
|
|
17
|
+
*/
|
|
18
|
+
configureSyncRules(options: UpdateSyncRulesOptions): Promise<{
|
|
19
|
+
updated: boolean;
|
|
20
|
+
persisted_sync_rules?: PersistedSyncRulesContent;
|
|
21
|
+
lock?: ReplicationLock;
|
|
22
|
+
}>;
|
|
23
|
+
/**
|
|
24
|
+
* Get a storage instance to query sync data for specific sync rules.
|
|
25
|
+
*/
|
|
26
|
+
getInstance(options: PersistedSyncRulesContent): SyncRulesBucketStorage;
|
|
27
|
+
/**
|
|
28
|
+
* Deploy new sync rules.
|
|
29
|
+
*/
|
|
30
|
+
updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
|
|
31
|
+
/**
|
|
32
|
+
* Indicate that a slot was removed, and we should re-sync by creating
|
|
33
|
+
* a new sync rules instance.
|
|
34
|
+
*
|
|
35
|
+
* This is roughly the same as deploying a new version of the current sync
|
|
36
|
+
* rules, but also accounts for cases where the current sync rules are not
|
|
37
|
+
* the latest ones.
|
|
38
|
+
*
|
|
39
|
+
* Replication should be restarted after this.
|
|
40
|
+
*
|
|
41
|
+
* @param slot_name The removed slot
|
|
42
|
+
*/
|
|
43
|
+
slotRemoved(slot_name: string): Promise<void>;
|
|
44
|
+
/**
|
|
45
|
+
* Get the sync rules used for querying.
|
|
46
|
+
*/
|
|
47
|
+
getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null>;
|
|
48
|
+
/**
|
|
49
|
+
* Get the sync rules used for querying.
|
|
50
|
+
*/
|
|
51
|
+
getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
|
|
52
|
+
/**
|
|
53
|
+
* Get the sync rules that will be active next once done with initial replicatino.
|
|
54
|
+
*/
|
|
55
|
+
getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null>;
|
|
56
|
+
/**
|
|
57
|
+
* Get the sync rules that will be active next once done with initial replicatino.
|
|
58
|
+
*/
|
|
59
|
+
getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
|
|
60
|
+
/**
|
|
61
|
+
* Get all sync rules currently replicating. Typically this is the "active" and "next" sync rules.
|
|
62
|
+
*/
|
|
63
|
+
getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
|
|
64
|
+
/**
|
|
65
|
+
* Get all sync rules stopped but not terminated yet.
|
|
66
|
+
*/
|
|
67
|
+
getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
|
|
68
|
+
/**
|
|
69
|
+
* Get the active storage instance.
|
|
70
|
+
*/
|
|
71
|
+
getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
|
|
72
|
+
/**
|
|
73
|
+
* Get storage size of active sync rules.
|
|
74
|
+
*/
|
|
75
|
+
getStorageMetrics(): Promise<StorageMetrics>;
|
|
76
|
+
/**
|
|
77
|
+
* Get the unique identifier for this instance of Powersync
|
|
78
|
+
*/
|
|
79
|
+
getPowerSyncInstanceId(): Promise<string>;
|
|
80
|
+
/**
|
|
81
|
+
* Get a unique identifier for the system used for storage.
|
|
82
|
+
*/
|
|
83
|
+
getSystemIdentifier(): Promise<BucketStorageSystemIdentifier>;
|
|
84
|
+
}
|
|
85
|
+
export interface BucketStorageFactoryListener {
|
|
86
|
+
syncStorageCreated: (storage: SyncRulesBucketStorage) => void;
|
|
87
|
+
replicationEvent: (event: ReplicationEventPayload) => void;
|
|
88
|
+
}
|
|
89
|
+
export interface StorageMetrics {
|
|
90
|
+
/**
|
|
91
|
+
* Size of operations (bucket_data)
|
|
92
|
+
*/
|
|
93
|
+
operations_size_bytes: number;
|
|
94
|
+
/**
|
|
95
|
+
* Size of parameter storage.
|
|
96
|
+
*
|
|
97
|
+
* Replication storage -> raw data as received from Postgres.
|
|
98
|
+
*/
|
|
99
|
+
parameters_size_bytes: number;
|
|
100
|
+
/**
|
|
101
|
+
* Size of current_data.
|
|
102
|
+
*/
|
|
103
|
+
replication_size_bytes: number;
|
|
104
|
+
}
|
|
105
|
+
export interface UpdateSyncRulesOptions {
|
|
106
|
+
content: string;
|
|
107
|
+
lock?: boolean;
|
|
108
|
+
validate?: boolean;
|
|
109
|
+
}
|
|
110
|
+
export interface BucketStorageSystemIdentifier {
|
|
111
|
+
/**
|
|
112
|
+
* A unique identifier for the system used for storage.
|
|
113
|
+
* For Postgres this can be the cluster `system_identifier` and database name.
|
|
114
|
+
* For MongoDB this can be the replica set name.
|
|
115
|
+
*/
|
|
116
|
+
id: string;
|
|
117
|
+
/**
|
|
118
|
+
* A unique type for the storage implementation.
|
|
119
|
+
* e.g. `mongodb`, `postgresql`.
|
|
120
|
+
*/
|
|
121
|
+
type: string;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Helper for tests.
|
|
125
|
+
* This is not in the `service-core-tests` package in order for storage modules
|
|
126
|
+
* to provide relevant factories without requiring `service-core-tests` as a direct dependency.
|
|
127
|
+
*/
|
|
128
|
+
export interface TestStorageOptions {
|
|
129
|
+
/**
|
|
130
|
+
* By default, collections are only cleared/
|
|
131
|
+
* Setting this to true will drop the collections completely.
|
|
132
|
+
*/
|
|
133
|
+
dropAll?: boolean;
|
|
134
|
+
doNotClear?: boolean;
|
|
135
|
+
}
|
|
136
|
+
export type TestStorageFactory = (options?: TestStorageOptions) => Promise<BucketStorageFactory>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BucketStorageFactory.js","sourceRoot":"","sources":["../../src/storage/BucketStorageFactory.ts"],"names":[],"mappings":""}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ChecksumCache.js","sourceRoot":"","sources":["../../src/storage/ChecksumCache.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"ChecksumCache.js","sourceRoot":"","sources":["../../src/storage/ChecksumCache.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAmDlD,+DAA+D;AAC/D,MAAM,gBAAgB,GAAG,OAAO,CAAC;AAEjC,MAAM,MAAM,GAAG,SAAS,CAAC;AAEzB;;;;;;;;GAQG;AACH,MAAM,OAAO,aAAa;IACxB;;OAEG;IACK,KAAK,CAAyD;IACtE;;OAEG;IACK,iBAAiB,GAAG,IAAI,GAAG,EAA8B,CAAC;IAE1D,cAAc,CAAiB;IAEvC,YAAY,OAA6B;QACvC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;QAE7C,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAA+C;YACtE,GAAG,EAAE,OAAO,CAAC,OAAO,IAAI,gBAAgB;YACxC,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE;gBACpD,oDAAoD;gBACpD,sEAAsE;gBACtE,MAAM,EAAE,MAAM,EAAE,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;gBAC3C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAEnD,uDAAuD;gBACvD,IAAI,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACvD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;oBAC1B,aAAa,GAAG,IAAI,UAAU,EAAE,CAAC;oBACjC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;gBACpD,CAAC;gBACD,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBACjD,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBACtB,2DAA2D;gBAC3D,MAAM,EAAE,gBAAgB,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;gBAChD,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;gBAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC/D,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;oBAC1B,OAAO;gBACT,CAAC;gBACD,aAAa,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;gBAC5C,IAAI,aAAa,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC9B,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;YAED,cAAc,EAAE,IAAI;YAEpB,8EAA8E;YAC9E,oCAAoC;YACpC,gBAAgB,EAAE,IAAI;YAEtB,0DAA0D;YAC1D,0DAA0D;YAC1D,oCAAoC;YACpC,GAAG,EAAE,MAAM;YACX,aAAa,EAAE,KAAK;YACpB,UAAU,EAAE,KAAK;SAClB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,UAAgB,EAAE,OAAiB;QACpD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACjE,kDAAkD;QAClD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,CAAC;IACzD,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,CAAC,UAAgB,EAAE,OAAiB;QACtD,oEAAoE;QACpE,IAAI,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAEhC,wBAAwB;QACxB,IAAI,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;QAErD,4CAA4C;QAC5C,IAAI,YAAyB,CAAC;QAC9B,IAAI,WAAgC,CAAC;QACrC,IAAI,YAAY,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACvD,YAAY,GAAG,OAAO,CAAC;YACvB,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,0EAA0E;QAC1E,IAAI,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;QAErD,MAAM,OAAO,GAAyB;YACpC,KAAK,CAAC,KAAK,CAAC,MAAM;gBAChB,MAAM,YAAY,CAAC;gBACnB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;oBACzB,sBAAsB;oBACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,EAAE,CAAC,CAAC;gBACjD,CAAC;gBACD,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC1C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,sBAAsB;oBACtB,MAAM,IAAI,KAAK,CAAC,uCAAuC,MAAM,EAAE,CAAC,CAAC;gBACnE,CAAC;gBACD,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC;SAC/B,CAAC;QAEF,+DAA+D;QAC/D,IAAI,cAAc,GAAiD,IAAI,CAAC;QAExE,IAAI,CAAC;YACH,kCAAkC;YAClC,IAAI,kBAAkB,GAAoB,EAAE,CAAC;YAE7C,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC3B,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAClD,IAAI,MAAM,GAAoC,EAAE,CAAC;gBACjD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;oBAC5F,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;wBACtB,sBAAsB;wBACtB,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;oBAC7D,CAAC;oBACD,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBACtC,CAAC,CAAC,CAAC;gBACH,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3B,IAAI,MAAM,CAAC,KAAK,IAAI,KAAK,IAAI,MAAM,CAAC,KAAK,IAAI,UAAU,EAAE,CAAC;oBACxD,2EAA2E;oBAC3E,uBAAuB;oBACvB,yDAAyD;gBAC3D,CAAC;qBAAM,CAAC;oBACN,2CAA2C;oBAC3C,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,6DAA6D;YAC7D,wDAAwD;YACxD,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;YAExD,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;gBACtB,wCAAwC;gBACxC,YAAY,EAAE,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,IAAI,cAAc,GAAiC,EAAE,CAAC;gBACtD,mEAAmE;gBACnE,IAAI,GAAG,GAAG,IAAI,GAAG,EAA0B,CAAC;gBAE5C,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC3B,IAAI,aAAa,GAAsC,IAAI,CAAC;oBAC5D,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACzD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;wBAC1B,uDAAuD;wBACvD,IAAI,IAAI,GAAG,aAAa,CAAC,iBAAiB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;wBAC/D,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,CAAC;wBACpC,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;4BAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;4BACxB,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;4BAC1C,mCAAmC;4BACnC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;4BACzC,iEAAiE;4BACjE,uEAAuE;4BACvE,8DAA8D;4BAC9D,8CAA8C;4BAC9C,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gCACnB,2DAA2D;gCAC3D,aAAa,GAAG;oCACd,MAAM;oCACN,KAAK,EAAE,EAAE,CAAC,QAAQ,EAAE;oCACpB,GAAG,EAAE,UAAU;iCAChB,CAAC;gCACF,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gCACxB,MAAM;4BACR,CAAC;4BAED,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gCACvB,yBAAyB;gCACzB,MAAM;4BACR,CAAC;4BACD,oBAAoB;4BACpB,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACpB,CAAC;oBACH,CAAC;oBAED,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;wBAC1B,+DAA+D;wBAC/D,aAAa,GAAG;4BACd,MAAM;4BACN,GAAG,EAAE,UAAU;yBAChB,CAAC;wBACF,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE;4BACd,MAAM;4BACN,QAAQ,EAAE,CAAC;4BACX,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;oBACL,CAAC;oBACD,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBACrC,CAAC;gBAED,wCAAwC;gBACxC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;gBAE1D,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACnC,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAC9B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;wBAClB,sBAAsB;wBACtB,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,EAAE,CAAC,CAAC;oBAC9C,CAAC;oBACD,mDAAmD;oBACnD,6DAA6D;oBAC7D,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC,CAAC;oBACxD,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;gBAClC,CAAC;gBAED,6DAA6D;gBAC7D,YAAY,EAAE,CAAC;YACjB,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,wDAAwD;YACxD,iFAAiF;YACjF,uBAAuB;YACvB,WAAW,CAAC,CAAC,CAAC,CAAC;YAEf,yFAAyF;YACzF,6BAA6B;YAC7B,MAAM,cAAc,CAAC;YAErB,MAAM,CAAC,CAAC;QACV,CAAC;QAED,gDAAgD;QAChD,MAAM,cAAc,GAAG,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QACpD,8BAA8B;QAC9B,KAAK,IAAI,MAAM,IAAI,cAAc,EAAE,CAAC;YAClC,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBAChC,MAAM,MAAM,CAAC,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACxC,oBAAoB;YACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,YAAY,CAAC,IAAI,OAAO,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACxF,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;CACF;AAED,SAAS,YAAY,CAAC,UAA2B,EAAE,MAAc;IAC/D,OAAO,GAAG,UAAU,IAAI,MAAM,EAAE,CAAC;AACnC,CAAC;AAED,SAAS,aAAa,CAAC,GAAW;IAChC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/B,OAAO,EAAE,gBAAgB,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;AACzF,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { SqlSyncRules } from '@powersync/service-sync-rules';
|
|
2
|
+
import { ReplicationLock } from './ReplicationLock.js';
|
|
3
|
+
export interface ParseSyncRulesOptions {
|
|
4
|
+
defaultSchema: string;
|
|
5
|
+
}
|
|
6
|
+
export interface PersistedSyncRulesContent {
|
|
7
|
+
readonly id: number;
|
|
8
|
+
readonly sync_rules_content: string;
|
|
9
|
+
readonly slot_name: string;
|
|
10
|
+
readonly last_fatal_error?: string | null;
|
|
11
|
+
readonly last_keepalive_ts?: Date | null;
|
|
12
|
+
readonly last_checkpoint_ts?: Date | null;
|
|
13
|
+
parsed(options: ParseSyncRulesOptions): PersistedSyncRules;
|
|
14
|
+
lock(): Promise<ReplicationLock>;
|
|
15
|
+
}
|
|
16
|
+
export interface PersistedSyncRules {
|
|
17
|
+
readonly id: number;
|
|
18
|
+
readonly sync_rules: SqlSyncRules;
|
|
19
|
+
readonly slot_name: string;
|
|
20
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"PersistedSyncRulesContent.js","sourceRoot":"","sources":["../../src/storage/PersistedSyncRulesContent.ts"],"names":[],"mappings":""}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
2
|
-
import { BucketStorageBatch, SaveOp } from './BucketStorage.js';
|
|
3
2
|
import { SourceTable } from './SourceTable.js';
|
|
3
|
+
import { BucketStorageBatch, SaveOp } from './BucketStorageBatch.js';
|
|
4
4
|
export type EventData = {
|
|
5
5
|
op: SaveOp;
|
|
6
6
|
before?: sync_rules.SqliteRow;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ReplicationLock.js","sourceRoot":"","sources":["../../src/storage/ReplicationLock.ts"],"names":[],"mappings":""}
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { BaseObserver } from '@powersync/lib-services-framework';
|
|
2
2
|
import { ResolvedPowerSyncConfig } from '../util/util-index.js';
|
|
3
|
-
import { BucketStorageFactory } from './BucketStorage.js';
|
|
4
3
|
import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
|
|
4
|
+
import { BucketStorageFactory } from './BucketStorageFactory.js';
|
|
5
5
|
export type StorageEngineOptions = {
|
|
6
6
|
configuration: ResolvedPowerSyncConfig;
|
|
7
7
|
};
|
|
8
|
-
export interface StorageEngineListener
|
|
8
|
+
export interface StorageEngineListener {
|
|
9
9
|
storageActivated: (storage: BucketStorageFactory) => void;
|
|
10
10
|
}
|
|
11
|
-
export declare class StorageEngine extends
|
|
11
|
+
export declare class StorageEngine extends BaseObserver<StorageEngineListener> {
|
|
12
12
|
private options;
|
|
13
13
|
private storageProviders;
|
|
14
14
|
private currentActiveStorage;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export class StorageEngine extends
|
|
1
|
+
import { BaseObserver, logger } from '@powersync/lib-services-framework';
|
|
2
|
+
export class StorageEngine extends BaseObserver {
|
|
3
3
|
options;
|
|
4
4
|
// TODO: This will need to revisited when we actually support multiple storage providers.
|
|
5
5
|
storageProviders = new Map();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"StorageEngine.js","sourceRoot":"","sources":["../../src/storage/StorageEngine.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"StorageEngine.js","sourceRoot":"","sources":["../../src/storage/StorageEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAazE,MAAM,OAAO,aAAc,SAAQ,YAAmC;IAKhD;IAJpB,yFAAyF;IACjF,gBAAgB,GAAuC,IAAI,GAAG,EAAE,CAAC;IACjE,oBAAoB,GAAyB,IAAI,CAAC;IAE1D,YAAoB,OAA6B;QAC/C,KAAK,EAAE,CAAC;QADU,YAAO,GAAP,OAAO,CAAsB;IAEjD,CAAC;IAED,IAAI,mBAAmB;QACrB,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC;IACpC,CAAC;IAED,IAAI,aAAa;QACf,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,gBAAgB,CAAC,QAA+B;QAC9C,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACrD,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC1C,MAAM,EAAE,aAAa,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACvC,IAAI,CAAC,oBAAoB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC,UAAU,CAAC;YAClG,cAAc,EAAE,aAAa;SAC9B,CAAC,CAAC;QACH,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC;QAC/E,MAAM,CAAC,IAAI,CAAC,mCAAmC,aAAa,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;QAC9E,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,QAAQ;QACnB,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QAC/C,MAAM,IAAI,CAAC,oBAAoB,EAAE,QAAQ,EAAE,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;IACxD,CAAC;CACF"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as util from '../util/util-index.js';
|
|
2
|
-
import { BucketStorageFactory } from './
|
|
2
|
+
import { BucketStorageFactory } from './BucketStorageFactory.js';
|
|
3
3
|
export interface ActiveStorage {
|
|
4
4
|
storage: BucketStorageFactory;
|
|
5
5
|
shutDown(): Promise<void>;
|
|
@@ -11,6 +11,9 @@ export interface ActiveStorage {
|
|
|
11
11
|
export interface GetStorageOptions {
|
|
12
12
|
resolvedConfig: util.ResolvedPowerSyncConfig;
|
|
13
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* Represents a provider that can create a storage instance for a specific storage type from configuration.
|
|
16
|
+
*/
|
|
14
17
|
export interface BucketStorageProvider {
|
|
15
18
|
/**
|
|
16
19
|
* The storage type that this provider provides.
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import { ObserverClient } from '@powersync/lib-services-framework';
|
|
2
|
+
import { SqlSyncRules, SqliteJsonRow, SqliteJsonValue } from '@powersync/service-sync-rules';
|
|
3
|
+
import * as util from '../util/util-index.js';
|
|
4
|
+
import { BucketStorageBatch, FlushedResult } from './BucketStorageBatch.js';
|
|
5
|
+
import { BucketStorageFactory } from './BucketStorageFactory.js';
|
|
6
|
+
import { ParseSyncRulesOptions } from './PersistedSyncRulesContent.js';
|
|
7
|
+
import { SourceEntityDescriptor } from './SourceEntity.js';
|
|
8
|
+
import { SourceTable } from './SourceTable.js';
|
|
9
|
+
import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
|
|
10
|
+
/**
|
|
11
|
+
* Storage for a specific copy of sync rules.
|
|
12
|
+
*/
|
|
13
|
+
export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketStorageListener>, SyncStorageWriteCheckpointAPI {
|
|
14
|
+
readonly group_id: number;
|
|
15
|
+
readonly slot_name: string;
|
|
16
|
+
readonly factory: BucketStorageFactory;
|
|
17
|
+
/**
|
|
18
|
+
* Resolve a table, keeping track of it internally.
|
|
19
|
+
*/
|
|
20
|
+
resolveTable(options: ResolveTableOptions): Promise<ResolveTableResult>;
|
|
21
|
+
/**
|
|
22
|
+
* Use this to get access to update storage data.
|
|
23
|
+
*/
|
|
24
|
+
startBatch(options: StartBatchOptions, callback: (batch: BucketStorageBatch) => Promise<void>): Promise<FlushedResult | null>;
|
|
25
|
+
getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules;
|
|
26
|
+
/**
|
|
27
|
+
* Terminate the sync rules.
|
|
28
|
+
*
|
|
29
|
+
* This clears the storage, and sets state to TERMINATED.
|
|
30
|
+
*
|
|
31
|
+
* Must only be called on stopped sync rules.
|
|
32
|
+
*/
|
|
33
|
+
terminate(options?: TerminateOptions): Promise<void>;
|
|
34
|
+
getStatus(): Promise<SyncRuleStatus>;
|
|
35
|
+
/**
|
|
36
|
+
* Clear the storage, without changing state.
|
|
37
|
+
*/
|
|
38
|
+
clear(): Promise<void>;
|
|
39
|
+
autoActivate(): Promise<void>;
|
|
40
|
+
/**
|
|
41
|
+
* Record a replication error.
|
|
42
|
+
*
|
|
43
|
+
* This could be a recoverable error (e.g. temporary network failure),
|
|
44
|
+
* or a permanent error (e.g. missing toast data).
|
|
45
|
+
*
|
|
46
|
+
* Errors are cleared on commit.
|
|
47
|
+
*/
|
|
48
|
+
reportError(e: any): Promise<void>;
|
|
49
|
+
compact(options?: CompactOptions): Promise<void>;
|
|
50
|
+
getCheckpoint(): Promise<ReplicationCheckpoint>;
|
|
51
|
+
/**
|
|
52
|
+
* Used to resolve "dynamic" parameter queries.
|
|
53
|
+
*/
|
|
54
|
+
getParameterSets(checkpoint: util.OpId, lookups: SqliteJsonValue[][]): Promise<SqliteJsonRow[]>;
|
|
55
|
+
getCheckpointChanges(options: GetCheckpointChangesOptions): Promise<CheckpointChanges>;
|
|
56
|
+
/**
|
|
57
|
+
* Yields the latest user write checkpoint whenever the sync checkpoint updates.
|
|
58
|
+
*
|
|
59
|
+
* The stream stops or errors if this is not the active sync rules (anymore).
|
|
60
|
+
*/
|
|
61
|
+
watchWriteCheckpoint(options: WatchWriteCheckpointOptions): AsyncIterable<StorageCheckpointUpdate>;
|
|
62
|
+
/**
|
|
63
|
+
* Get a "batch" of data for a checkpoint.
|
|
64
|
+
*
|
|
65
|
+
* The results will be split into separate SyncBucketData chunks to:
|
|
66
|
+
* 1. Separate buckets.
|
|
67
|
+
* 2. Limit the size of each individual chunk according to options.batchSizeLimitBytes.
|
|
68
|
+
*
|
|
69
|
+
* @param checkpoint the checkpoint
|
|
70
|
+
* @param dataBuckets current bucket states
|
|
71
|
+
* @param options batch size options
|
|
72
|
+
*/
|
|
73
|
+
getBucketDataBatch(checkpoint: util.OpId, dataBuckets: Map<string, string>, options?: BucketDataBatchOptions): AsyncIterable<SyncBucketDataBatch>;
|
|
74
|
+
/**
|
|
75
|
+
* Compute checksums for a given list of buckets.
|
|
76
|
+
*
|
|
77
|
+
* Returns zero checksums for any buckets not found.
|
|
78
|
+
*/
|
|
79
|
+
getChecksums(checkpoint: util.OpId, buckets: string[]): Promise<util.ChecksumMap>;
|
|
80
|
+
}
|
|
81
|
+
export interface SyncRulesBucketStorageListener {
|
|
82
|
+
batchStarted: (batch: BucketStorageBatch) => void;
|
|
83
|
+
}
|
|
84
|
+
export interface SyncRuleStatus {
|
|
85
|
+
checkpoint_lsn: string | null;
|
|
86
|
+
active: boolean;
|
|
87
|
+
snapshot_done: boolean;
|
|
88
|
+
}
|
|
89
|
+
export interface ResolveTableOptions {
|
|
90
|
+
group_id: number;
|
|
91
|
+
connection_id: number;
|
|
92
|
+
connection_tag: string;
|
|
93
|
+
entity_descriptor: SourceEntityDescriptor;
|
|
94
|
+
sync_rules: SqlSyncRules;
|
|
95
|
+
}
|
|
96
|
+
export interface ResolveTableResult {
|
|
97
|
+
table: SourceTable;
|
|
98
|
+
dropTables: SourceTable[];
|
|
99
|
+
}
|
|
100
|
+
export interface StartBatchOptions extends ParseSyncRulesOptions {
|
|
101
|
+
zeroLSN: string;
|
|
102
|
+
/**
|
|
103
|
+
* Whether or not to store a copy of the current data.
|
|
104
|
+
*
|
|
105
|
+
* This is needed if we need to apply partial updates, for example
|
|
106
|
+
* when we get TOAST values from Postgres.
|
|
107
|
+
*
|
|
108
|
+
* This is not needed when we get the full document from the source
|
|
109
|
+
* database, for example from MongoDB.
|
|
110
|
+
*/
|
|
111
|
+
storeCurrentData: boolean;
|
|
112
|
+
/**
|
|
113
|
+
* Set to true for initial replication.
|
|
114
|
+
*
|
|
115
|
+
* This will avoid creating new operations for rows previously replicated.
|
|
116
|
+
*/
|
|
117
|
+
skipExistingRows?: boolean;
|
|
118
|
+
}
|
|
119
|
+
export interface CompactOptions {
|
|
120
|
+
/**
|
|
121
|
+
* Heap memory limit for the compact process.
|
|
122
|
+
*
|
|
123
|
+
* Add around 64MB to this to determine the "--max-old-space-size" argument.
|
|
124
|
+
* Add another 80MB to get RSS usage / memory limits.
|
|
125
|
+
*/
|
|
126
|
+
memoryLimitMB?: number;
|
|
127
|
+
/**
|
|
128
|
+
* If specified, ignore any operations newer than this when compacting.
|
|
129
|
+
*
|
|
130
|
+
* This is primarily for tests, where we want to test compacting at a specific
|
|
131
|
+
* point.
|
|
132
|
+
*
|
|
133
|
+
* This can also be used to create a "safe buffer" of recent operations that should
|
|
134
|
+
* not be compacted, to avoid invalidating checkpoints in use.
|
|
135
|
+
*/
|
|
136
|
+
maxOpId?: bigint;
|
|
137
|
+
/**
|
|
138
|
+
* If specified, compact only the specific buckets.
|
|
139
|
+
*
|
|
140
|
+
* If not specified, compacts all buckets.
|
|
141
|
+
*
|
|
142
|
+
* These can be individual bucket names, or bucket definition names.
|
|
143
|
+
*/
|
|
144
|
+
compactBuckets?: string[];
|
|
145
|
+
}
|
|
146
|
+
export interface TerminateOptions {
|
|
147
|
+
/**
|
|
148
|
+
* If true, also clear the storage before terminating.
|
|
149
|
+
*/
|
|
150
|
+
clearStorage: boolean;
|
|
151
|
+
}
|
|
152
|
+
export interface BucketDataBatchOptions {
|
|
153
|
+
/** Limit number of documents returned. Defaults to 1000. */
|
|
154
|
+
limit?: number;
|
|
155
|
+
/**
|
|
156
|
+
* Limit size of chunks returned. Defaults to 1MB.
|
|
157
|
+
*
|
|
158
|
+
* This is a lower bound, not an upper bound. As soon as the chunk size goes over this limit,
|
|
159
|
+
* it is returned.
|
|
160
|
+
*
|
|
161
|
+
* Note that an individual data row can be close to 16MB in size, so this does not help in
|
|
162
|
+
* extreme cases.
|
|
163
|
+
*/
|
|
164
|
+
chunkLimitBytes?: number;
|
|
165
|
+
}
|
|
166
|
+
export interface SyncBucketDataBatch {
|
|
167
|
+
batch: util.SyncBucketData;
|
|
168
|
+
targetOp: bigint | null;
|
|
169
|
+
}
|
|
170
|
+
export interface ReplicationCheckpoint {
|
|
171
|
+
readonly checkpoint: util.OpId;
|
|
172
|
+
readonly lsn: string | null;
|
|
173
|
+
}
|
|
174
|
+
export interface WatchWriteCheckpointOptions {
|
|
175
|
+
/** user_id and client_id combined. */
|
|
176
|
+
user_id: string;
|
|
177
|
+
signal: AbortSignal;
|
|
178
|
+
}
|
|
179
|
+
export interface WatchFilterEvent {
|
|
180
|
+
changedDataBucket?: string;
|
|
181
|
+
changedParameterBucketDefinition?: string;
|
|
182
|
+
invalidate?: boolean;
|
|
183
|
+
}
|
|
184
|
+
export interface WriteCheckpoint {
|
|
185
|
+
base: ReplicationCheckpoint;
|
|
186
|
+
writeCheckpoint: bigint | null;
|
|
187
|
+
}
|
|
188
|
+
export interface StorageCheckpointUpdate extends WriteCheckpoint {
|
|
189
|
+
update: CheckpointChanges;
|
|
190
|
+
}
|
|
191
|
+
export interface GetCheckpointChangesOptions {
|
|
192
|
+
lastCheckpoint: util.OpId;
|
|
193
|
+
nextCheckpoint: util.OpId;
|
|
194
|
+
}
|
|
195
|
+
export interface CheckpointChanges {
|
|
196
|
+
updatedDataBuckets: string[];
|
|
197
|
+
invalidateDataBuckets: boolean;
|
|
198
|
+
updatedParameterBucketDefinitions: string[];
|
|
199
|
+
invalidateParameterBuckets: boolean;
|
|
200
|
+
}
|
|
201
|
+
export declare const CHECKPOINT_INVALIDATE_ALL: CheckpointChanges;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SyncRulesBucketStorage.js","sourceRoot":"","sources":["../../src/storage/SyncRulesBucketStorage.ts"],"names":[],"mappings":"AA0PA,MAAM,CAAC,MAAM,yBAAyB,GAAsB;IAC1D,kBAAkB,EAAE,EAAE;IACtB,qBAAqB,EAAE,IAAI;IAC3B,iCAAiC,EAAE,EAAE;IACrC,0BAA0B,EAAE,IAAI;CACjC,CAAC"}
|
package/dist/storage/bson.d.ts
CHANGED
|
@@ -1,14 +1,22 @@
|
|
|
1
1
|
import * as bson from 'bson';
|
|
2
2
|
import { SqliteJsonValue } from '@powersync/service-sync-rules';
|
|
3
|
-
import { ReplicaId } from './
|
|
3
|
+
import { ReplicaId } from './BucketStorageBatch.js';
|
|
4
4
|
type NodeBuffer = Buffer<ArrayBuffer>;
|
|
5
|
-
|
|
5
|
+
/**
|
|
6
|
+
* Use for internal (bucket storage) data, where we control each field.
|
|
7
|
+
*/
|
|
8
|
+
export declare const BSON_DESERIALIZE_INTERNAL_OPTIONS: bson.DeserializeOptions;
|
|
9
|
+
/**
|
|
10
|
+
* Use for data from external sources.
|
|
11
|
+
*/
|
|
12
|
+
export declare const BSON_DESERIALIZE_DATA_OPTIONS: bson.DeserializeOptions;
|
|
6
13
|
/**
|
|
7
14
|
* Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
|
|
8
15
|
* @param lookup
|
|
9
16
|
*/
|
|
10
17
|
export declare const serializeLookupBuffer: (lookup: SqliteJsonValue[]) => NodeBuffer;
|
|
11
18
|
export declare const serializeLookup: (lookup: SqliteJsonValue[]) => bson.Binary;
|
|
19
|
+
export declare const getLookupBucketDefinitionName: (lookup: bson.Binary) => string;
|
|
12
20
|
/**
|
|
13
21
|
* True if this is a bson.UUID.
|
|
14
22
|
*
|
|
@@ -17,7 +25,7 @@ export declare const serializeLookup: (lookup: SqliteJsonValue[]) => bson.Binary
|
|
|
17
25
|
export declare const isUUID: (value: any) => value is bson.UUID;
|
|
18
26
|
export declare const serializeReplicaId: (id: ReplicaId) => NodeBuffer;
|
|
19
27
|
export declare const deserializeReplicaId: (id: Buffer) => ReplicaId;
|
|
20
|
-
export declare const deserializeBson: (buffer:
|
|
28
|
+
export declare const deserializeBson: (buffer: Uint8Array) => bson.Document;
|
|
21
29
|
export declare const serializeBson: (document: any) => NodeBuffer;
|
|
22
30
|
/**
|
|
23
31
|
* Returns true if two ReplicaId values are the same (serializes to the same BSON value).
|
package/dist/storage/bson.js
CHANGED
|
@@ -1,8 +1,18 @@
|
|
|
1
1
|
import * as bson from 'bson';
|
|
2
|
-
|
|
2
|
+
/**
|
|
3
|
+
* Use for internal (bucket storage) data, where we control each field.
|
|
4
|
+
*/
|
|
5
|
+
export const BSON_DESERIALIZE_INTERNAL_OPTIONS = {
|
|
3
6
|
// use bigint instead of Long
|
|
4
7
|
useBigInt64: true
|
|
5
8
|
};
|
|
9
|
+
/**
|
|
10
|
+
* Use for data from external sources.
|
|
11
|
+
*/
|
|
12
|
+
export const BSON_DESERIALIZE_DATA_OPTIONS = {
|
|
13
|
+
// Temporarily disable due to https://jira.mongodb.org/browse/NODE-6764
|
|
14
|
+
useBigInt64: false
|
|
15
|
+
};
|
|
6
16
|
/**
|
|
7
17
|
* Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
|
|
8
18
|
* @param lookup
|
|
@@ -21,6 +31,10 @@ export const serializeLookupBuffer = (lookup) => {
|
|
|
21
31
|
export const serializeLookup = (lookup) => {
|
|
22
32
|
return new bson.Binary(serializeLookupBuffer(lookup));
|
|
23
33
|
};
|
|
34
|
+
export const getLookupBucketDefinitionName = (lookup) => {
|
|
35
|
+
const parsed = bson.deserialize(lookup.buffer, BSON_DESERIALIZE_INTERNAL_OPTIONS).l;
|
|
36
|
+
return parsed[0];
|
|
37
|
+
};
|
|
24
38
|
/**
|
|
25
39
|
* True if this is a bson.UUID.
|
|
26
40
|
*
|
|
@@ -41,7 +55,15 @@ export const deserializeReplicaId = (id) => {
|
|
|
41
55
|
return deserialized.id;
|
|
42
56
|
};
|
|
43
57
|
export const deserializeBson = (buffer) => {
|
|
44
|
-
|
|
58
|
+
const doc = bson.deserialize(buffer, BSON_DESERIALIZE_DATA_OPTIONS);
|
|
59
|
+
// Temporary workaround due to https://jira.mongodb.org/browse/NODE-6764
|
|
60
|
+
for (let key in doc) {
|
|
61
|
+
const value = doc[key];
|
|
62
|
+
if (value instanceof bson.Long) {
|
|
63
|
+
doc[key] = value.toBigInt();
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return doc;
|
|
45
67
|
};
|
|
46
68
|
export const serializeBson = (document) => {
|
|
47
69
|
return bson.serialize(document);
|
package/dist/storage/bson.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bson.js","sourceRoot":"","sources":["../../src/storage/bson.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAO7B,MAAM,CAAC,MAAM,
|
|
1
|
+
{"version":3,"file":"bson.js","sourceRoot":"","sources":["../../src/storage/bson.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAO7B;;GAEG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAA4B;IACxE,6BAA6B;IAC7B,WAAW,EAAE,IAAI;CAClB,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAA4B;IACpE,uEAAuE;IACvE,WAAW,EAAE,KAAK;CACnB,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,MAAyB,EAAc,EAAE;IAC7E,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACtC,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,CAAe,CAAC;AACzD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,MAAyB,EAAE,EAAE;IAC3D,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC,CAAC;AACxD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAAC,MAAmB,EAAE,EAAE;IACnE,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,iCAAiC,CAAC,CAAC,CAAsB,CAAC;IACzG,OAAO,MAAM,CAAC,CAAC,CAAW,CAAC;AAC7B,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,KAAU,EAAsB,EAAE;IACvD,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC9C,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,IAAI,GAAG,KAAkB,CAAC;IAChC,OAAO,IAAI,CAAC,SAAS,IAAI,QAAQ,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;AACjF,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,EAAa,EAAc,EAAE;IAC9D,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAe,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,EAAU,EAAa,EAAE;IAC5D,MAAM,YAAY,GAAG,eAAe,CAAC,EAAE,CAAC,CAAC;IACzC,OAAO,YAAY,CAAC,EAAE,CAAC;AACzB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,MAAkB,EAAiB,EAAE;IACnE,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,6BAA6B,CAAC,CAAC;IACpE,wEAAwE;IACxE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,KAAK,YAAY,IAAI,CAAC,IAAI,EAAE,CAAC;YAC/B,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC9B,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,QAAa,EAAc,EAAE;IACzD,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAe,CAAC;AAChD,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAY,EAAE,CAAY,EAAE,EAAE;IAC5D,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,CAAC,IAAI,QAAQ,IAAI,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;SAAM,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACrB,CAAC;SAAM,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;QAChE,OAAO,KAAK,CAAC;IACf,CAAC;SAAM,CAAC;QACN,iEAAiE;QACjE,OAAO,kBAAkB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;AACH,CAAC,CAAC"}
|
|
@@ -7,3 +7,8 @@ export * from './SourceTable.js';
|
|
|
7
7
|
export * from './StorageEngine.js';
|
|
8
8
|
export * from './StorageProvider.js';
|
|
9
9
|
export * from './WriteCheckpointAPI.js';
|
|
10
|
+
export * from './BucketStorageFactory.js';
|
|
11
|
+
export * from './BucketStorageBatch.js';
|
|
12
|
+
export * from './SyncRulesBucketStorage.js';
|
|
13
|
+
export * from './PersistedSyncRulesContent.js';
|
|
14
|
+
export * from './ReplicationLock.js';
|
|
@@ -7,4 +7,9 @@ export * from './SourceTable.js';
|
|
|
7
7
|
export * from './StorageEngine.js';
|
|
8
8
|
export * from './StorageProvider.js';
|
|
9
9
|
export * from './WriteCheckpointAPI.js';
|
|
10
|
+
export * from './BucketStorageFactory.js';
|
|
11
|
+
export * from './BucketStorageBatch.js';
|
|
12
|
+
export * from './SyncRulesBucketStorage.js';
|
|
13
|
+
export * from './PersistedSyncRulesContent.js';
|
|
14
|
+
export * from './ReplicationLock.js';
|
|
10
15
|
//# sourceMappingURL=storage-index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,cAAc,WAAW,CAAC;AAC1B,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AACnC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,mBAAmB,CAAC;AAClC,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,yBAAyB,CAAC"}
|
|
1
|
+
{"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,cAAc,WAAW,CAAC;AAC1B,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AACnC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,mBAAmB,CAAC;AAClC,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,yBAAyB,CAAC;AACxC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,sBAAsB,CAAC"}
|