@powersync/service-core 1.20.0 → 1.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/dist/auth/utils.d.ts +2 -1
- package/dist/auth/utils.js +1 -1
- package/dist/auth/utils.js.map +1 -1
- package/dist/entry/commands/compact-action.js +26 -5
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/routes/endpoints/admin.js +1 -0
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +6 -1
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +29 -8
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +5 -0
- package/dist/storage/ChecksumCache.d.ts +5 -2
- package/dist/storage/ChecksumCache.js +8 -4
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +6 -2
- package/dist/storage/PersistedSyncRulesContent.js +2 -1
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/SourceTable.d.ts +7 -2
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageVersionConfig.d.ts +33 -0
- package/dist/storage/StorageVersionConfig.js +39 -6
- package/dist/storage/StorageVersionConfig.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +26 -6
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +3 -3
- package/dist/sync/BucketChecksumState.js +12 -42
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +1 -0
- package/dist/sync/util.js +10 -0
- package/dist/sync/util.js.map +1 -1
- package/package.json +4 -4
- package/src/auth/utils.ts +5 -2
- package/src/entry/commands/compact-action.ts +29 -5
- package/src/routes/endpoints/admin.ts +1 -0
- package/src/routes/endpoints/sync-stream.ts +6 -1
- package/src/storage/BucketStorageBatch.ts +33 -9
- package/src/storage/BucketStorageFactory.ts +6 -0
- package/src/storage/ChecksumCache.ts +14 -6
- package/src/storage/PersistedSyncRulesContent.ts +7 -2
- package/src/storage/SourceTable.ts +7 -1
- package/src/storage/StorageVersionConfig.ts +54 -6
- package/src/storage/SyncRulesBucketStorage.ts +33 -6
- package/src/sync/BucketChecksumState.ts +18 -49
- package/src/sync/sync.ts +9 -3
- package/src/sync/util.ts +10 -0
- package/test/src/auth.test.ts +20 -1
- package/test/src/checksum_cache.test.ts +102 -57
- package/test/src/config.test.ts +1 -0
- package/test/src/sync/BucketChecksumState.test.ts +53 -21
- package/test/src/utils.ts +9 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -12,6 +12,16 @@ export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptio
|
|
|
12
12
|
};
|
|
13
13
|
|
|
14
14
|
export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageListener>, AsyncDisposable {
|
|
15
|
+
/**
|
|
16
|
+
* Alias for [Symbol.asyncDispose]
|
|
17
|
+
*/
|
|
18
|
+
dispose(): Promise<void>;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Last written op, if any. This may not reflect a consistent checkpoint.
|
|
22
|
+
*/
|
|
23
|
+
last_flushed_op: InternalOpId | null;
|
|
24
|
+
|
|
15
25
|
/**
|
|
16
26
|
* Save an op, and potentially flush.
|
|
17
27
|
*
|
|
@@ -45,19 +55,15 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
|
|
|
45
55
|
* Flush and commit any saved ops. This creates a new checkpoint by default.
|
|
46
56
|
*
|
|
47
57
|
* Only call this after a transaction.
|
|
48
|
-
*
|
|
49
|
-
* Returns true if either (1) a new checkpoint was created, or (2) there are no changes to commit.
|
|
50
58
|
*/
|
|
51
|
-
commit(lsn: string, options?: BucketBatchCommitOptions): Promise<
|
|
59
|
+
commit(lsn: string, options?: BucketBatchCommitOptions): Promise<CheckpointResult>;
|
|
52
60
|
|
|
53
61
|
/**
|
|
54
62
|
* Advance the checkpoint LSN position, without any associated op.
|
|
55
63
|
*
|
|
56
64
|
* This must only be called when not inside a transaction.
|
|
57
|
-
*
|
|
58
|
-
* @returns true if the checkpoint was advanced, false if this was a no-op
|
|
59
65
|
*/
|
|
60
|
-
keepalive(lsn: string): Promise<
|
|
66
|
+
keepalive(lsn: string): Promise<CheckpointResult>;
|
|
61
67
|
|
|
62
68
|
/**
|
|
63
69
|
* Set the LSN that replication should resume from.
|
|
@@ -83,9 +89,9 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
|
|
|
83
89
|
*/
|
|
84
90
|
resumeFromLsn: string | null;
|
|
85
91
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
92
|
+
markTableSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn?: string): Promise<SourceTable[]>;
|
|
93
|
+
markTableSnapshotRequired(table: SourceTable): Promise<void>;
|
|
94
|
+
markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void>;
|
|
89
95
|
|
|
90
96
|
updateTableProgress(table: SourceTable, progress: Partial<TableSnapshotStatus>): Promise<SourceTable>;
|
|
91
97
|
|
|
@@ -166,6 +172,24 @@ export interface SaveDelete {
|
|
|
166
172
|
afterReplicaId?: undefined;
|
|
167
173
|
}
|
|
168
174
|
|
|
175
|
+
export interface CheckpointResult {
|
|
176
|
+
/**
|
|
177
|
+
* True if any of these are true:
|
|
178
|
+
* 1. A snapshot is in progress.
|
|
179
|
+
* 2. The last checkpoint is older than "no_checkpoint_before" (if provided).
|
|
180
|
+
* 3. Replication was restarted with a lower LSN, and has not caught up yet.
|
|
181
|
+
*/
|
|
182
|
+
checkpointBlocked: boolean;
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* True if a checkpoint was actually created by this operation. This can be false even if checkpointBlocked is false,
|
|
186
|
+
* if the checkpoint was empty.
|
|
187
|
+
*
|
|
188
|
+
* This is primarily used for testing.
|
|
189
|
+
*/
|
|
190
|
+
checkpointCreated: boolean;
|
|
191
|
+
}
|
|
192
|
+
|
|
169
193
|
export interface BucketBatchStorageListener {
|
|
170
194
|
replicationEvent: (payload: ReplicationEventPayload) => void;
|
|
171
195
|
}
|
|
@@ -257,3 +257,9 @@ export interface TestStorageOptions {
|
|
|
257
257
|
}
|
|
258
258
|
export type TestStorageFactory = (options?: TestStorageOptions) => Promise<BucketStorageFactory>;
|
|
259
259
|
export type TestReportStorageFactory = (options?: TestStorageOptions) => Promise<ReportStorage>;
|
|
260
|
+
|
|
261
|
+
export interface TestStorageConfig {
|
|
262
|
+
factory: TestStorageFactory;
|
|
263
|
+
tableIdStrings: boolean;
|
|
264
|
+
storageVersion?: number;
|
|
265
|
+
}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { OrderedSet } from '@js-sdsl/ordered-set';
|
|
2
2
|
import { LRUCache } from 'lru-cache/min';
|
|
3
|
+
import { BucketDataSource } from '@powersync/service-sync-rules';
|
|
3
4
|
import { BucketChecksum } from '../util/protocol-types.js';
|
|
4
5
|
import { addBucketChecksums, ChecksumMap, InternalOpId, PartialChecksum } from '../util/utils.js';
|
|
6
|
+
import { BucketChecksumRequest } from './SyncRulesBucketStorage.js';
|
|
5
7
|
|
|
6
8
|
interface ChecksumFetchContext {
|
|
7
9
|
fetch(bucket: string): Promise<BucketChecksum>;
|
|
@@ -10,6 +12,7 @@ interface ChecksumFetchContext {
|
|
|
10
12
|
|
|
11
13
|
export interface FetchPartialBucketChecksum {
|
|
12
14
|
bucket: string;
|
|
15
|
+
source: BucketDataSource;
|
|
13
16
|
start?: InternalOpId;
|
|
14
17
|
end: InternalOpId;
|
|
15
18
|
}
|
|
@@ -113,10 +116,10 @@ export class ChecksumCache {
|
|
|
113
116
|
this.bucketCheckpoints.clear();
|
|
114
117
|
}
|
|
115
118
|
|
|
116
|
-
async getChecksums(checkpoint: InternalOpId, buckets:
|
|
119
|
+
async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<BucketChecksum[]> {
|
|
117
120
|
const checksums = await this.getChecksumMap(checkpoint, buckets);
|
|
118
121
|
// Return results in the same order as the request
|
|
119
|
-
return buckets.map((bucket) => checksums.get(bucket)!);
|
|
122
|
+
return buckets.map((bucket) => checksums.get(bucket.bucket)!);
|
|
120
123
|
}
|
|
121
124
|
|
|
122
125
|
/**
|
|
@@ -126,7 +129,7 @@ export class ChecksumCache {
|
|
|
126
129
|
*
|
|
127
130
|
* @returns a Map with exactly one entry for each bucket requested
|
|
128
131
|
*/
|
|
129
|
-
async getChecksumMap(checkpoint: InternalOpId, buckets:
|
|
132
|
+
async getChecksumMap(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<ChecksumMap> {
|
|
130
133
|
// Buckets that don't have a cached checksum for this checkpoint yet
|
|
131
134
|
let toFetch = new Set<string>();
|
|
132
135
|
|
|
@@ -164,19 +167,21 @@ export class ChecksumCache {
|
|
|
164
167
|
// One promise to await to ensure all fetch requests completed.
|
|
165
168
|
let settledPromise: Promise<PromiseSettledResult<void>[]> | null = null;
|
|
166
169
|
|
|
170
|
+
const sourceMap = new Map<string, BucketDataSource>();
|
|
171
|
+
|
|
167
172
|
try {
|
|
168
173
|
// Individual cache fetch promises
|
|
169
174
|
let cacheFetchPromises: Promise<void>[] = [];
|
|
170
175
|
|
|
171
176
|
for (let bucket of buckets) {
|
|
172
|
-
const cacheKey = makeCacheKey(checkpoint, bucket);
|
|
177
|
+
const cacheKey = makeCacheKey(checkpoint, bucket.bucket);
|
|
173
178
|
let status: LRUCache.Status<BucketChecksum> = {};
|
|
174
179
|
const p = this.cache.fetch(cacheKey, { context: context, status: status }).then((checksums) => {
|
|
175
180
|
if (checksums == null) {
|
|
176
181
|
// Should never happen
|
|
177
182
|
throw new Error(`Failed to get checksums for ${cacheKey}`);
|
|
178
183
|
}
|
|
179
|
-
finalResults.set(bucket, checksums);
|
|
184
|
+
finalResults.set(bucket.bucket, checksums);
|
|
180
185
|
});
|
|
181
186
|
cacheFetchPromises.push(p);
|
|
182
187
|
if (status.fetch == 'hit' || status.fetch == 'inflight') {
|
|
@@ -185,7 +190,8 @@ export class ChecksumCache {
|
|
|
185
190
|
// In either case, we don't need to fetch a new checksum.
|
|
186
191
|
} else {
|
|
187
192
|
// We need a new request for this checksum.
|
|
188
|
-
toFetch.add(bucket);
|
|
193
|
+
toFetch.add(bucket.bucket);
|
|
194
|
+
sourceMap.set(bucket.bucket, bucket.source);
|
|
189
195
|
}
|
|
190
196
|
}
|
|
191
197
|
// We do this directly after creating the promises, otherwise
|
|
@@ -220,6 +226,7 @@ export class ChecksumCache {
|
|
|
220
226
|
// Partial checksum found - make a partial checksum request
|
|
221
227
|
bucketRequest = {
|
|
222
228
|
bucket,
|
|
229
|
+
source: sourceMap.get(bucket)!,
|
|
223
230
|
start: cp,
|
|
224
231
|
end: checkpoint
|
|
225
232
|
};
|
|
@@ -240,6 +247,7 @@ export class ChecksumCache {
|
|
|
240
247
|
// No partial checksum found - make a new full checksum request
|
|
241
248
|
bucketRequest = {
|
|
242
249
|
bucket,
|
|
250
|
+
source: sourceMap.get(bucket)!,
|
|
243
251
|
end: checkpoint
|
|
244
252
|
};
|
|
245
253
|
add.set(bucket, {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
1
2
|
import {
|
|
2
3
|
CompatibilityContext,
|
|
3
4
|
CompatibilityOption,
|
|
@@ -12,10 +13,9 @@ import {
|
|
|
12
13
|
SyncConfigWithErrors,
|
|
13
14
|
versionedHydrationState
|
|
14
15
|
} from '@powersync/service-sync-rules';
|
|
16
|
+
import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
|
|
15
17
|
import { ReplicationLock } from './ReplicationLock.js';
|
|
16
18
|
import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js';
|
|
17
|
-
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
18
|
-
import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
|
|
19
19
|
|
|
20
20
|
export interface ParseSyncRulesOptions {
|
|
21
21
|
defaultSchema: string;
|
|
@@ -120,6 +120,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
|
|
|
120
120
|
id: this.id,
|
|
121
121
|
slot_name: this.slot_name,
|
|
122
122
|
sync_rules: config,
|
|
123
|
+
hydrationState,
|
|
123
124
|
hydratedSyncRules: () => {
|
|
124
125
|
return config.config.hydrate({ hydrationState });
|
|
125
126
|
}
|
|
@@ -140,6 +141,10 @@ export interface PersistedSyncRules {
|
|
|
140
141
|
readonly id: number;
|
|
141
142
|
readonly sync_rules: SyncConfigWithErrors;
|
|
142
143
|
readonly slot_name: string;
|
|
144
|
+
/**
|
|
145
|
+
* For testing only.
|
|
146
|
+
*/
|
|
147
|
+
readonly hydrationState: HydrationState;
|
|
143
148
|
|
|
144
149
|
hydratedSyncRules(): HydratedSyncRules;
|
|
145
150
|
}
|
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
import { DEFAULT_TAG } from '@powersync/service-sync-rules';
|
|
2
2
|
import * as util from '../util/util-index.js';
|
|
3
3
|
import { ColumnDescriptor, SourceEntityDescriptor } from './SourceEntity.js';
|
|
4
|
+
import { bson } from '../index.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Format of the id depends on the bucket storage module. It should be consistent within the module.
|
|
8
|
+
*/
|
|
9
|
+
export type SourceTableId = string | bson.ObjectId;
|
|
4
10
|
|
|
5
11
|
export interface SourceTableOptions {
|
|
6
|
-
id:
|
|
12
|
+
id: SourceTableId;
|
|
7
13
|
connectionTag: string;
|
|
8
14
|
objectId: number | string | undefined;
|
|
9
15
|
schema: string;
|
|
@@ -1,30 +1,78 @@
|
|
|
1
1
|
export interface StorageVersionConfig {
|
|
2
|
+
version: number;
|
|
3
|
+
|
|
2
4
|
/**
|
|
3
5
|
* Whether versioned bucket names are automatically enabled.
|
|
4
6
|
*
|
|
5
7
|
* If this is false, bucket names may still be versioned depending on the sync config.
|
|
8
|
+
*
|
|
9
|
+
* Introduced in STORAGE_VERSION_2.
|
|
6
10
|
*/
|
|
7
11
|
versionedBuckets: boolean;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Whether to use soft deletes for current_data, improving replication concurrency support.
|
|
15
|
+
*
|
|
16
|
+
* Introduced in STORAGE_VERSION_3.
|
|
17
|
+
*/
|
|
18
|
+
softDeleteCurrentData: boolean;
|
|
8
19
|
}
|
|
9
20
|
|
|
21
|
+
/**
|
|
22
|
+
* Corresponds to the storage version initially used, before we started explicitly versioning storage.
|
|
23
|
+
*/
|
|
24
|
+
export const STORAGE_VERSION_1 = 1;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* First new storage version.
|
|
28
|
+
*
|
|
29
|
+
* Uses versioned bucket names.
|
|
30
|
+
*
|
|
31
|
+
* On MongoDB storage, this always uses Long for checksums.
|
|
32
|
+
*/
|
|
33
|
+
export const STORAGE_VERSION_2 = 2;
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* This version is currently unstable, and not enabled by default yet.
|
|
37
|
+
*
|
|
38
|
+
* This is used to build towards incremental reprocessing.
|
|
39
|
+
*/
|
|
40
|
+
export const STORAGE_VERSION_3 = 3;
|
|
41
|
+
|
|
10
42
|
/**
|
|
11
43
|
* Oldest supported storage version.
|
|
12
44
|
*/
|
|
13
|
-
export const LEGACY_STORAGE_VERSION =
|
|
45
|
+
export const LEGACY_STORAGE_VERSION = STORAGE_VERSION_1;
|
|
14
46
|
|
|
15
47
|
/**
|
|
16
48
|
* Default storage version for newly persisted sync rules.
|
|
17
49
|
*/
|
|
18
|
-
export const CURRENT_STORAGE_VERSION =
|
|
50
|
+
export const CURRENT_STORAGE_VERSION = STORAGE_VERSION_2;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* All versions that can be loaded.
|
|
54
|
+
*
|
|
55
|
+
* This includes unstable versions.
|
|
56
|
+
*/
|
|
57
|
+
export const SUPPORTED_STORAGE_VERSIONS = [STORAGE_VERSION_1, STORAGE_VERSION_2, STORAGE_VERSION_3];
|
|
19
58
|
|
|
20
59
|
/**
|
|
21
60
|
* Shared storage-version behavior across storage implementations.
|
|
22
61
|
*/
|
|
23
62
|
export const STORAGE_VERSION_CONFIG: Record<number, StorageVersionConfig | undefined> = {
|
|
24
|
-
[
|
|
25
|
-
|
|
63
|
+
[STORAGE_VERSION_1]: {
|
|
64
|
+
version: STORAGE_VERSION_1,
|
|
65
|
+
versionedBuckets: false,
|
|
66
|
+
softDeleteCurrentData: false
|
|
67
|
+
},
|
|
68
|
+
[STORAGE_VERSION_2]: {
|
|
69
|
+
version: STORAGE_VERSION_2,
|
|
70
|
+
versionedBuckets: true,
|
|
71
|
+
softDeleteCurrentData: false
|
|
26
72
|
},
|
|
27
|
-
[
|
|
28
|
-
|
|
73
|
+
[STORAGE_VERSION_3]: {
|
|
74
|
+
version: STORAGE_VERSION_3,
|
|
75
|
+
versionedBuckets: true,
|
|
76
|
+
softDeleteCurrentData: true
|
|
29
77
|
}
|
|
30
78
|
};
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { Logger, ObserverClient } from '@powersync/lib-services-framework';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
BucketDataSource,
|
|
4
|
+
HydratedSyncRules,
|
|
5
|
+
ScopedParameterLookup,
|
|
6
|
+
SqliteJsonRow
|
|
7
|
+
} from '@powersync/service-sync-rules';
|
|
3
8
|
import * as util from '../util/util-index.js';
|
|
4
9
|
import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
|
|
5
10
|
import { BucketStorageFactory } from './BucketStorageFactory.js';
|
|
@@ -25,10 +30,17 @@ export interface SyncRulesBucketStorage
|
|
|
25
30
|
resolveTable(options: ResolveTableOptions): Promise<ResolveTableResult>;
|
|
26
31
|
|
|
27
32
|
/**
|
|
28
|
-
*
|
|
33
|
+
* Create a new writer.
|
|
34
|
+
*
|
|
35
|
+
* The writer must be flushed and disposed when done.
|
|
36
|
+
*/
|
|
37
|
+
createWriter(options: CreateWriterOptions): Promise<BucketStorageBatch>;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* @deprecated Use `createWriter()` with `await using` instead.
|
|
29
41
|
*/
|
|
30
42
|
startBatch(
|
|
31
|
-
options:
|
|
43
|
+
options: CreateWriterOptions,
|
|
32
44
|
callback: (batch: BucketStorageBatch) => Promise<void>
|
|
33
45
|
): Promise<FlushedResult | null>;
|
|
34
46
|
|
|
@@ -103,7 +115,7 @@ export interface SyncRulesBucketStorage
|
|
|
103
115
|
*/
|
|
104
116
|
getBucketDataBatch(
|
|
105
117
|
checkpoint: util.InternalOpId,
|
|
106
|
-
dataBuckets:
|
|
118
|
+
dataBuckets: BucketDataRequest[],
|
|
107
119
|
options?: BucketDataBatchOptions
|
|
108
120
|
): AsyncIterable<SyncBucketDataChunk>;
|
|
109
121
|
|
|
@@ -115,7 +127,7 @@ export interface SyncRulesBucketStorage
|
|
|
115
127
|
* This may be slow, depending on the size of the buckets.
|
|
116
128
|
* The checksums are cached internally to compensate for this, but does not cover all cases.
|
|
117
129
|
*/
|
|
118
|
-
getChecksums(checkpoint: util.InternalOpId, buckets:
|
|
130
|
+
getChecksums(checkpoint: util.InternalOpId, buckets: BucketChecksumRequest[]): Promise<util.ChecksumMap>;
|
|
119
131
|
|
|
120
132
|
/**
|
|
121
133
|
* Clear checksum cache. Primarily intended for tests.
|
|
@@ -127,6 +139,16 @@ export interface SyncRulesBucketStorageListener {
|
|
|
127
139
|
batchStarted: (batch: BucketStorageBatch) => void;
|
|
128
140
|
}
|
|
129
141
|
|
|
142
|
+
export interface BucketDataRequest {
|
|
143
|
+
bucket: string;
|
|
144
|
+
start: util.InternalOpId;
|
|
145
|
+
source: BucketDataSource;
|
|
146
|
+
}
|
|
147
|
+
export interface BucketChecksumRequest {
|
|
148
|
+
bucket: string;
|
|
149
|
+
source: BucketDataSource;
|
|
150
|
+
}
|
|
151
|
+
|
|
130
152
|
export interface SyncRuleStatus {
|
|
131
153
|
checkpoint_lsn: string | null;
|
|
132
154
|
active: boolean;
|
|
@@ -147,7 +169,7 @@ export interface ResolveTableResult {
|
|
|
147
169
|
dropTables: SourceTable[];
|
|
148
170
|
}
|
|
149
171
|
|
|
150
|
-
export interface
|
|
172
|
+
export interface CreateWriterOptions extends ParseSyncRulesOptions {
|
|
151
173
|
zeroLSN: string;
|
|
152
174
|
/**
|
|
153
175
|
* Whether or not to store a copy of the current data.
|
|
@@ -177,6 +199,11 @@ export interface StartBatchOptions extends ParseSyncRulesOptions {
|
|
|
177
199
|
logger?: Logger;
|
|
178
200
|
}
|
|
179
201
|
|
|
202
|
+
/**
|
|
203
|
+
* @deprecated Use `CreateWriterOptions`.
|
|
204
|
+
*/
|
|
205
|
+
export interface StartBatchOptions extends CreateWriterOptions {}
|
|
206
|
+
|
|
180
207
|
export interface CompactOptions {
|
|
181
208
|
/**
|
|
182
209
|
* Heap memory limit for the compact process.
|
|
@@ -7,7 +7,8 @@ import {
|
|
|
7
7
|
QuerierError,
|
|
8
8
|
RequestedStream,
|
|
9
9
|
RequestParameters,
|
|
10
|
-
ResolvedBucket
|
|
10
|
+
ResolvedBucket,
|
|
11
|
+
mergeBuckets
|
|
11
12
|
} from '@powersync/service-sync-rules';
|
|
12
13
|
|
|
13
14
|
import * as storage from '../storage/storage-index.js';
|
|
@@ -137,20 +138,20 @@ export class BucketChecksumState {
|
|
|
137
138
|
}
|
|
138
139
|
|
|
139
140
|
// Re-check updated buckets only
|
|
140
|
-
let checksumLookups:
|
|
141
|
+
let checksumLookups: storage.BucketChecksumRequest[] = [];
|
|
141
142
|
|
|
142
143
|
let newChecksums = new Map<string, util.BucketChecksum>();
|
|
143
|
-
for (let
|
|
144
|
-
if (!updatedBuckets.has(bucket)) {
|
|
145
|
-
const existing = this.lastChecksums.get(bucket);
|
|
144
|
+
for (let desc of bucketDescriptionMap.values()) {
|
|
145
|
+
if (!updatedBuckets.has(desc.bucket)) {
|
|
146
|
+
const existing = this.lastChecksums.get(desc.bucket);
|
|
146
147
|
if (existing == null) {
|
|
147
148
|
// If this happens, it means updatedBuckets did not correctly include all new buckets
|
|
148
|
-
throw new ServiceAssertionError(`Existing checksum not found for bucket ${bucket}`);
|
|
149
|
+
throw new ServiceAssertionError(`Existing checksum not found for bucket ${desc.bucket}`);
|
|
149
150
|
}
|
|
150
151
|
// Bucket is not specifically updated, and we have a previous checksum
|
|
151
|
-
newChecksums.set(bucket, existing);
|
|
152
|
+
newChecksums.set(desc.bucket, existing);
|
|
152
153
|
} else {
|
|
153
|
-
checksumLookups.push(bucket);
|
|
154
|
+
checksumLookups.push({ bucket: desc.bucket, source: desc.source });
|
|
154
155
|
}
|
|
155
156
|
}
|
|
156
157
|
|
|
@@ -163,12 +164,12 @@ export class BucketChecksumState {
|
|
|
163
164
|
checksumMap = newChecksums;
|
|
164
165
|
} else {
|
|
165
166
|
// Re-check all buckets
|
|
166
|
-
const bucketList = [...bucketDescriptionMap.
|
|
167
|
+
const bucketList = [...bucketDescriptionMap.values()].map((b) => ({ bucket: b.bucket, source: b.source }));
|
|
167
168
|
checksumMap = await storage.getChecksums(base.checkpoint, bucketList);
|
|
168
169
|
}
|
|
169
170
|
|
|
170
171
|
// Subset of buckets for which there may be new data in this batch.
|
|
171
|
-
let bucketsToFetch:
|
|
172
|
+
let bucketsToFetch: ResolvedBucket[];
|
|
172
173
|
|
|
173
174
|
let checkpointLine: util.StreamingSyncCheckpointDiff | util.StreamingSyncCheckpoint;
|
|
174
175
|
|
|
@@ -207,10 +208,7 @@ export class BucketChecksumState {
|
|
|
207
208
|
...this.parameterState.translateResolvedBucket(bucketDescriptionMap.get(e.bucket)!, streamNameToIndex)
|
|
208
209
|
}));
|
|
209
210
|
bucketsToFetch = [...generateBucketsToFetch].map((b) => {
|
|
210
|
-
return
|
|
211
|
-
priority: bucketDescriptionMap.get(b)!.priority,
|
|
212
|
-
bucket: b
|
|
213
|
-
};
|
|
211
|
+
return bucketDescriptionMap.get(b)!;
|
|
214
212
|
});
|
|
215
213
|
|
|
216
214
|
deferredLog = () => {
|
|
@@ -265,7 +263,7 @@ export class BucketChecksumState {
|
|
|
265
263
|
totalParamResults
|
|
266
264
|
);
|
|
267
265
|
};
|
|
268
|
-
bucketsToFetch = allBuckets
|
|
266
|
+
bucketsToFetch = allBuckets;
|
|
269
267
|
|
|
270
268
|
const subscriptions: util.StreamDescription[] = [];
|
|
271
269
|
const streamNameToIndex = new Map<string, number>();
|
|
@@ -342,17 +340,17 @@ export class BucketChecksumState {
|
|
|
342
340
|
deferredLog();
|
|
343
341
|
},
|
|
344
342
|
|
|
345
|
-
getFilteredBucketPositions: (buckets?:
|
|
343
|
+
getFilteredBucketPositions: (buckets?: ResolvedBucket[]): storage.BucketDataRequest[] => {
|
|
346
344
|
if (!hasAdvanced) {
|
|
347
345
|
throw new ServiceAssertionError('Call line.advance() before getFilteredBucketPositions()');
|
|
348
346
|
}
|
|
349
347
|
buckets ??= bucketsToFetch;
|
|
350
|
-
const filtered =
|
|
348
|
+
const filtered: storage.BucketDataRequest[] = [];
|
|
351
349
|
|
|
352
350
|
for (let bucket of buckets) {
|
|
353
351
|
const state = this.bucketDataPositions.get(bucket.bucket);
|
|
354
352
|
if (state) {
|
|
355
|
-
filtered.
|
|
353
|
+
filtered.push({ bucket: bucket.bucket, start: state.start_op_id, source: bucket.source });
|
|
356
354
|
}
|
|
357
355
|
}
|
|
358
356
|
return filtered;
|
|
@@ -660,7 +658,7 @@ export class BucketParameterState {
|
|
|
660
658
|
|
|
661
659
|
export interface CheckpointLine {
|
|
662
660
|
checkpointLine: util.StreamingSyncCheckpointDiff | util.StreamingSyncCheckpoint;
|
|
663
|
-
bucketsToFetch:
|
|
661
|
+
bucketsToFetch: ResolvedBucket[];
|
|
664
662
|
|
|
665
663
|
/**
|
|
666
664
|
* Call when a checkpoint line is being sent to a client, to update the internal state.
|
|
@@ -672,7 +670,7 @@ export interface CheckpointLine {
|
|
|
672
670
|
*
|
|
673
671
|
* @param bucketsToFetch List of buckets to fetch - either this.bucketsToFetch, or a subset of it. Defaults to this.bucketsToFetch.
|
|
674
672
|
*/
|
|
675
|
-
getFilteredBucketPositions(bucketsToFetch?:
|
|
673
|
+
getFilteredBucketPositions(bucketsToFetch?: ResolvedBucket[]): storage.BucketDataRequest[];
|
|
676
674
|
|
|
677
675
|
/**
|
|
678
676
|
* Update the position of bucket data the client has, after it was sent to the client.
|
|
@@ -762,32 +760,3 @@ function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number)
|
|
|
762
760
|
const limited = buckets.slice(0, limit);
|
|
763
761
|
return `${JSON.stringify(limited)}...`;
|
|
764
762
|
}
|
|
765
|
-
|
|
766
|
-
/**
|
|
767
|
-
* Resolves duplicate buckets in the given array, merging the inclusion reasons for duplicate.
|
|
768
|
-
*
|
|
769
|
-
* It's possible for duplicates to occur when a stream has multiple subscriptions, consider e.g.
|
|
770
|
-
*
|
|
771
|
-
* ```
|
|
772
|
-
* sync_streams:
|
|
773
|
-
* assets_by_category:
|
|
774
|
-
* query: select * from assets where category in (request.parameters() -> 'categories')
|
|
775
|
-
* ```
|
|
776
|
-
*
|
|
777
|
-
* Here, a client might subscribe once with `{"categories": [1]}` and once with `{"categories": [1, 2]}`. Since each
|
|
778
|
-
* subscription is evaluated independently, this would lead to three buckets, with a duplicate `assets_by_category[1]`
|
|
779
|
-
* bucket.
|
|
780
|
-
*/
|
|
781
|
-
function mergeBuckets(buckets: ResolvedBucket[]): ResolvedBucket[] {
|
|
782
|
-
const byBucketId: Record<string, ResolvedBucket> = {};
|
|
783
|
-
|
|
784
|
-
for (const bucket of buckets) {
|
|
785
|
-
if (Object.hasOwn(byBucketId, bucket.bucket)) {
|
|
786
|
-
byBucketId[bucket.bucket].inclusion_reasons.push(...bucket.inclusion_reasons);
|
|
787
|
-
} else {
|
|
788
|
-
byBucketId[bucket.bucket] = structuredClone(bucket);
|
|
789
|
-
}
|
|
790
|
-
}
|
|
791
|
-
|
|
792
|
-
return Object.values(byBucketId);
|
|
793
|
-
}
|
package/src/sync/sync.ts
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
BucketDescription,
|
|
4
|
+
BucketPriority,
|
|
5
|
+
HydratedSyncRules,
|
|
6
|
+
ResolvedBucket,
|
|
7
|
+
SqliteJsonValue
|
|
8
|
+
} from '@powersync/service-sync-rules';
|
|
3
9
|
|
|
4
10
|
import { AbortError } from 'ix/aborterror.js';
|
|
5
11
|
|
|
@@ -179,7 +185,7 @@ async function* streamResponseInner(
|
|
|
179
185
|
// receive a sync complete message after the synchronization is done (which happens in the last
|
|
180
186
|
// bucketDataInBatches iteration). Without any batch, the line is missing and clients might not complete their
|
|
181
187
|
// sync properly.
|
|
182
|
-
const priorityBatches: [BucketPriority | null,
|
|
188
|
+
const priorityBatches: [BucketPriority | null, ResolvedBucket[]][] = bucketsByPriority;
|
|
183
189
|
if (priorityBatches.length == 0) {
|
|
184
190
|
priorityBatches.push([null, []]);
|
|
185
191
|
}
|
|
@@ -257,7 +263,7 @@ interface BucketDataRequest {
|
|
|
257
263
|
/** Contains current bucket state. Modified by the request as data is sent. */
|
|
258
264
|
checkpointLine: CheckpointLine;
|
|
259
265
|
/** Subset of checkpointLine.bucketsToFetch, filtered by priority. */
|
|
260
|
-
bucketsToFetch:
|
|
266
|
+
bucketsToFetch: ResolvedBucket[];
|
|
261
267
|
/** Whether data lines should be encoded in a legacy format where {@link util.OplogEntry.data} is a nested object. */
|
|
262
268
|
legacyDataLines: boolean;
|
|
263
269
|
/** Signals that the connection was aborted and that streaming should stop ASAP. */
|
package/src/sync/util.ts
CHANGED
|
@@ -183,6 +183,16 @@ export function settledPromise<T>(promise: Promise<T>): Promise<PromiseSettledRe
|
|
|
183
183
|
);
|
|
184
184
|
}
|
|
185
185
|
|
|
186
|
+
export function unsettledPromise<T>(settled: Promise<PromiseSettledResult<T>>): Promise<T> {
|
|
187
|
+
return settled.then((result) => {
|
|
188
|
+
if (result.status === 'fulfilled') {
|
|
189
|
+
return Promise.resolve(result.value);
|
|
190
|
+
} else {
|
|
191
|
+
return Promise.reject(result.reason);
|
|
192
|
+
}
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
|
|
186
196
|
export type MapOrSet<T> = Map<T, any> | Set<T>;
|
|
187
197
|
|
|
188
198
|
/**
|
package/test/src/auth.test.ts
CHANGED
|
@@ -7,7 +7,7 @@ import { RemoteJWKSCollector } from '../../src/auth/RemoteJWKSCollector.js';
|
|
|
7
7
|
import { KeyResult } from '../../src/auth/KeyCollector.js';
|
|
8
8
|
import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
|
|
9
9
|
import { JwtPayload, StaticSupabaseKeyCollector } from '@/index.js';
|
|
10
|
-
import { debugKeyNotFound } from '../../src/auth/utils.js';
|
|
10
|
+
import { debugKeyNotFound, getSupabaseJwksUrl } from '../../src/auth/utils.js';
|
|
11
11
|
|
|
12
12
|
const publicKeyRSA: jose.JWK = {
|
|
13
13
|
use: 'sig',
|
|
@@ -496,6 +496,25 @@ describe('JWT Auth', () => {
|
|
|
496
496
|
expect(verified.parsedPayload.claim).toEqual('test-claim-2');
|
|
497
497
|
});
|
|
498
498
|
|
|
499
|
+
test('Supabase connection parsing', () => {
|
|
500
|
+
const details = getSupabaseJwksUrl({
|
|
501
|
+
type: 'postgresql',
|
|
502
|
+
uri: 'postgresql://db.abc123.supabase.co:5432/postgres'
|
|
503
|
+
});
|
|
504
|
+
expect(details).toEqual({
|
|
505
|
+
hostname: 'db.abc123.supabase.co',
|
|
506
|
+
projectId: 'abc123',
|
|
507
|
+
url: 'https://abc123.supabase.co/auth/v1/.well-known/jwks.json'
|
|
508
|
+
});
|
|
509
|
+
|
|
510
|
+
// supabase.com is not a supabase URI
|
|
511
|
+
const other = getSupabaseJwksUrl({
|
|
512
|
+
type: 'postgresql',
|
|
513
|
+
uri: 'postgresql://db.abc123.supabase.com:5432/postgres'
|
|
514
|
+
});
|
|
515
|
+
expect(other).toEqual(null);
|
|
516
|
+
});
|
|
517
|
+
|
|
499
518
|
describe('debugKeyNotFound', () => {
|
|
500
519
|
test('Supabase token with legacy auth not configured', async () => {
|
|
501
520
|
const keys = await StaticSupabaseKeyCollector.importKeys([]);
|