@powersync/service-core 0.0.0-dev-20241021151922 → 0.0.0-dev-20241021185145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -5
- package/dist/api/RouteAPI.d.ts +6 -4
- package/dist/api/diagnostics.js +3 -1
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/api/schema.js +2 -2
- package/dist/api/schema.js.map +1 -1
- package/dist/replication/AbstractReplicationJob.js +2 -2
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/ReplicationModule.js +3 -0
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/routes/configure-fastify.js +12 -12
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/configure-rsocket.js +4 -1
- package/dist/routes/configure-rsocket.js.map +1 -1
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +14 -82
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/router.d.ts +8 -1
- package/dist/routes/router.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +19 -15
- package/dist/storage/BucketStorage.js +6 -0
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/MongoBucketStorage.d.ts +9 -3
- package/dist/storage/MongoBucketStorage.js +21 -4
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/StorageEngine.d.ts +5 -1
- package/dist/storage/StorageEngine.js +19 -1
- package/dist/storage/StorageEngine.js.map +1 -1
- package/dist/storage/StorageProvider.d.ts +8 -1
- package/dist/storage/mongo/MongoBucketBatch.d.ts +2 -2
- package/dist/storage/mongo/MongoBucketBatch.js +1 -0
- package/dist/storage/mongo/MongoBucketBatch.js.map +1 -1
- package/dist/storage/mongo/MongoStorageProvider.js +2 -1
- package/dist/storage/mongo/MongoStorageProvider.js.map +1 -1
- package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +3 -13
- package/dist/storage/mongo/MongoSyncBucketStorage.js +7 -35
- package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/mongo/MongoWriteCheckpointAPI.d.ts +2 -4
- package/dist/storage/mongo/MongoWriteCheckpointAPI.js +7 -13
- package/dist/storage/mongo/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/storage-index.d.ts +2 -2
- package/dist/storage/storage-index.js +2 -2
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/storage/{WriteCheckpointAPI.d.ts → write-checkpoint.d.ts} +9 -28
- package/dist/storage/{WriteCheckpointAPI.js → write-checkpoint.js} +1 -1
- package/dist/storage/write-checkpoint.js.map +1 -0
- package/dist/util/config/compound-config-collector.js +1 -2
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/types.d.ts +0 -1
- package/dist/util/protocol-types.d.ts +2 -1
- package/package.json +5 -5
- package/src/api/RouteAPI.ts +7 -4
- package/src/api/diagnostics.ts +3 -1
- package/src/api/schema.ts +3 -3
- package/src/replication/AbstractReplicationJob.ts +2 -2
- package/src/replication/ReplicationModule.ts +4 -0
- package/src/routes/configure-fastify.ts +16 -17
- package/src/routes/configure-rsocket.ts +7 -2
- package/src/routes/endpoints/admin.ts +2 -2
- package/src/routes/endpoints/checkpointing.ts +1 -8
- package/src/routes/endpoints/sync-rules.ts +1 -0
- package/src/routes/router.ts +7 -1
- package/src/storage/BucketStorage.ts +23 -16
- package/src/storage/MongoBucketStorage.ts +39 -9
- package/src/storage/StorageEngine.ts +24 -2
- package/src/storage/StorageProvider.ts +9 -1
- package/src/storage/mongo/MongoBucketBatch.ts +3 -2
- package/src/storage/mongo/MongoStorageProvider.ts +2 -1
- package/src/storage/mongo/MongoSyncBucketStorage.ts +9 -52
- package/src/storage/mongo/MongoWriteCheckpointAPI.ts +8 -16
- package/src/storage/storage-index.ts +2 -2
- package/src/storage/{WriteCheckpointAPI.ts → write-checkpoint.ts} +12 -30
- package/src/util/config/compound-config-collector.ts +1 -2
- package/src/util/config/types.ts +0 -1
- package/src/util/protocol-types.ts +1 -1
- package/test/src/compacting.test.ts +13 -15
- package/test/src/data_storage.test.ts +56 -56
- package/test/src/sync.test.ts +10 -9
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/WriteCheckpointAPI.js.map +0 -1
|
@@ -21,13 +21,20 @@ import {
|
|
|
21
21
|
UpdateSyncRulesOptions,
|
|
22
22
|
WriteCheckpoint
|
|
23
23
|
} from './BucketStorage.js';
|
|
24
|
-
import { PowerSyncMongo
|
|
24
|
+
import { PowerSyncMongo } from './mongo/db.js';
|
|
25
25
|
import { SyncRuleDocument, SyncRuleState } from './mongo/models.js';
|
|
26
26
|
import { MongoPersistedSyncRulesContent } from './mongo/MongoPersistedSyncRulesContent.js';
|
|
27
27
|
import { MongoSyncBucketStorage } from './mongo/MongoSyncBucketStorage.js';
|
|
28
|
+
import { MongoWriteCheckpointAPI } from './mongo/MongoWriteCheckpointAPI.js';
|
|
28
29
|
import { generateSlotName } from './mongo/util.js';
|
|
29
|
-
|
|
30
|
-
|
|
30
|
+
import {
|
|
31
|
+
CustomWriteCheckpointOptions,
|
|
32
|
+
DEFAULT_WRITE_CHECKPOINT_MODE,
|
|
33
|
+
LastWriteCheckpointFilters,
|
|
34
|
+
ManagedWriteCheckpointOptions,
|
|
35
|
+
WriteCheckpointAPI,
|
|
36
|
+
WriteCheckpointMode
|
|
37
|
+
} from './write-checkpoint.js';
|
|
31
38
|
|
|
32
39
|
export class MongoBucketStorage
|
|
33
40
|
extends DisposableObserver<BucketStorageFactoryListener>
|
|
@@ -38,6 +45,10 @@ export class MongoBucketStorage
|
|
|
38
45
|
// TODO: This is still Postgres specific and needs to be reworked
|
|
39
46
|
public readonly slot_name_prefix: string;
|
|
40
47
|
|
|
48
|
+
readonly write_checkpoint_mode: WriteCheckpointMode;
|
|
49
|
+
|
|
50
|
+
protected readonly writeCheckpointAPI: WriteCheckpointAPI;
|
|
51
|
+
|
|
41
52
|
private readonly storageCache = new LRUCache<number, MongoSyncBucketStorage>({
|
|
42
53
|
max: 3,
|
|
43
54
|
fetchMethod: async (id) => {
|
|
@@ -65,6 +76,7 @@ export class MongoBucketStorage
|
|
|
65
76
|
db: PowerSyncMongo,
|
|
66
77
|
options: {
|
|
67
78
|
slot_name_prefix: string;
|
|
79
|
+
write_checkpoint_mode?: WriteCheckpointMode;
|
|
68
80
|
}
|
|
69
81
|
) {
|
|
70
82
|
super();
|
|
@@ -72,6 +84,11 @@ export class MongoBucketStorage
|
|
|
72
84
|
this.db = db;
|
|
73
85
|
this.session = this.client.startSession();
|
|
74
86
|
this.slot_name_prefix = options.slot_name_prefix;
|
|
87
|
+
this.write_checkpoint_mode = options.write_checkpoint_mode ?? DEFAULT_WRITE_CHECKPOINT_MODE;
|
|
88
|
+
this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
|
|
89
|
+
db,
|
|
90
|
+
mode: this.write_checkpoint_mode
|
|
91
|
+
});
|
|
75
92
|
}
|
|
76
93
|
|
|
77
94
|
getInstance(options: PersistedSyncRulesContent): MongoSyncBucketStorage {
|
|
@@ -280,6 +297,22 @@ export class MongoBucketStorage
|
|
|
280
297
|
});
|
|
281
298
|
}
|
|
282
299
|
|
|
300
|
+
async batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void> {
|
|
301
|
+
return this.writeCheckpointAPI.batchCreateCustomWriteCheckpoints(checkpoints);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
|
|
305
|
+
return this.writeCheckpointAPI.createCustomWriteCheckpoint(options);
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
async createManagedWriteCheckpoint(options: ManagedWriteCheckpointOptions): Promise<bigint> {
|
|
309
|
+
return this.writeCheckpointAPI.createManagedWriteCheckpoint(options);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
async lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null> {
|
|
313
|
+
return this.writeCheckpointAPI.lastWriteCheckpoint(filters);
|
|
314
|
+
}
|
|
315
|
+
|
|
283
316
|
async getActiveCheckpoint(): Promise<ActiveCheckpoint> {
|
|
284
317
|
const doc = await this.db.sync_rules.findOne(
|
|
285
318
|
{
|
|
@@ -391,7 +424,7 @@ export class MongoBucketStorage
|
|
|
391
424
|
}
|
|
392
425
|
return (await this.storageCache.fetch(doc._id)) ?? null;
|
|
393
426
|
}
|
|
394
|
-
}
|
|
427
|
+
};
|
|
395
428
|
}
|
|
396
429
|
|
|
397
430
|
/**
|
|
@@ -481,7 +514,6 @@ export class MongoBucketStorage
|
|
|
481
514
|
if (doc == null) {
|
|
482
515
|
continue;
|
|
483
516
|
}
|
|
484
|
-
|
|
485
517
|
const op = this.makeActiveCheckpoint(doc);
|
|
486
518
|
// Check for LSN / checkpoint changes - ignore other metadata changes
|
|
487
519
|
if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
|
|
@@ -512,14 +544,12 @@ export class MongoBucketStorage
|
|
|
512
544
|
// 1. checkpoint (op_id) changes.
|
|
513
545
|
// 2. write checkpoint changes for the specific user
|
|
514
546
|
const bucketStorage = await cp.getBucketStorage();
|
|
515
|
-
if (!bucketStorage) {
|
|
516
|
-
continue;
|
|
517
|
-
}
|
|
518
547
|
|
|
519
548
|
const lsnFilters: Record<string, string> = lsn ? { 1: lsn } : {};
|
|
520
549
|
|
|
521
|
-
const currentWriteCheckpoint = await
|
|
550
|
+
const currentWriteCheckpoint = await this.lastWriteCheckpoint({
|
|
522
551
|
user_id,
|
|
552
|
+
sync_rules_id: bucketStorage?.group_id,
|
|
523
553
|
heads: {
|
|
524
554
|
...lsnFilters
|
|
525
555
|
}
|
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
import { DisposableListener, DisposableObserver, logger } from '@powersync/lib-services-framework';
|
|
2
2
|
import { ResolvedPowerSyncConfig } from '../util/util-index.js';
|
|
3
3
|
import { BucketStorageFactory } from './BucketStorage.js';
|
|
4
|
-
import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
|
|
4
|
+
import { ActiveStorage, BucketStorageProvider, StorageSettings } from './StorageProvider.js';
|
|
5
|
+
import { DEFAULT_WRITE_CHECKPOINT_MODE } from './write-checkpoint.js';
|
|
5
6
|
|
|
6
7
|
export type StorageEngineOptions = {
|
|
7
8
|
configuration: ResolvedPowerSyncConfig;
|
|
8
9
|
};
|
|
9
10
|
|
|
11
|
+
export const DEFAULT_STORAGE_SETTINGS: StorageSettings = {
|
|
12
|
+
writeCheckpointMode: DEFAULT_WRITE_CHECKPOINT_MODE
|
|
13
|
+
};
|
|
14
|
+
|
|
10
15
|
export interface StorageEngineListener extends DisposableListener {
|
|
11
16
|
storageActivated: (storage: BucketStorageFactory) => void;
|
|
12
17
|
}
|
|
@@ -15,9 +20,11 @@ export class StorageEngine extends DisposableObserver<StorageEngineListener> {
|
|
|
15
20
|
// TODO: This will need to revisited when we actually support multiple storage providers.
|
|
16
21
|
private storageProviders: Map<string, BucketStorageProvider> = new Map();
|
|
17
22
|
private currentActiveStorage: ActiveStorage | null = null;
|
|
23
|
+
private _activeSettings: StorageSettings;
|
|
18
24
|
|
|
19
25
|
constructor(private options: StorageEngineOptions) {
|
|
20
26
|
super();
|
|
27
|
+
this._activeSettings = DEFAULT_STORAGE_SETTINGS;
|
|
21
28
|
}
|
|
22
29
|
|
|
23
30
|
get activeBucketStorage(): BucketStorageFactory {
|
|
@@ -32,6 +39,20 @@ export class StorageEngine extends DisposableObserver<StorageEngineListener> {
|
|
|
32
39
|
return this.currentActiveStorage;
|
|
33
40
|
}
|
|
34
41
|
|
|
42
|
+
get activeSettings(): StorageSettings {
|
|
43
|
+
return { ...this._activeSettings };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
updateSettings(settings: Partial<StorageSettings>) {
|
|
47
|
+
if (this.currentActiveStorage) {
|
|
48
|
+
throw new Error(`Storage is already active, settings cannot be modified.`);
|
|
49
|
+
}
|
|
50
|
+
this._activeSettings = {
|
|
51
|
+
...this._activeSettings,
|
|
52
|
+
...settings
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
35
56
|
/**
|
|
36
57
|
* Register a provider which generates a {@link BucketStorageFactory}
|
|
37
58
|
* given the matching config specified in the loaded {@link ResolvedPowerSyncConfig}
|
|
@@ -44,7 +65,8 @@ export class StorageEngine extends DisposableObserver<StorageEngineListener> {
|
|
|
44
65
|
logger.info('Starting Storage Engine...');
|
|
45
66
|
const { configuration } = this.options;
|
|
46
67
|
this.currentActiveStorage = await this.storageProviders.get(configuration.storage.type)!.getStorage({
|
|
47
|
-
resolvedConfig: configuration
|
|
68
|
+
resolvedConfig: configuration,
|
|
69
|
+
...this.activeSettings
|
|
48
70
|
});
|
|
49
71
|
this.iterateListeners((cb) => cb.storageActivated?.(this.activeBucketStorage));
|
|
50
72
|
logger.info(`Successfully activated storage: ${configuration.storage.type}.`);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as util from '../util/util-index.js';
|
|
2
2
|
import { BucketStorageFactory } from './BucketStorage.js';
|
|
3
|
+
import { WriteCheckpointMode } from './write-checkpoint.js';
|
|
3
4
|
|
|
4
5
|
export interface ActiveStorage {
|
|
5
6
|
storage: BucketStorageFactory;
|
|
@@ -11,7 +12,14 @@ export interface ActiveStorage {
|
|
|
11
12
|
tearDown(): Promise<boolean>;
|
|
12
13
|
}
|
|
13
14
|
|
|
14
|
-
|
|
15
|
+
/**
|
|
16
|
+
* Settings which can be modified by various modules in their initialization.
|
|
17
|
+
*/
|
|
18
|
+
export interface StorageSettings {
|
|
19
|
+
writeCheckpointMode: WriteCheckpointMode;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface GetStorageOptions extends StorageSettings {
|
|
15
23
|
// TODO: This should just be the storage config. Update once the slot name prefix coupling has been removed from the storage
|
|
16
24
|
resolvedConfig: util.ResolvedPowerSyncConfig;
|
|
17
25
|
}
|
|
@@ -12,7 +12,7 @@ import {
|
|
|
12
12
|
SaveOptions
|
|
13
13
|
} from '../BucketStorage.js';
|
|
14
14
|
import { SourceTable } from '../SourceTable.js';
|
|
15
|
-
import {
|
|
15
|
+
import { CustomWriteCheckpointOptions } from '../write-checkpoint.js';
|
|
16
16
|
import { PowerSyncMongo } from './db.js';
|
|
17
17
|
import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js';
|
|
18
18
|
import { MongoIdSequence } from './MongoIdSequence.js';
|
|
@@ -81,9 +81,10 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
|
|
|
81
81
|
this.session = this.client.startSession();
|
|
82
82
|
this.slot_name = slot_name;
|
|
83
83
|
this.sync_rules = sync_rules;
|
|
84
|
+
this.batch = new OperationBatch();
|
|
84
85
|
}
|
|
85
86
|
|
|
86
|
-
addCustomWriteCheckpoint(checkpoint:
|
|
87
|
+
addCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): void {
|
|
87
88
|
this.write_checkpoint_batch.push({
|
|
88
89
|
...checkpoint,
|
|
89
90
|
sync_rules_id: this.group_id
|
|
@@ -19,7 +19,8 @@ export class MongoStorageProvider implements BucketStorageProvider {
|
|
|
19
19
|
return {
|
|
20
20
|
storage: new MongoBucketStorage(database, {
|
|
21
21
|
// TODO currently need the entire resolved config due to this
|
|
22
|
-
slot_name_prefix: resolvedConfig.slot_name_prefix
|
|
22
|
+
slot_name_prefix: resolvedConfig.slot_name_prefix,
|
|
23
|
+
write_checkpoint_mode: options.writeCheckpointMode
|
|
23
24
|
}),
|
|
24
25
|
shutDown: () => client.close(),
|
|
25
26
|
tearDown: () => {
|
|
@@ -2,13 +2,13 @@ import { SqliteJsonRow, SqliteJsonValue, SqlSyncRules } from '@powersync/service
|
|
|
2
2
|
import * as bson from 'bson';
|
|
3
3
|
import * as mongo from 'mongodb';
|
|
4
4
|
|
|
5
|
-
import { DisposableObserver
|
|
6
|
-
import * as timers from 'timers/promises';
|
|
5
|
+
import { DisposableObserver } from '@powersync/lib-services-framework';
|
|
7
6
|
import * as db from '../../db/db-index.js';
|
|
8
7
|
import * as util from '../../util/util-index.js';
|
|
9
8
|
import {
|
|
10
9
|
BucketDataBatchOptions,
|
|
11
10
|
BucketStorageBatch,
|
|
11
|
+
Checkpoint,
|
|
12
12
|
CompactOptions,
|
|
13
13
|
DEFAULT_DOCUMENT_BATCH_LIMIT,
|
|
14
14
|
DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES,
|
|
@@ -27,19 +27,13 @@ import {
|
|
|
27
27
|
import { ChecksumCache, FetchPartialBucketChecksum, PartialChecksum, PartialChecksumMap } from '../ChecksumCache.js';
|
|
28
28
|
import { MongoBucketStorage } from '../MongoBucketStorage.js';
|
|
29
29
|
import { SourceTable } from '../SourceTable.js';
|
|
30
|
-
import {
|
|
31
|
-
BatchedCustomWriteCheckpointOptions,
|
|
32
|
-
ManagedWriteCheckpointOptions,
|
|
33
|
-
SyncStorageLastWriteCheckpointFilters,
|
|
34
|
-
WriteCheckpointAPI,
|
|
35
|
-
WriteCheckpointMode
|
|
36
|
-
} from '../WriteCheckpointAPI.js';
|
|
37
30
|
import { PowerSyncMongo } from './db.js';
|
|
38
31
|
import { BucketDataDocument, BucketDataKey, SourceKey, SyncRuleState } from './models.js';
|
|
39
32
|
import { MongoBucketBatch } from './MongoBucketBatch.js';
|
|
40
33
|
import { MongoCompactor } from './MongoCompactor.js';
|
|
41
|
-
import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
|
|
42
34
|
import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, mapOpEntry, readSingleBatch, serializeLookup } from './util.js';
|
|
35
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
36
|
+
import * as timers from 'timers/promises';
|
|
43
37
|
|
|
44
38
|
export class MongoSyncBucketStorage
|
|
45
39
|
extends DisposableObserver<SyncRulesBucketStorageListener>
|
|
@@ -53,53 +47,15 @@ export class MongoSyncBucketStorage
|
|
|
53
47
|
});
|
|
54
48
|
|
|
55
49
|
private parsedSyncRulesCache: SqlSyncRules | undefined;
|
|
56
|
-
private writeCheckpointAPI: WriteCheckpointAPI;
|
|
57
50
|
|
|
58
51
|
constructor(
|
|
59
52
|
public readonly factory: MongoBucketStorage,
|
|
60
53
|
public readonly group_id: number,
|
|
61
54
|
private readonly sync_rules: PersistedSyncRulesContent,
|
|
62
|
-
public readonly slot_name: string
|
|
63
|
-
writeCheckpointMode: WriteCheckpointMode = WriteCheckpointMode.MANAGED
|
|
55
|
+
public readonly slot_name: string
|
|
64
56
|
) {
|
|
65
57
|
super();
|
|
66
58
|
this.db = factory.db;
|
|
67
|
-
this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
|
|
68
|
-
db: this.db,
|
|
69
|
-
mode: writeCheckpointMode
|
|
70
|
-
});
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
get writeCheckpointMode() {
|
|
74
|
-
return this.writeCheckpointAPI.writeCheckpointMode;
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
setWriteCheckpointMode(mode: WriteCheckpointMode): void {
|
|
78
|
-
this.writeCheckpointAPI.setWriteCheckpointMode(mode);
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
batchCreateCustomWriteCheckpoints(checkpoints: BatchedCustomWriteCheckpointOptions[]): Promise<void> {
|
|
82
|
-
return this.writeCheckpointAPI.batchCreateCustomWriteCheckpoints(
|
|
83
|
-
checkpoints.map((checkpoint) => ({ ...checkpoint, sync_rules_id: this.group_id }))
|
|
84
|
-
);
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
createCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): Promise<bigint> {
|
|
88
|
-
return this.writeCheckpointAPI.createCustomWriteCheckpoint({
|
|
89
|
-
...checkpoint,
|
|
90
|
-
sync_rules_id: this.group_id
|
|
91
|
-
});
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint> {
|
|
95
|
-
return this.writeCheckpointAPI.createManagedWriteCheckpoint(checkpoint);
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null> {
|
|
99
|
-
return this.writeCheckpointAPI.lastWriteCheckpoint({
|
|
100
|
-
...filters,
|
|
101
|
-
sync_rules_id: this.group_id
|
|
102
|
-
});
|
|
103
59
|
}
|
|
104
60
|
|
|
105
61
|
getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules {
|
|
@@ -107,15 +63,16 @@ export class MongoSyncBucketStorage
|
|
|
107
63
|
return this.parsedSyncRulesCache;
|
|
108
64
|
}
|
|
109
65
|
|
|
110
|
-
async getCheckpoint() {
|
|
66
|
+
async getCheckpoint(): Promise<Checkpoint> {
|
|
111
67
|
const doc = await this.db.sync_rules.findOne(
|
|
112
68
|
{ _id: this.group_id },
|
|
113
69
|
{
|
|
114
|
-
projection: { last_checkpoint: 1 }
|
|
70
|
+
projection: { last_checkpoint: 1, last_checkpoint_lsn: 1 }
|
|
115
71
|
}
|
|
116
72
|
);
|
|
117
73
|
return {
|
|
118
|
-
checkpoint: util.timestampToOpId(doc?.last_checkpoint ?? 0n)
|
|
74
|
+
checkpoint: util.timestampToOpId(doc?.last_checkpoint ?? 0n),
|
|
75
|
+
lsn: doc?.last_checkpoint_lsn ?? null
|
|
119
76
|
};
|
|
120
77
|
}
|
|
121
78
|
|
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
ManagedWriteCheckpointOptions,
|
|
8
8
|
WriteCheckpointAPI,
|
|
9
9
|
WriteCheckpointMode
|
|
10
|
-
} from '../
|
|
10
|
+
} from '../write-checkpoint.js';
|
|
11
11
|
import { PowerSyncMongo } from './db.js';
|
|
12
12
|
|
|
13
13
|
export type MongoCheckpointAPIOptions = {
|
|
@@ -17,19 +17,11 @@ export type MongoCheckpointAPIOptions = {
|
|
|
17
17
|
|
|
18
18
|
export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
|
|
19
19
|
readonly db: PowerSyncMongo;
|
|
20
|
-
|
|
20
|
+
readonly mode: WriteCheckpointMode;
|
|
21
21
|
|
|
22
22
|
constructor(options: MongoCheckpointAPIOptions) {
|
|
23
23
|
this.db = options.db;
|
|
24
|
-
this.
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
get writeCheckpointMode() {
|
|
28
|
-
return this._mode;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
setWriteCheckpointMode(mode: WriteCheckpointMode): void {
|
|
32
|
-
this._mode = mode;
|
|
24
|
+
this.mode = options.mode;
|
|
33
25
|
}
|
|
34
26
|
|
|
35
27
|
async batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void> {
|
|
@@ -37,9 +29,9 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
|
|
|
37
29
|
}
|
|
38
30
|
|
|
39
31
|
async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
|
|
40
|
-
if (this.
|
|
32
|
+
if (this.mode !== WriteCheckpointMode.CUSTOM) {
|
|
41
33
|
throw new framework.errors.ValidationError(
|
|
42
|
-
`Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.
|
|
34
|
+
`Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"`
|
|
43
35
|
);
|
|
44
36
|
}
|
|
45
37
|
|
|
@@ -60,9 +52,9 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
|
|
|
60
52
|
}
|
|
61
53
|
|
|
62
54
|
async createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint> {
|
|
63
|
-
if (this.
|
|
55
|
+
if (this.mode !== WriteCheckpointMode.MANAGED) {
|
|
64
56
|
throw new framework.errors.ValidationError(
|
|
65
|
-
`
|
|
57
|
+
`Creating a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"`
|
|
66
58
|
);
|
|
67
59
|
}
|
|
68
60
|
|
|
@@ -85,7 +77,7 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
|
|
|
85
77
|
}
|
|
86
78
|
|
|
87
79
|
async lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null> {
|
|
88
|
-
switch (this.
|
|
80
|
+
switch (this.mode) {
|
|
89
81
|
case WriteCheckpointMode.CUSTOM:
|
|
90
82
|
if (false == 'sync_rules_id' in filters) {
|
|
91
83
|
throw new framework.errors.ValidationError(`Sync rules ID is required for custom Write Checkpoint filtering`);
|
|
@@ -5,7 +5,6 @@ export * from './SourceEntity.js';
|
|
|
5
5
|
export * from './SourceTable.js';
|
|
6
6
|
export * from './StorageEngine.js';
|
|
7
7
|
|
|
8
|
-
export * from './mongo/config.js';
|
|
9
8
|
export * from './mongo/db.js';
|
|
10
9
|
export * from './mongo/models.js';
|
|
11
10
|
export * from './mongo/MongoBucketBatch.js';
|
|
@@ -18,4 +17,5 @@ export * from './mongo/MongoSyncRulesLock.js';
|
|
|
18
17
|
export * from './mongo/OperationBatch.js';
|
|
19
18
|
export * from './mongo/PersistedBatch.js';
|
|
20
19
|
export * from './mongo/util.js';
|
|
21
|
-
export * from './
|
|
20
|
+
export * from './mongo/config.js';
|
|
21
|
+
export * from './write-checkpoint.js';
|
|
@@ -26,19 +26,19 @@ export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentif
|
|
|
26
26
|
sync_rules_id: number;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
export interface
|
|
29
|
+
export interface CustomWriteCheckpointOptions extends CustomWriteCheckpointFilters {
|
|
30
30
|
/**
|
|
31
31
|
* A supplied incrementing Write Checkpoint number
|
|
32
32
|
*/
|
|
33
33
|
checkpoint: bigint;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
36
|
+
/**
|
|
37
|
+
* Options for creating a custom Write Checkpoint in a batch.
|
|
38
|
+
* A {@link BucketStorageBatch} is already associated with a Sync Rules instance.
|
|
39
|
+
* The `sync_rules_id` is not required here.
|
|
40
|
+
*/
|
|
41
|
+
export type BatchedCustomWriteCheckpointOptions = Omit<CustomWriteCheckpointOptions, 'sync_rules_id'>;
|
|
42
42
|
|
|
43
43
|
/**
|
|
44
44
|
* Managed Write Checkpoints are a mapping of User ID to replication HEAD
|
|
@@ -52,33 +52,15 @@ export interface ManagedWriteCheckpointFilters extends BaseWriteCheckpointIdenti
|
|
|
52
52
|
|
|
53
53
|
export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
|
|
54
54
|
|
|
55
|
-
export type SyncStorageLastWriteCheckpointFilters = BaseWriteCheckpointIdentifier | ManagedWriteCheckpointFilters;
|
|
56
55
|
export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
|
|
57
56
|
|
|
58
|
-
export interface
|
|
59
|
-
readonly writeCheckpointMode: WriteCheckpointMode;
|
|
60
|
-
setWriteCheckpointMode(mode: WriteCheckpointMode): void;
|
|
61
|
-
createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint>;
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
/**
|
|
65
|
-
* Write Checkpoint API to be used in conjunction with a {@link SyncRulesBucketStorage}.
|
|
66
|
-
* This storage corresponds with a set of sync rules. These APIs don't require specifying a
|
|
67
|
-
* sync rules id.
|
|
68
|
-
*/
|
|
69
|
-
export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
|
|
70
|
-
batchCreateCustomWriteCheckpoints(checkpoints: BatchedCustomWriteCheckpointOptions[]): Promise<void>;
|
|
71
|
-
createCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): Promise<bigint>;
|
|
72
|
-
lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
/**
|
|
76
|
-
* Write Checkpoint API which is interfaced directly with the storage layer. This requires
|
|
77
|
-
* sync rules identifiers for custom write checkpoints.
|
|
78
|
-
*/
|
|
79
|
-
export interface WriteCheckpointAPI extends BaseWriteCheckpointAPI {
|
|
57
|
+
export interface WriteCheckpointAPI {
|
|
80
58
|
batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
|
|
59
|
+
|
|
81
60
|
createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise<bigint>;
|
|
61
|
+
|
|
62
|
+
createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint>;
|
|
63
|
+
|
|
82
64
|
lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null>;
|
|
83
65
|
}
|
|
84
66
|
|
|
@@ -122,8 +122,7 @@ export class CompoundConfigCollector {
|
|
|
122
122
|
},
|
|
123
123
|
// TODO maybe move this out of the connection or something
|
|
124
124
|
// slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
|
|
125
|
-
slot_name_prefix: 'powersync_'
|
|
126
|
-
parameters: baseConfig.parameters ?? {}
|
|
125
|
+
slot_name_prefix: 'powersync_'
|
|
127
126
|
};
|
|
128
127
|
|
|
129
128
|
return config;
|
package/src/util/config/types.ts
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
|
+
import { SaveOperationTag } from '@/storage/BucketStorage.js';
|
|
1
2
|
import { MongoCompactOptions } from '@/storage/mongo/MongoCompactor.js';
|
|
2
|
-
import { SqlSyncRules } from '@powersync/service-sync-rules';
|
|
3
3
|
import { describe, expect, test } from 'vitest';
|
|
4
4
|
import { validateCompactedBucket } from './bucket_validation.js';
|
|
5
5
|
import { oneFromAsync } from './stream_utils.js';
|
|
6
|
-
import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, rid, testRules
|
|
7
|
-
import { ParseSyncRulesOptions, PersistedSyncRulesContent, StartBatchOptions } from '@/storage/BucketStorage.js';
|
|
8
|
-
import { getUuidReplicaIdentityBson } from '@/util/util-index.js';
|
|
6
|
+
import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, rid, testRules } from './util.js';
|
|
9
7
|
|
|
10
8
|
const TEST_TABLE = makeTestTable('test', ['id']);
|
|
11
9
|
|
|
@@ -31,7 +29,7 @@ bucket_definitions:
|
|
|
31
29
|
const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
|
|
32
30
|
await batch.save({
|
|
33
31
|
sourceTable: TEST_TABLE,
|
|
34
|
-
tag:
|
|
32
|
+
tag: SaveOperationTag.INSERT,
|
|
35
33
|
after: {
|
|
36
34
|
id: 't1'
|
|
37
35
|
},
|
|
@@ -40,7 +38,7 @@ bucket_definitions:
|
|
|
40
38
|
|
|
41
39
|
await batch.save({
|
|
42
40
|
sourceTable: TEST_TABLE,
|
|
43
|
-
tag:
|
|
41
|
+
tag: SaveOperationTag.INSERT,
|
|
44
42
|
after: {
|
|
45
43
|
id: 't2'
|
|
46
44
|
},
|
|
@@ -49,7 +47,7 @@ bucket_definitions:
|
|
|
49
47
|
|
|
50
48
|
await batch.save({
|
|
51
49
|
sourceTable: TEST_TABLE,
|
|
52
|
-
tag:
|
|
50
|
+
tag: SaveOperationTag.UPDATE,
|
|
53
51
|
after: {
|
|
54
52
|
id: 't2'
|
|
55
53
|
},
|
|
@@ -128,7 +126,7 @@ bucket_definitions:
|
|
|
128
126
|
const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
|
|
129
127
|
await batch.save({
|
|
130
128
|
sourceTable: TEST_TABLE,
|
|
131
|
-
tag:
|
|
129
|
+
tag: SaveOperationTag.INSERT,
|
|
132
130
|
after: {
|
|
133
131
|
id: 't1'
|
|
134
132
|
},
|
|
@@ -137,7 +135,7 @@ bucket_definitions:
|
|
|
137
135
|
|
|
138
136
|
await batch.save({
|
|
139
137
|
sourceTable: TEST_TABLE,
|
|
140
|
-
tag:
|
|
138
|
+
tag: SaveOperationTag.INSERT,
|
|
141
139
|
after: {
|
|
142
140
|
id: 't2'
|
|
143
141
|
},
|
|
@@ -146,7 +144,7 @@ bucket_definitions:
|
|
|
146
144
|
|
|
147
145
|
await batch.save({
|
|
148
146
|
sourceTable: TEST_TABLE,
|
|
149
|
-
tag:
|
|
147
|
+
tag: SaveOperationTag.DELETE,
|
|
150
148
|
before: {
|
|
151
149
|
id: 't1'
|
|
152
150
|
},
|
|
@@ -155,7 +153,7 @@ bucket_definitions:
|
|
|
155
153
|
|
|
156
154
|
await batch.save({
|
|
157
155
|
sourceTable: TEST_TABLE,
|
|
158
|
-
tag:
|
|
156
|
+
tag: SaveOperationTag.UPDATE,
|
|
159
157
|
after: {
|
|
160
158
|
id: 't2'
|
|
161
159
|
},
|
|
@@ -233,7 +231,7 @@ bucket_definitions:
|
|
|
233
231
|
const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
|
|
234
232
|
await batch.save({
|
|
235
233
|
sourceTable: TEST_TABLE,
|
|
236
|
-
tag:
|
|
234
|
+
tag: SaveOperationTag.INSERT,
|
|
237
235
|
after: {
|
|
238
236
|
id: 't1'
|
|
239
237
|
},
|
|
@@ -242,7 +240,7 @@ bucket_definitions:
|
|
|
242
240
|
|
|
243
241
|
await batch.save({
|
|
244
242
|
sourceTable: TEST_TABLE,
|
|
245
|
-
tag:
|
|
243
|
+
tag: SaveOperationTag.INSERT,
|
|
246
244
|
after: {
|
|
247
245
|
id: 't2'
|
|
248
246
|
},
|
|
@@ -251,7 +249,7 @@ bucket_definitions:
|
|
|
251
249
|
|
|
252
250
|
await batch.save({
|
|
253
251
|
sourceTable: TEST_TABLE,
|
|
254
|
-
tag:
|
|
252
|
+
tag: SaveOperationTag.DELETE,
|
|
255
253
|
before: {
|
|
256
254
|
id: 't1'
|
|
257
255
|
},
|
|
@@ -265,7 +263,7 @@ bucket_definitions:
|
|
|
265
263
|
const result2 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
|
|
266
264
|
await batch.save({
|
|
267
265
|
sourceTable: TEST_TABLE,
|
|
268
|
-
tag:
|
|
266
|
+
tag: SaveOperationTag.DELETE,
|
|
269
267
|
before: {
|
|
270
268
|
id: 't2'
|
|
271
269
|
},
|