@powersync/service-core 0.0.0-dev-20241021151922 → 0.0.0-dev-20241021185145

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/CHANGELOG.md +5 -5
  2. package/dist/api/RouteAPI.d.ts +6 -4
  3. package/dist/api/diagnostics.js +3 -1
  4. package/dist/api/diagnostics.js.map +1 -1
  5. package/dist/api/schema.js +2 -2
  6. package/dist/api/schema.js.map +1 -1
  7. package/dist/replication/AbstractReplicationJob.js +2 -2
  8. package/dist/replication/AbstractReplicationJob.js.map +1 -1
  9. package/dist/replication/ReplicationModule.js +3 -0
  10. package/dist/replication/ReplicationModule.js.map +1 -1
  11. package/dist/routes/configure-fastify.js +12 -12
  12. package/dist/routes/configure-fastify.js.map +1 -1
  13. package/dist/routes/configure-rsocket.js +4 -1
  14. package/dist/routes/configure-rsocket.js.map +1 -1
  15. package/dist/routes/endpoints/admin.js.map +1 -1
  16. package/dist/routes/endpoints/checkpointing.js +14 -82
  17. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  18. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  19. package/dist/routes/router.d.ts +8 -1
  20. package/dist/routes/router.js.map +1 -1
  21. package/dist/storage/BucketStorage.d.ts +19 -15
  22. package/dist/storage/BucketStorage.js +6 -0
  23. package/dist/storage/BucketStorage.js.map +1 -1
  24. package/dist/storage/MongoBucketStorage.d.ts +9 -3
  25. package/dist/storage/MongoBucketStorage.js +21 -4
  26. package/dist/storage/MongoBucketStorage.js.map +1 -1
  27. package/dist/storage/StorageEngine.d.ts +5 -1
  28. package/dist/storage/StorageEngine.js +19 -1
  29. package/dist/storage/StorageEngine.js.map +1 -1
  30. package/dist/storage/StorageProvider.d.ts +8 -1
  31. package/dist/storage/mongo/MongoBucketBatch.d.ts +2 -2
  32. package/dist/storage/mongo/MongoBucketBatch.js +1 -0
  33. package/dist/storage/mongo/MongoBucketBatch.js.map +1 -1
  34. package/dist/storage/mongo/MongoStorageProvider.js +2 -1
  35. package/dist/storage/mongo/MongoStorageProvider.js.map +1 -1
  36. package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +3 -13
  37. package/dist/storage/mongo/MongoSyncBucketStorage.js +7 -35
  38. package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -1
  39. package/dist/storage/mongo/MongoWriteCheckpointAPI.d.ts +2 -4
  40. package/dist/storage/mongo/MongoWriteCheckpointAPI.js +7 -13
  41. package/dist/storage/mongo/MongoWriteCheckpointAPI.js.map +1 -1
  42. package/dist/storage/storage-index.d.ts +2 -2
  43. package/dist/storage/storage-index.js +2 -2
  44. package/dist/storage/storage-index.js.map +1 -1
  45. package/dist/storage/{WriteCheckpointAPI.d.ts → write-checkpoint.d.ts} +9 -28
  46. package/dist/storage/{WriteCheckpointAPI.js → write-checkpoint.js} +1 -1
  47. package/dist/storage/write-checkpoint.js.map +1 -0
  48. package/dist/util/config/compound-config-collector.js +1 -2
  49. package/dist/util/config/compound-config-collector.js.map +1 -1
  50. package/dist/util/config/types.d.ts +0 -1
  51. package/dist/util/protocol-types.d.ts +2 -1
  52. package/package.json +5 -5
  53. package/src/api/RouteAPI.ts +7 -4
  54. package/src/api/diagnostics.ts +3 -1
  55. package/src/api/schema.ts +3 -3
  56. package/src/replication/AbstractReplicationJob.ts +2 -2
  57. package/src/replication/ReplicationModule.ts +4 -0
  58. package/src/routes/configure-fastify.ts +16 -17
  59. package/src/routes/configure-rsocket.ts +7 -2
  60. package/src/routes/endpoints/admin.ts +2 -2
  61. package/src/routes/endpoints/checkpointing.ts +1 -8
  62. package/src/routes/endpoints/sync-rules.ts +1 -0
  63. package/src/routes/router.ts +7 -1
  64. package/src/storage/BucketStorage.ts +23 -16
  65. package/src/storage/MongoBucketStorage.ts +39 -9
  66. package/src/storage/StorageEngine.ts +24 -2
  67. package/src/storage/StorageProvider.ts +9 -1
  68. package/src/storage/mongo/MongoBucketBatch.ts +3 -2
  69. package/src/storage/mongo/MongoStorageProvider.ts +2 -1
  70. package/src/storage/mongo/MongoSyncBucketStorage.ts +9 -52
  71. package/src/storage/mongo/MongoWriteCheckpointAPI.ts +8 -16
  72. package/src/storage/storage-index.ts +2 -2
  73. package/src/storage/{WriteCheckpointAPI.ts → write-checkpoint.ts} +12 -30
  74. package/src/util/config/compound-config-collector.ts +1 -2
  75. package/src/util/config/types.ts +0 -1
  76. package/src/util/protocol-types.ts +1 -1
  77. package/test/src/compacting.test.ts +13 -15
  78. package/test/src/data_storage.test.ts +56 -56
  79. package/test/src/sync.test.ts +10 -9
  80. package/tsconfig.tsbuildinfo +1 -1
  81. package/dist/storage/WriteCheckpointAPI.js.map +0 -1
@@ -21,13 +21,20 @@ import {
21
21
  UpdateSyncRulesOptions,
22
22
  WriteCheckpoint
23
23
  } from './BucketStorage.js';
24
- import { PowerSyncMongo, PowerSyncMongoOptions } from './mongo/db.js';
24
+ import { PowerSyncMongo } from './mongo/db.js';
25
25
  import { SyncRuleDocument, SyncRuleState } from './mongo/models.js';
26
26
  import { MongoPersistedSyncRulesContent } from './mongo/MongoPersistedSyncRulesContent.js';
27
27
  import { MongoSyncBucketStorage } from './mongo/MongoSyncBucketStorage.js';
28
+ import { MongoWriteCheckpointAPI } from './mongo/MongoWriteCheckpointAPI.js';
28
29
  import { generateSlotName } from './mongo/util.js';
29
-
30
- export interface MongoBucketStorageOptions extends PowerSyncMongoOptions {}
30
+ import {
31
+ CustomWriteCheckpointOptions,
32
+ DEFAULT_WRITE_CHECKPOINT_MODE,
33
+ LastWriteCheckpointFilters,
34
+ ManagedWriteCheckpointOptions,
35
+ WriteCheckpointAPI,
36
+ WriteCheckpointMode
37
+ } from './write-checkpoint.js';
31
38
 
32
39
  export class MongoBucketStorage
33
40
  extends DisposableObserver<BucketStorageFactoryListener>
@@ -38,6 +45,10 @@ export class MongoBucketStorage
38
45
  // TODO: This is still Postgres specific and needs to be reworked
39
46
  public readonly slot_name_prefix: string;
40
47
 
48
+ readonly write_checkpoint_mode: WriteCheckpointMode;
49
+
50
+ protected readonly writeCheckpointAPI: WriteCheckpointAPI;
51
+
41
52
  private readonly storageCache = new LRUCache<number, MongoSyncBucketStorage>({
42
53
  max: 3,
43
54
  fetchMethod: async (id) => {
@@ -65,6 +76,7 @@ export class MongoBucketStorage
65
76
  db: PowerSyncMongo,
66
77
  options: {
67
78
  slot_name_prefix: string;
79
+ write_checkpoint_mode?: WriteCheckpointMode;
68
80
  }
69
81
  ) {
70
82
  super();
@@ -72,6 +84,11 @@ export class MongoBucketStorage
72
84
  this.db = db;
73
85
  this.session = this.client.startSession();
74
86
  this.slot_name_prefix = options.slot_name_prefix;
87
+ this.write_checkpoint_mode = options.write_checkpoint_mode ?? DEFAULT_WRITE_CHECKPOINT_MODE;
88
+ this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
89
+ db,
90
+ mode: this.write_checkpoint_mode
91
+ });
75
92
  }
76
93
 
77
94
  getInstance(options: PersistedSyncRulesContent): MongoSyncBucketStorage {
@@ -280,6 +297,22 @@ export class MongoBucketStorage
280
297
  });
281
298
  }
282
299
 
300
+ async batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void> {
301
+ return this.writeCheckpointAPI.batchCreateCustomWriteCheckpoints(checkpoints);
302
+ }
303
+
304
+ async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
305
+ return this.writeCheckpointAPI.createCustomWriteCheckpoint(options);
306
+ }
307
+
308
+ async createManagedWriteCheckpoint(options: ManagedWriteCheckpointOptions): Promise<bigint> {
309
+ return this.writeCheckpointAPI.createManagedWriteCheckpoint(options);
310
+ }
311
+
312
+ async lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null> {
313
+ return this.writeCheckpointAPI.lastWriteCheckpoint(filters);
314
+ }
315
+
283
316
  async getActiveCheckpoint(): Promise<ActiveCheckpoint> {
284
317
  const doc = await this.db.sync_rules.findOne(
285
318
  {
@@ -391,7 +424,7 @@ export class MongoBucketStorage
391
424
  }
392
425
  return (await this.storageCache.fetch(doc._id)) ?? null;
393
426
  }
394
- } satisfies ActiveCheckpoint;
427
+ };
395
428
  }
396
429
 
397
430
  /**
@@ -481,7 +514,6 @@ export class MongoBucketStorage
481
514
  if (doc == null) {
482
515
  continue;
483
516
  }
484
-
485
517
  const op = this.makeActiveCheckpoint(doc);
486
518
  // Check for LSN / checkpoint changes - ignore other metadata changes
487
519
  if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
@@ -512,14 +544,12 @@ export class MongoBucketStorage
512
544
  // 1. checkpoint (op_id) changes.
513
545
  // 2. write checkpoint changes for the specific user
514
546
  const bucketStorage = await cp.getBucketStorage();
515
- if (!bucketStorage) {
516
- continue;
517
- }
518
547
 
519
548
  const lsnFilters: Record<string, string> = lsn ? { 1: lsn } : {};
520
549
 
521
- const currentWriteCheckpoint = await bucketStorage.lastWriteCheckpoint({
550
+ const currentWriteCheckpoint = await this.lastWriteCheckpoint({
522
551
  user_id,
552
+ sync_rules_id: bucketStorage?.group_id,
523
553
  heads: {
524
554
  ...lsnFilters
525
555
  }
@@ -1,12 +1,17 @@
1
1
  import { DisposableListener, DisposableObserver, logger } from '@powersync/lib-services-framework';
2
2
  import { ResolvedPowerSyncConfig } from '../util/util-index.js';
3
3
  import { BucketStorageFactory } from './BucketStorage.js';
4
- import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
4
+ import { ActiveStorage, BucketStorageProvider, StorageSettings } from './StorageProvider.js';
5
+ import { DEFAULT_WRITE_CHECKPOINT_MODE } from './write-checkpoint.js';
5
6
 
6
7
  export type StorageEngineOptions = {
7
8
  configuration: ResolvedPowerSyncConfig;
8
9
  };
9
10
 
11
+ export const DEFAULT_STORAGE_SETTINGS: StorageSettings = {
12
+ writeCheckpointMode: DEFAULT_WRITE_CHECKPOINT_MODE
13
+ };
14
+
10
15
  export interface StorageEngineListener extends DisposableListener {
11
16
  storageActivated: (storage: BucketStorageFactory) => void;
12
17
  }
@@ -15,9 +20,11 @@ export class StorageEngine extends DisposableObserver<StorageEngineListener> {
15
20
  // TODO: This will need to revisited when we actually support multiple storage providers.
16
21
  private storageProviders: Map<string, BucketStorageProvider> = new Map();
17
22
  private currentActiveStorage: ActiveStorage | null = null;
23
+ private _activeSettings: StorageSettings;
18
24
 
19
25
  constructor(private options: StorageEngineOptions) {
20
26
  super();
27
+ this._activeSettings = DEFAULT_STORAGE_SETTINGS;
21
28
  }
22
29
 
23
30
  get activeBucketStorage(): BucketStorageFactory {
@@ -32,6 +39,20 @@ export class StorageEngine extends DisposableObserver<StorageEngineListener> {
32
39
  return this.currentActiveStorage;
33
40
  }
34
41
 
42
+ get activeSettings(): StorageSettings {
43
+ return { ...this._activeSettings };
44
+ }
45
+
46
+ updateSettings(settings: Partial<StorageSettings>) {
47
+ if (this.currentActiveStorage) {
48
+ throw new Error(`Storage is already active, settings cannot be modified.`);
49
+ }
50
+ this._activeSettings = {
51
+ ...this._activeSettings,
52
+ ...settings
53
+ };
54
+ }
55
+
35
56
  /**
36
57
  * Register a provider which generates a {@link BucketStorageFactory}
37
58
  * given the matching config specified in the loaded {@link ResolvedPowerSyncConfig}
@@ -44,7 +65,8 @@ export class StorageEngine extends DisposableObserver<StorageEngineListener> {
44
65
  logger.info('Starting Storage Engine...');
45
66
  const { configuration } = this.options;
46
67
  this.currentActiveStorage = await this.storageProviders.get(configuration.storage.type)!.getStorage({
47
- resolvedConfig: configuration
68
+ resolvedConfig: configuration,
69
+ ...this.activeSettings
48
70
  });
49
71
  this.iterateListeners((cb) => cb.storageActivated?.(this.activeBucketStorage));
50
72
  logger.info(`Successfully activated storage: ${configuration.storage.type}.`);
@@ -1,5 +1,6 @@
1
1
  import * as util from '../util/util-index.js';
2
2
  import { BucketStorageFactory } from './BucketStorage.js';
3
+ import { WriteCheckpointMode } from './write-checkpoint.js';
3
4
 
4
5
  export interface ActiveStorage {
5
6
  storage: BucketStorageFactory;
@@ -11,7 +12,14 @@ export interface ActiveStorage {
11
12
  tearDown(): Promise<boolean>;
12
13
  }
13
14
 
14
- export interface GetStorageOptions {
15
+ /**
16
+ * Settings which can be modified by various modules in their initialization.
17
+ */
18
+ export interface StorageSettings {
19
+ writeCheckpointMode: WriteCheckpointMode;
20
+ }
21
+
22
+ export interface GetStorageOptions extends StorageSettings {
15
23
  // TODO: This should just be the storage config. Update once the slot name prefix coupling has been removed from the storage
16
24
  resolvedConfig: util.ResolvedPowerSyncConfig;
17
25
  }
@@ -12,7 +12,7 @@ import {
12
12
  SaveOptions
13
13
  } from '../BucketStorage.js';
14
14
  import { SourceTable } from '../SourceTable.js';
15
- import { BatchedCustomWriteCheckpointOptions, CustomWriteCheckpointOptions } from '../WriteCheckpointAPI.js';
15
+ import { CustomWriteCheckpointOptions } from '../write-checkpoint.js';
16
16
  import { PowerSyncMongo } from './db.js';
17
17
  import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js';
18
18
  import { MongoIdSequence } from './MongoIdSequence.js';
@@ -81,9 +81,10 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
81
81
  this.session = this.client.startSession();
82
82
  this.slot_name = slot_name;
83
83
  this.sync_rules = sync_rules;
84
+ this.batch = new OperationBatch();
84
85
  }
85
86
 
86
- addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void {
87
+ addCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): void {
87
88
  this.write_checkpoint_batch.push({
88
89
  ...checkpoint,
89
90
  sync_rules_id: this.group_id
@@ -19,7 +19,8 @@ export class MongoStorageProvider implements BucketStorageProvider {
19
19
  return {
20
20
  storage: new MongoBucketStorage(database, {
21
21
  // TODO currently need the entire resolved config due to this
22
- slot_name_prefix: resolvedConfig.slot_name_prefix
22
+ slot_name_prefix: resolvedConfig.slot_name_prefix,
23
+ write_checkpoint_mode: options.writeCheckpointMode
23
24
  }),
24
25
  shutDown: () => client.close(),
25
26
  tearDown: () => {
@@ -2,13 +2,13 @@ import { SqliteJsonRow, SqliteJsonValue, SqlSyncRules } from '@powersync/service
2
2
  import * as bson from 'bson';
3
3
  import * as mongo from 'mongodb';
4
4
 
5
- import { DisposableObserver, logger } from '@powersync/lib-services-framework';
6
- import * as timers from 'timers/promises';
5
+ import { DisposableObserver } from '@powersync/lib-services-framework';
7
6
  import * as db from '../../db/db-index.js';
8
7
  import * as util from '../../util/util-index.js';
9
8
  import {
10
9
  BucketDataBatchOptions,
11
10
  BucketStorageBatch,
11
+ Checkpoint,
12
12
  CompactOptions,
13
13
  DEFAULT_DOCUMENT_BATCH_LIMIT,
14
14
  DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES,
@@ -27,19 +27,13 @@ import {
27
27
  import { ChecksumCache, FetchPartialBucketChecksum, PartialChecksum, PartialChecksumMap } from '../ChecksumCache.js';
28
28
  import { MongoBucketStorage } from '../MongoBucketStorage.js';
29
29
  import { SourceTable } from '../SourceTable.js';
30
- import {
31
- BatchedCustomWriteCheckpointOptions,
32
- ManagedWriteCheckpointOptions,
33
- SyncStorageLastWriteCheckpointFilters,
34
- WriteCheckpointAPI,
35
- WriteCheckpointMode
36
- } from '../WriteCheckpointAPI.js';
37
30
  import { PowerSyncMongo } from './db.js';
38
31
  import { BucketDataDocument, BucketDataKey, SourceKey, SyncRuleState } from './models.js';
39
32
  import { MongoBucketBatch } from './MongoBucketBatch.js';
40
33
  import { MongoCompactor } from './MongoCompactor.js';
41
- import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
42
34
  import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, mapOpEntry, readSingleBatch, serializeLookup } from './util.js';
35
+ import { logger } from '@powersync/lib-services-framework';
36
+ import * as timers from 'timers/promises';
43
37
 
44
38
  export class MongoSyncBucketStorage
45
39
  extends DisposableObserver<SyncRulesBucketStorageListener>
@@ -53,53 +47,15 @@ export class MongoSyncBucketStorage
53
47
  });
54
48
 
55
49
  private parsedSyncRulesCache: SqlSyncRules | undefined;
56
- private writeCheckpointAPI: WriteCheckpointAPI;
57
50
 
58
51
  constructor(
59
52
  public readonly factory: MongoBucketStorage,
60
53
  public readonly group_id: number,
61
54
  private readonly sync_rules: PersistedSyncRulesContent,
62
- public readonly slot_name: string,
63
- writeCheckpointMode: WriteCheckpointMode = WriteCheckpointMode.MANAGED
55
+ public readonly slot_name: string
64
56
  ) {
65
57
  super();
66
58
  this.db = factory.db;
67
- this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
68
- db: this.db,
69
- mode: writeCheckpointMode
70
- });
71
- }
72
-
73
- get writeCheckpointMode() {
74
- return this.writeCheckpointAPI.writeCheckpointMode;
75
- }
76
-
77
- setWriteCheckpointMode(mode: WriteCheckpointMode): void {
78
- this.writeCheckpointAPI.setWriteCheckpointMode(mode);
79
- }
80
-
81
- batchCreateCustomWriteCheckpoints(checkpoints: BatchedCustomWriteCheckpointOptions[]): Promise<void> {
82
- return this.writeCheckpointAPI.batchCreateCustomWriteCheckpoints(
83
- checkpoints.map((checkpoint) => ({ ...checkpoint, sync_rules_id: this.group_id }))
84
- );
85
- }
86
-
87
- createCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): Promise<bigint> {
88
- return this.writeCheckpointAPI.createCustomWriteCheckpoint({
89
- ...checkpoint,
90
- sync_rules_id: this.group_id
91
- });
92
- }
93
-
94
- createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint> {
95
- return this.writeCheckpointAPI.createManagedWriteCheckpoint(checkpoint);
96
- }
97
-
98
- lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null> {
99
- return this.writeCheckpointAPI.lastWriteCheckpoint({
100
- ...filters,
101
- sync_rules_id: this.group_id
102
- });
103
59
  }
104
60
 
105
61
  getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules {
@@ -107,15 +63,16 @@ export class MongoSyncBucketStorage
107
63
  return this.parsedSyncRulesCache;
108
64
  }
109
65
 
110
- async getCheckpoint() {
66
+ async getCheckpoint(): Promise<Checkpoint> {
111
67
  const doc = await this.db.sync_rules.findOne(
112
68
  { _id: this.group_id },
113
69
  {
114
- projection: { last_checkpoint: 1 }
70
+ projection: { last_checkpoint: 1, last_checkpoint_lsn: 1 }
115
71
  }
116
72
  );
117
73
  return {
118
- checkpoint: util.timestampToOpId(doc?.last_checkpoint ?? 0n)
74
+ checkpoint: util.timestampToOpId(doc?.last_checkpoint ?? 0n),
75
+ lsn: doc?.last_checkpoint_lsn ?? null
119
76
  };
120
77
  }
121
78
 
@@ -7,7 +7,7 @@ import {
7
7
  ManagedWriteCheckpointOptions,
8
8
  WriteCheckpointAPI,
9
9
  WriteCheckpointMode
10
- } from '../WriteCheckpointAPI.js';
10
+ } from '../write-checkpoint.js';
11
11
  import { PowerSyncMongo } from './db.js';
12
12
 
13
13
  export type MongoCheckpointAPIOptions = {
@@ -17,19 +17,11 @@ export type MongoCheckpointAPIOptions = {
17
17
 
18
18
  export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
19
19
  readonly db: PowerSyncMongo;
20
- private _mode: WriteCheckpointMode;
20
+ readonly mode: WriteCheckpointMode;
21
21
 
22
22
  constructor(options: MongoCheckpointAPIOptions) {
23
23
  this.db = options.db;
24
- this._mode = options.mode;
25
- }
26
-
27
- get writeCheckpointMode() {
28
- return this._mode;
29
- }
30
-
31
- setWriteCheckpointMode(mode: WriteCheckpointMode): void {
32
- this._mode = mode;
24
+ this.mode = options.mode;
33
25
  }
34
26
 
35
27
  async batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void> {
@@ -37,9 +29,9 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
37
29
  }
38
30
 
39
31
  async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
40
- if (this.writeCheckpointMode !== WriteCheckpointMode.CUSTOM) {
32
+ if (this.mode !== WriteCheckpointMode.CUSTOM) {
41
33
  throw new framework.errors.ValidationError(
42
- `Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
34
+ `Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"`
43
35
  );
44
36
  }
45
37
 
@@ -60,9 +52,9 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
60
52
  }
61
53
 
62
54
  async createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint> {
63
- if (this.writeCheckpointMode !== WriteCheckpointMode.MANAGED) {
55
+ if (this.mode !== WriteCheckpointMode.MANAGED) {
64
56
  throw new framework.errors.ValidationError(
65
- `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
57
+ `Creating a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"`
66
58
  );
67
59
  }
68
60
 
@@ -85,7 +77,7 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
85
77
  }
86
78
 
87
79
  async lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null> {
88
- switch (this.writeCheckpointMode) {
80
+ switch (this.mode) {
89
81
  case WriteCheckpointMode.CUSTOM:
90
82
  if (false == 'sync_rules_id' in filters) {
91
83
  throw new framework.errors.ValidationError(`Sync rules ID is required for custom Write Checkpoint filtering`);
@@ -5,7 +5,6 @@ export * from './SourceEntity.js';
5
5
  export * from './SourceTable.js';
6
6
  export * from './StorageEngine.js';
7
7
 
8
- export * from './mongo/config.js';
9
8
  export * from './mongo/db.js';
10
9
  export * from './mongo/models.js';
11
10
  export * from './mongo/MongoBucketBatch.js';
@@ -18,4 +17,5 @@ export * from './mongo/MongoSyncRulesLock.js';
18
17
  export * from './mongo/OperationBatch.js';
19
18
  export * from './mongo/PersistedBatch.js';
20
19
  export * from './mongo/util.js';
21
- export * from './WriteCheckpointAPI.js';
20
+ export * from './mongo/config.js';
21
+ export * from './write-checkpoint.js';
@@ -26,19 +26,19 @@ export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentif
26
26
  sync_rules_id: number;
27
27
  }
28
28
 
29
- export interface BatchedCustomWriteCheckpointOptions extends BaseWriteCheckpointIdentifier {
29
+ export interface CustomWriteCheckpointOptions extends CustomWriteCheckpointFilters {
30
30
  /**
31
31
  * A supplied incrementing Write Checkpoint number
32
32
  */
33
33
  checkpoint: bigint;
34
34
  }
35
35
 
36
- export interface CustomWriteCheckpointOptions extends BatchedCustomWriteCheckpointOptions {
37
- /**
38
- * Sync rules which were active when this checkpoint was created.
39
- */
40
- sync_rules_id: number;
41
- }
36
+ /**
37
+ * Options for creating a custom Write Checkpoint in a batch.
38
+ * A {@link BucketStorageBatch} is already associated with a Sync Rules instance.
39
+ * The `sync_rules_id` is not required here.
40
+ */
41
+ export type BatchedCustomWriteCheckpointOptions = Omit<CustomWriteCheckpointOptions, 'sync_rules_id'>;
42
42
 
43
43
  /**
44
44
  * Managed Write Checkpoints are a mapping of User ID to replication HEAD
@@ -52,33 +52,15 @@ export interface ManagedWriteCheckpointFilters extends BaseWriteCheckpointIdenti
52
52
 
53
53
  export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
54
54
 
55
- export type SyncStorageLastWriteCheckpointFilters = BaseWriteCheckpointIdentifier | ManagedWriteCheckpointFilters;
56
55
  export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
57
56
 
58
- export interface BaseWriteCheckpointAPI {
59
- readonly writeCheckpointMode: WriteCheckpointMode;
60
- setWriteCheckpointMode(mode: WriteCheckpointMode): void;
61
- createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint>;
62
- }
63
-
64
- /**
65
- * Write Checkpoint API to be used in conjunction with a {@link SyncRulesBucketStorage}.
66
- * This storage corresponds with a set of sync rules. These APIs don't require specifying a
67
- * sync rules id.
68
- */
69
- export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
70
- batchCreateCustomWriteCheckpoints(checkpoints: BatchedCustomWriteCheckpointOptions[]): Promise<void>;
71
- createCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): Promise<bigint>;
72
- lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
73
- }
74
-
75
- /**
76
- * Write Checkpoint API which is interfaced directly with the storage layer. This requires
77
- * sync rules identifiers for custom write checkpoints.
78
- */
79
- export interface WriteCheckpointAPI extends BaseWriteCheckpointAPI {
57
+ export interface WriteCheckpointAPI {
80
58
  batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
59
+
81
60
  createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise<bigint>;
61
+
62
+ createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint>;
63
+
82
64
  lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null>;
83
65
  }
84
66
 
@@ -122,8 +122,7 @@ export class CompoundConfigCollector {
122
122
  },
123
123
  // TODO maybe move this out of the connection or something
124
124
  // slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
125
- slot_name_prefix: 'powersync_',
126
- parameters: baseConfig.parameters ?? {}
125
+ slot_name_prefix: 'powersync_'
127
126
  };
128
127
 
129
128
  return config;
@@ -64,5 +64,4 @@ export type ResolvedPowerSyncConfig = {
64
64
 
65
65
  /** Prefix for postgres replication slot names. May eventually be connection-specific. */
66
66
  slot_name_prefix: string;
67
- parameters: Record<string, number | string | boolean | null>;
68
67
  };
@@ -88,7 +88,7 @@ export type StreamingSyncLine =
88
88
  */
89
89
  export type OpId = string;
90
90
 
91
- export interface Checkpoint {
91
+ interface Checkpoint {
92
92
  last_op_id: OpId;
93
93
  write_checkpoint?: OpId;
94
94
  buckets: BucketChecksum[];
@@ -1,11 +1,9 @@
1
+ import { SaveOperationTag } from '@/storage/BucketStorage.js';
1
2
  import { MongoCompactOptions } from '@/storage/mongo/MongoCompactor.js';
2
- import { SqlSyncRules } from '@powersync/service-sync-rules';
3
3
  import { describe, expect, test } from 'vitest';
4
4
  import { validateCompactedBucket } from './bucket_validation.js';
5
5
  import { oneFromAsync } from './stream_utils.js';
6
- import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, rid, testRules, ZERO_LSN } from './util.js';
7
- import { ParseSyncRulesOptions, PersistedSyncRulesContent, StartBatchOptions } from '@/storage/BucketStorage.js';
8
- import { getUuidReplicaIdentityBson } from '@/util/util-index.js';
6
+ import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, rid, testRules } from './util.js';
9
7
 
10
8
  const TEST_TABLE = makeTestTable('test', ['id']);
11
9
 
@@ -31,7 +29,7 @@ bucket_definitions:
31
29
  const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
32
30
  await batch.save({
33
31
  sourceTable: TEST_TABLE,
34
- tag: 'insert',
32
+ tag: SaveOperationTag.INSERT,
35
33
  after: {
36
34
  id: 't1'
37
35
  },
@@ -40,7 +38,7 @@ bucket_definitions:
40
38
 
41
39
  await batch.save({
42
40
  sourceTable: TEST_TABLE,
43
- tag: 'insert',
41
+ tag: SaveOperationTag.INSERT,
44
42
  after: {
45
43
  id: 't2'
46
44
  },
@@ -49,7 +47,7 @@ bucket_definitions:
49
47
 
50
48
  await batch.save({
51
49
  sourceTable: TEST_TABLE,
52
- tag: 'update',
50
+ tag: SaveOperationTag.UPDATE,
53
51
  after: {
54
52
  id: 't2'
55
53
  },
@@ -128,7 +126,7 @@ bucket_definitions:
128
126
  const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
129
127
  await batch.save({
130
128
  sourceTable: TEST_TABLE,
131
- tag: 'insert',
129
+ tag: SaveOperationTag.INSERT,
132
130
  after: {
133
131
  id: 't1'
134
132
  },
@@ -137,7 +135,7 @@ bucket_definitions:
137
135
 
138
136
  await batch.save({
139
137
  sourceTable: TEST_TABLE,
140
- tag: 'insert',
138
+ tag: SaveOperationTag.INSERT,
141
139
  after: {
142
140
  id: 't2'
143
141
  },
@@ -146,7 +144,7 @@ bucket_definitions:
146
144
 
147
145
  await batch.save({
148
146
  sourceTable: TEST_TABLE,
149
- tag: 'delete',
147
+ tag: SaveOperationTag.DELETE,
150
148
  before: {
151
149
  id: 't1'
152
150
  },
@@ -155,7 +153,7 @@ bucket_definitions:
155
153
 
156
154
  await batch.save({
157
155
  sourceTable: TEST_TABLE,
158
- tag: 'update',
156
+ tag: SaveOperationTag.UPDATE,
159
157
  after: {
160
158
  id: 't2'
161
159
  },
@@ -233,7 +231,7 @@ bucket_definitions:
233
231
  const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
234
232
  await batch.save({
235
233
  sourceTable: TEST_TABLE,
236
- tag: 'insert',
234
+ tag: SaveOperationTag.INSERT,
237
235
  after: {
238
236
  id: 't1'
239
237
  },
@@ -242,7 +240,7 @@ bucket_definitions:
242
240
 
243
241
  await batch.save({
244
242
  sourceTable: TEST_TABLE,
245
- tag: 'insert',
243
+ tag: SaveOperationTag.INSERT,
246
244
  after: {
247
245
  id: 't2'
248
246
  },
@@ -251,7 +249,7 @@ bucket_definitions:
251
249
 
252
250
  await batch.save({
253
251
  sourceTable: TEST_TABLE,
254
- tag: 'delete',
252
+ tag: SaveOperationTag.DELETE,
255
253
  before: {
256
254
  id: 't1'
257
255
  },
@@ -265,7 +263,7 @@ bucket_definitions:
265
263
  const result2 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
266
264
  await batch.save({
267
265
  sourceTable: TEST_TABLE,
268
- tag: 'delete',
266
+ tag: SaveOperationTag.DELETE,
269
267
  before: {
270
268
  id: 't2'
271
269
  },