@powersync/service-module-mongodb-storage 0.16.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/dist/storage/MongoBucketStorage.d.ts +6 -4
  3. package/dist/storage/MongoBucketStorage.js +110 -36
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +4 -6
  6. package/dist/storage/implementation/BucketDefinitionMapping.js +3 -3
  7. package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -1
  8. package/dist/storage/implementation/CheckpointState.d.ts +20 -0
  9. package/dist/storage/implementation/CheckpointState.js +31 -0
  10. package/dist/storage/implementation/CheckpointState.js.map +1 -0
  11. package/dist/storage/implementation/MongoBucketBatch.d.ts +33 -22
  12. package/dist/storage/implementation/MongoBucketBatch.js +45 -271
  13. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  14. package/dist/storage/implementation/MongoChecksums.d.ts +2 -1
  15. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  16. package/dist/storage/implementation/MongoCompactor.d.ts +1 -1
  17. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +4 -4
  18. package/dist/storage/implementation/MongoPersistedSyncRules.js +11 -8
  19. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +1 -1
  20. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +19 -5
  21. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +53 -19
  22. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +1 -1
  23. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +21 -10
  24. package/dist/storage/implementation/MongoSyncBucketStorage.js +18 -163
  25. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  26. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +5 -1
  27. package/dist/storage/implementation/MongoSyncRulesLock.js +7 -3
  28. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  29. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +14 -0
  30. package/dist/storage/implementation/SyncRuleStateUpdate.js +36 -0
  31. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -0
  32. package/dist/storage/implementation/common/BucketDataDoc.d.ts +1 -1
  33. package/dist/storage/implementation/common/PersistedBatch.d.ts +2 -2
  34. package/dist/storage/implementation/common/SourceRecordStore.d.ts +1 -2
  35. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +1 -1
  36. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  37. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  38. package/dist/storage/implementation/db.d.ts +10 -2
  39. package/dist/storage/implementation/db.js.map +1 -1
  40. package/dist/storage/implementation/models.d.ts +31 -47
  41. package/dist/storage/implementation/models.js.map +1 -1
  42. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
  43. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +385 -0
  44. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  45. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +1 -1
  46. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +16 -7
  47. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +77 -6
  48. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  49. package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +1 -2
  50. package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
  51. package/dist/storage/implementation/v1/models.d.ts +12 -1
  52. package/dist/storage/implementation/v1/models.js.map +1 -1
  53. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +17 -0
  54. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +429 -0
  55. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  56. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +1 -1
  57. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +1 -2
  58. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +1 -1
  59. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +29 -7
  60. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +117 -16
  61. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  62. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +1 -2
  63. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  64. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +3 -2
  65. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +3 -0
  66. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  67. package/dist/storage/implementation/v3/models.d.ts +61 -3
  68. package/dist/storage/implementation/v3/models.js.map +1 -1
  69. package/package.json +6 -6
  70. package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +1 -1
  71. package/src/storage/MongoBucketStorage.ts +166 -44
  72. package/src/storage/implementation/BucketDefinitionMapping.ts +12 -9
  73. package/src/storage/implementation/CheckpointState.ts +59 -0
  74. package/src/storage/implementation/MongoBucketBatch.ts +81 -355
  75. package/src/storage/implementation/MongoChecksums.ts +2 -1
  76. package/src/storage/implementation/MongoCompactor.ts +1 -1
  77. package/src/storage/implementation/MongoPersistedSyncRules.ts +13 -7
  78. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +69 -24
  79. package/src/storage/implementation/MongoSyncBucketStorage.ts +40 -215
  80. package/src/storage/implementation/MongoSyncRulesLock.ts +9 -3
  81. package/src/storage/implementation/SyncRuleStateUpdate.ts +38 -0
  82. package/src/storage/implementation/common/BucketDataDoc.ts +1 -1
  83. package/src/storage/implementation/common/PersistedBatch.ts +2 -2
  84. package/src/storage/implementation/common/SourceRecordStore.ts +1 -2
  85. package/src/storage/implementation/createMongoSyncBucketStorage.ts +2 -2
  86. package/src/storage/implementation/db.ts +5 -2
  87. package/src/storage/implementation/models.ts +35 -58
  88. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +478 -1
  89. package/src/storage/implementation/v1/MongoCompactorV1.ts +1 -1
  90. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +111 -16
  91. package/src/storage/implementation/v1/PersistedBatchV1.ts +1 -2
  92. package/src/storage/implementation/v1/models.ts +15 -0
  93. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +564 -1
  94. package/src/storage/implementation/v3/MongoCompactorV3.ts +1 -1
  95. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +1 -2
  96. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +150 -22
  97. package/src/storage/implementation/v3/PersistedBatchV3.ts +1 -2
  98. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +7 -2
  99. package/src/storage/implementation/v3/models.ts +70 -2
  100. package/test/src/storage_sync.test.ts +422 -6
  101. package/test/src/storeCurrentData.test.ts +211 -0
  102. package/tsconfig.tsbuildinfo +1 -1
@@ -1,6 +1,6 @@
1
1
  import { storage } from '@powersync/service-core';
2
2
  import { MongoBucketStorage } from '../MongoBucketStorage.js';
3
- import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js';
3
+ import { MongoPersistedSyncRulesContentV1 } from './MongoPersistedSyncRulesContent.js';
4
4
  import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js';
5
5
  import { MongoSyncBucketStorageV1 } from './v1/MongoSyncBucketStorageV1.js';
6
6
  import { MongoSyncBucketStorageV3 } from './v3/MongoSyncBucketStorageV3.js';
@@ -12,7 +12,7 @@ export type { MongoSyncBucketStorage };
12
12
  export function createMongoSyncBucketStorage(
13
13
  factory: MongoBucketStorage,
14
14
  group_id: number,
15
- sync_rules: MongoPersistedSyncRulesContent,
15
+ sync_rules: MongoPersistedSyncRulesContentV1,
16
16
  slot_name: string,
17
17
  writeCheckpointMode: storage.WriteCheckpointMode | undefined,
18
18
  options: MongoSyncBucketStorageOptions
@@ -14,7 +14,7 @@ import {
14
14
  InstanceDocument,
15
15
  SourceTableDocument,
16
16
  StorageConfig,
17
- SyncRuleDocument,
17
+ SyncRuleDocumentBase,
18
18
  WriteCheckpointDocument
19
19
  } from './models.js';
20
20
  import {
@@ -41,7 +41,7 @@ export class PowerSyncMongo {
41
41
  readonly bucket_data: mongo.Collection<BucketDataDocumentV1>;
42
42
  readonly bucket_parameters: mongo.Collection<BucketParameterDocument>;
43
43
  readonly op_id_sequence: mongo.Collection<IdSequenceDocument>;
44
- readonly sync_rules: mongo.Collection<SyncRuleDocument>;
44
+ readonly sync_rules: mongo.Collection<SyncRuleDocumentBase>;
45
45
  readonly source_tables: mongo.Collection<SourceTableDocument>;
46
46
  readonly custom_write_checkpoints: mongo.Collection<CustomWriteCheckpointDocument>;
47
47
  readonly write_checkpoints: mongo.Collection<WriteCheckpointDocument>;
@@ -77,6 +77,9 @@ export class PowerSyncMongo {
77
77
  this.connection_report_events = this.db.collection('connection_report_events');
78
78
  }
79
79
 
80
+ versioned(storageConfig: StorageConfig & { incrementalReprocessing: true }): VersionedPowerSyncMongoV3;
81
+ versioned(storageConfig: StorageConfig & { incrementalReprocessing: false }): VersionedPowerSyncMongoV1;
82
+ versioned(storageConfig: StorageConfig): VersionedPowerSyncMongo;
80
83
  versioned(storageConfig: StorageConfig): VersionedPowerSyncMongo {
81
84
  if (storageConfig.incrementalReprocessing) {
82
85
  return new VersionedPowerSyncMongoV3(this, storageConfig);
@@ -1,9 +1,8 @@
1
1
  import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
2
- import { InternalOpId, SerializedSyncPlan, storage } from '@powersync/service-core';
3
- import { SqliteJsonValue } from '@powersync/service-sync-rules';
2
+ import { InternalOpId, storage } from '@powersync/service-core';
3
+ import { ParameterIndexId, SqliteJsonValue } from '@powersync/service-sync-rules';
4
4
  import { event_types } from '@powersync/service-types';
5
5
  import * as bson from 'bson';
6
- import { ParameterIndexId } from './BucketDefinitionMapping.js';
7
6
  import type { CurrentDataDocument, SourceTableDocumentV1 } from './v1/models.js';
8
7
  import type { CurrentBucketV3, CurrentDataDocumentV3, RecordedLookupV3, SourceTableDocumentV3 } from './v3/models.js';
9
8
 
@@ -157,18 +156,14 @@ export interface IdSequenceDocument {
157
156
  op_id: bigint;
158
157
  }
159
158
 
160
- export interface SyncRuleDocument {
159
+ /**
160
+ * Base for sync_rules collection.
161
+ */
162
+ export interface SyncRuleDocumentBase {
161
163
  _id: number;
162
164
 
163
165
  state: storage.SyncRuleState;
164
166
 
165
- /**
166
- * True if initial snapshot has been replicated.
167
- *
168
- * Can only be false if state == PROCESSING.
169
- */
170
- snapshot_done: boolean;
171
-
172
167
  /**
173
168
  * This is now used for "resumeLsn".
174
169
  *
@@ -182,31 +177,6 @@ export interface SyncRuleDocument {
182
177
  */
183
178
  snapshot_lsn: string | undefined;
184
179
 
185
- /**
186
- * The last consistent checkpoint.
187
- *
188
- * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
189
- */
190
- last_checkpoint: bigint | null;
191
-
192
- /**
193
- * The LSN associated with the last consistent checkpoint.
194
- */
195
- last_checkpoint_lsn: string | null;
196
-
197
- /**
198
- * If set, no new checkpoints may be created < this value.
199
- */
200
- no_checkpoint_before: string | null;
201
-
202
- /**
203
- * Goes together with no_checkpoint_before.
204
- *
205
- * If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
206
- * then the checkpoint must be equal to this keepalive_op.
207
- */
208
- keepalive_op: string | null;
209
-
210
180
  slot_name: string | null;
211
181
 
212
182
  /**
@@ -230,23 +200,6 @@ export interface SyncRuleDocument {
230
200
 
231
201
  last_fatal_error_ts: Date | null;
232
202
 
233
- content: string;
234
- serialized_plan?: SerializedSyncPlan | null;
235
-
236
- /**
237
- * Required for V3+ storage.
238
- */
239
- rule_mapping?: {
240
- /**
241
- * Map of uniqueName -> id, unique per replication stream.
242
- */
243
- definitions: Record<string, string>;
244
- /**
245
- * Map of (lookupName, queryId) -> id, unique per replication stream.
246
- */
247
- parameter_indexes: Record<string, string>;
248
- };
249
-
250
203
  lock?: {
251
204
  id: string;
252
205
  expires_at: Date;
@@ -255,6 +208,35 @@ export interface SyncRuleDocument {
255
208
  storage_version?: number;
256
209
  }
257
210
 
211
+ export interface SyncRuleCheckpointFields<TKeepaliveOp extends string | bigint | null> {
212
+ /**
213
+ * The last consistent checkpoint.
214
+ *
215
+ * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
216
+ */
217
+ last_checkpoint: bigint | null;
218
+
219
+ /**
220
+ * The LSN associated with the last consistent checkpoint.
221
+ */
222
+ last_checkpoint_lsn: string | null;
223
+
224
+ /**
225
+ * If set, no new checkpoints may be created < this value.
226
+ */
227
+ no_checkpoint_before: string | null;
228
+
229
+ /**
230
+ * Goes together with no_checkpoint_before.
231
+ *
232
+ * If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
233
+ * then the checkpoint must be equal to this keepalive_op.
234
+ *
235
+ * This is a string in V1, bigint in V3.
236
+ */
237
+ keepalive_op: TKeepaliveOp;
238
+ }
239
+
258
240
  export interface StorageConfig extends storage.StorageVersionConfig {
259
241
  /**
260
242
  * When true, bucket_data.checksum is guaranteed to be persisted as a Long.
@@ -289,11 +271,6 @@ export interface CheckpointEventDocument {
289
271
  _id: bson.ObjectId;
290
272
  }
291
273
 
292
- export type SyncRuleCheckpointState = Pick<
293
- SyncRuleDocument,
294
- 'last_checkpoint' | 'last_checkpoint_lsn' | '_id' | 'state'
295
- >;
296
-
297
274
  export interface CustomWriteCheckpointDocument {
298
275
  _id: bson.ObjectId;
299
276
  user_id: string;
@@ -1,15 +1,22 @@
1
- import { SourceTable } from '@powersync/service-core';
1
+ import { ReplicationAssertionError } from '@powersync/lib-services-framework';
2
+ import { ColumnDescriptor, SourceTable, storage } from '@powersync/service-core';
3
+ import * as bson from 'bson';
4
+ import { mongoTableId } from '../../../utils/util.js';
5
+ import { calculateCheckpointState } from '../CheckpointState.js';
2
6
  import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
3
7
  import { PersistedBatch } from '../common/PersistedBatch.js';
4
8
  import { SourceRecordStore } from '../common/SourceRecordStore.js';
5
9
  import { PersistedBatchV1 } from './PersistedBatchV1.js';
6
10
  import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js';
7
11
  import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
12
+ import { SourceTableDocumentV1, SyncRuleDocumentV1 } from './models.js';
8
13
 
9
14
  export class MongoBucketBatchV1 extends MongoBucketBatch {
10
15
  declare public readonly db: VersionedPowerSyncMongoV1;
11
16
 
12
17
  private readonly store: SourceRecordStore;
18
+ private needsActivation = true;
19
+ private lastWaitingLogThrottled = 0;
13
20
 
14
21
  constructor(options: MongoBucketBatchOptions) {
15
22
  super(options);
@@ -29,4 +36,474 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
29
36
  protected async cleanupDroppedSourceTables(_tables: SourceTable[]) {
30
37
  // No-op for V1: source records live in a shared collection.
31
38
  }
39
+
40
+ async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
41
+ const syncRules = options.syncRules ?? this.sync_rules;
42
+ const { connection_id, source } = options;
43
+ const { schema, name, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
44
+
45
+ const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
46
+ name: column.name,
47
+ type: column.type,
48
+ type_oid: column.typeId
49
+ }));
50
+
51
+ let result: storage.ResolveTablesResult | null = null;
52
+ await this.db.client.withSession(async (session) => {
53
+ const col = this.db.commonSourceTables(this.group_id);
54
+ const filter: any = {
55
+ group_id: this.group_id,
56
+ connection_id,
57
+ schema_name: schema,
58
+ table_name: name,
59
+ replica_id_columns2: normalizedReplicaIdColumns
60
+ };
61
+ if (objectId != null) {
62
+ filter.relation_id = objectId;
63
+ }
64
+
65
+ let doc = await col.findOne(filter, { session });
66
+ if (doc == null) {
67
+ doc = {
68
+ _id: options.idGenerator ? (options.idGenerator() as bson.ObjectId) : new bson.ObjectId(),
69
+ group_id: this.group_id,
70
+ connection_id,
71
+ relation_id: objectId,
72
+ schema_name: schema,
73
+ table_name: name,
74
+ replica_id_columns: null,
75
+ replica_id_columns2: normalizedReplicaIdColumns,
76
+ snapshot_done: false,
77
+ snapshot_status: undefined
78
+ };
79
+ await col.insertOne(doc, { session });
80
+ }
81
+
82
+ const sourceTable = new storage.SourceTable({
83
+ id: doc._id,
84
+ ref: source,
85
+ objectId,
86
+ replicaIdColumns,
87
+ snapshotComplete: doc.snapshot_done ?? true,
88
+ ...syncRules.getMatchingSources(source)
89
+ });
90
+ sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
91
+ sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
92
+ sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
93
+ sourceTable.storeCurrentData = sendsCompleteRows !== true;
94
+ sourceTable.snapshotStatus =
95
+ doc.snapshot_status == null
96
+ ? undefined
97
+ : {
98
+ lastKey: doc.snapshot_status.last_key?.buffer ?? null,
99
+ totalEstimatedCount: doc.snapshot_status.total_estimated_count,
100
+ replicatedCount: doc.snapshot_status.replicated_count
101
+ };
102
+
103
+ const truncateFilter = [{ schema_name: schema, table_name: name }] as any[];
104
+ if (objectId != null) {
105
+ truncateFilter.push({ relation_id: objectId });
106
+ }
107
+ const truncate = await col
108
+ .find(
109
+ {
110
+ group_id: this.group_id,
111
+ connection_id,
112
+ _id: { $ne: doc._id },
113
+ $or: truncateFilter
114
+ },
115
+ { session }
116
+ )
117
+ .toArray();
118
+ const dropTables = truncate.map((dropDoc) => {
119
+ const ref = {
120
+ connectionTag,
121
+ schema: dropDoc.schema_name,
122
+ name: dropDoc.table_name
123
+ };
124
+ const table = new storage.SourceTable({
125
+ id: dropDoc._id,
126
+ ref,
127
+ objectId: dropDoc.relation_id,
128
+ replicaIdColumns:
129
+ dropDoc.replica_id_columns2?.map(
130
+ (c) => ({ name: c.name, typeId: c.type_oid, type: c.type }) satisfies ColumnDescriptor
131
+ ) ?? [],
132
+ snapshotComplete: dropDoc.snapshot_done ?? true,
133
+ ...syncRules.getMatchingSources(ref)
134
+ });
135
+ table.syncEvent = syncRules.tableTriggersEvent(ref);
136
+ table.syncData = table.bucketDataSources.length > 0;
137
+ table.syncParameters = table.parameterLookupSources.length > 0;
138
+ return table;
139
+ });
140
+
141
+ result = { tables: [sourceTable], dropTables };
142
+ });
143
+
144
+ return result!;
145
+ }
146
+
147
+ async getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null> {
148
+ const doc = (await this.db.commonSourceTables(this.group_id).findOne(
149
+ {
150
+ group_id: this.group_id,
151
+ _id: mongoTableId(table.id)
152
+ },
153
+ { session: this.session }
154
+ )) as SourceTableDocumentV1 | null;
155
+ if (doc == null) {
156
+ return null;
157
+ }
158
+
159
+ const ref = {
160
+ connectionTag: table.ref.connectionTag,
161
+ schema: doc.schema_name,
162
+ name: doc.table_name
163
+ };
164
+ const sourceTable = new storage.SourceTable({
165
+ id: doc._id,
166
+ ref,
167
+ objectId: doc.relation_id,
168
+ replicaIdColumns:
169
+ doc.replica_id_columns2?.map(
170
+ (c) => ({ name: c.name, typeId: c.type_oid, type: c.type }) satisfies ColumnDescriptor
171
+ ) ?? [],
172
+ snapshotComplete: doc.snapshot_done ?? true,
173
+ ...this.sync_rules.getMatchingSources(ref)
174
+ });
175
+ sourceTable.syncEvent = this.sync_rules.tableTriggersEvent(ref);
176
+ sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
177
+ sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
178
+ sourceTable.snapshotStatus =
179
+ doc.snapshot_status == null
180
+ ? undefined
181
+ : {
182
+ lastKey: doc.snapshot_status.last_key?.buffer ?? null,
183
+ totalEstimatedCount: doc.snapshot_status.total_estimated_count,
184
+ replicatedCount: doc.snapshot_status.replicated_count
185
+ };
186
+ return sourceTable;
187
+ }
188
+
189
+ async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult> {
190
+ const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
191
+
192
+ await this.flush(options);
193
+
194
+ const now = new Date();
195
+
196
+ await this.db.write_checkpoints.updateMany(
197
+ {
198
+ processed_at_lsn: null,
199
+ 'lsns.1': { $lte: lsn }
200
+ },
201
+ {
202
+ $set: {
203
+ processed_at_lsn: lsn
204
+ }
205
+ },
206
+ {
207
+ session: this.session
208
+ }
209
+ );
210
+
211
+ const can_checkpoint = {
212
+ $and: [
213
+ { $eq: ['$snapshot_done', true] },
214
+ {
215
+ $or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }]
216
+ },
217
+ {
218
+ $or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }]
219
+ }
220
+ ]
221
+ };
222
+
223
+ const new_keepalive_op = {
224
+ $cond: [
225
+ can_checkpoint,
226
+ { $literal: null },
227
+ {
228
+ $toString: {
229
+ $max: [{ $toLong: '$keepalive_op' }, { $literal: this.persisted_op }, 0n]
230
+ }
231
+ }
232
+ ]
233
+ };
234
+
235
+ const new_last_checkpoint = {
236
+ $cond: [
237
+ can_checkpoint,
238
+ {
239
+ $max: ['$last_checkpoint', { $literal: this.persisted_op }, { $toLong: '$keepalive_op' }, 0n]
240
+ },
241
+ '$last_checkpoint'
242
+ ]
243
+ };
244
+
245
+ const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate(
246
+ { _id: this.group_id },
247
+ [
248
+ {
249
+ $set: {
250
+ _can_checkpoint: can_checkpoint,
251
+ _not_empty: createEmptyCheckpoints
252
+ ? true
253
+ : {
254
+ $or: [
255
+ { $literal: createEmptyCheckpoints },
256
+ { $ne: ['$keepalive_op', new_keepalive_op] },
257
+ { $ne: ['$last_checkpoint', new_last_checkpoint] }
258
+ ]
259
+ }
260
+ }
261
+ },
262
+ {
263
+ $set: {
264
+ last_checkpoint_lsn: {
265
+ $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
266
+ },
267
+ last_checkpoint_ts: {
268
+ $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
269
+ },
270
+ last_keepalive_ts: { $literal: now },
271
+ last_fatal_error: { $literal: null },
272
+ last_fatal_error_ts: { $literal: null },
273
+ keepalive_op: new_keepalive_op,
274
+ last_checkpoint: new_last_checkpoint,
275
+ snapshot_lsn: {
276
+ $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
277
+ }
278
+ }
279
+ },
280
+ {
281
+ $unset: ['_can_checkpoint', '_not_empty']
282
+ }
283
+ ],
284
+ {
285
+ session: this.session,
286
+ returnDocument: 'before',
287
+ projection: {
288
+ snapshot_done: 1,
289
+ last_checkpoint_lsn: 1,
290
+ no_checkpoint_before: 1,
291
+ keepalive_op: 1,
292
+ last_checkpoint: 1
293
+ }
294
+ }
295
+ )) as SyncRuleDocumentV1;
296
+
297
+ if (preUpdateDocument == null) {
298
+ throw new ReplicationAssertionError(
299
+ 'Failed to update checkpoint - no matching sync_rules document for _id: ' + this.group_id
300
+ );
301
+ }
302
+
303
+ const checkpointState = calculateCheckpointState({
304
+ lsn,
305
+ snapshotDone: preUpdateDocument.snapshot_done === true,
306
+ lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn,
307
+ noCheckpointBefore: preUpdateDocument.no_checkpoint_before,
308
+ keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op),
309
+ lastCheckpoint: preUpdateDocument.last_checkpoint,
310
+ persistedOp: this.persisted_op,
311
+ createEmptyCheckpoints
312
+ });
313
+ if (checkpointState.checkpointBlocked) {
314
+ if (Date.now() - this.lastWaitingLogThrottled > 5_000) {
315
+ this.logger.info(
316
+ `Waiting before creating checkpoint, currently at ${lsn} / ${checkpointState.newKeepaliveOp}. Current state: ${JSON.stringify(
317
+ {
318
+ snapshot_done: preUpdateDocument.snapshot_done,
319
+ last_checkpoint_lsn: preUpdateDocument.last_checkpoint_lsn,
320
+ no_checkpoint_before: preUpdateDocument.no_checkpoint_before
321
+ }
322
+ )}`
323
+ );
324
+ this.lastWaitingLogThrottled = Date.now();
325
+ }
326
+ } else {
327
+ if (checkpointState.checkpointCreated) {
328
+ this.logger.debug(`Created checkpoint at ${lsn} / ${checkpointState.newLastCheckpoint}`);
329
+ }
330
+ await this.autoActivate(lsn);
331
+ await this.db.notifyCheckpoint();
332
+ this.persisted_op = null;
333
+ this.last_checkpoint_lsn = lsn;
334
+ if (checkpointState.newLastCheckpoint != null) {
335
+ await this.sourceRecordStore.postCommitCleanup(checkpointState.newLastCheckpoint, this.logger);
336
+ }
337
+ }
338
+
339
+ return {
340
+ checkpointBlocked: checkpointState.checkpointBlocked,
341
+ checkpointCreated: checkpointState.checkpointCreated
342
+ };
343
+ }
344
+
345
+ async keepalive(lsn: string): Promise<storage.CheckpointResult> {
346
+ return await this.commit(lsn, { createEmptyCheckpoints: true });
347
+ }
348
+
349
+ async setResumeLsn(lsn: string): Promise<void> {
350
+ await this.db.sync_rules.updateOne(
351
+ {
352
+ _id: this.group_id
353
+ },
354
+ {
355
+ $set: {
356
+ snapshot_lsn: lsn
357
+ }
358
+ },
359
+ { session: this.session }
360
+ );
361
+ }
362
+
363
+ async markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void> {
364
+ await this.db.sync_rules.updateOne(
365
+ {
366
+ _id: this.group_id
367
+ },
368
+ {
369
+ $set: {
370
+ snapshot_done: true,
371
+ last_keepalive_ts: new Date()
372
+ },
373
+ $max: {
374
+ no_checkpoint_before: no_checkpoint_before_lsn
375
+ }
376
+ },
377
+ { session: this.session }
378
+ );
379
+ }
380
+
381
+ async markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void> {
382
+ await this.withTransaction(async () => {
383
+ // Protect against race conditions
384
+ const count = await this.db.commonSourceTables(this.group_id).countDocuments(
385
+ {
386
+ group_id: this.group_id,
387
+ snapshot_done: false
388
+ },
389
+ { session: this.session }
390
+ );
391
+ if (count > 0) {
392
+ if (options?.throwOnConflict ?? true) {
393
+ throw new ReplicationAssertionError(
394
+ `Cannot mark snapshot done while ${count} source table${count == 1 ? '' : 's'} still require snapshotting`
395
+ );
396
+ } else {
397
+ return;
398
+ }
399
+ }
400
+
401
+ await this.markAllSnapshotDone(no_checkpoint_before_lsn);
402
+ });
403
+ }
404
+
405
+ async markTableSnapshotRequired(_table: storage.SourceTable): Promise<void> {
406
+ await this.db.sync_rules.updateOne(
407
+ {
408
+ _id: this.group_id
409
+ },
410
+ {
411
+ $set: {
412
+ snapshot_done: false
413
+ }
414
+ },
415
+ { session: this.session }
416
+ );
417
+ }
418
+
419
+ async markTableSnapshotDone(
420
+ tables: storage.SourceTable[],
421
+ no_checkpoint_before_lsn?: string
422
+ ): Promise<storage.SourceTable[]> {
423
+ const session = this.session;
424
+ const ids = tables.map((table) => mongoTableId(table.id));
425
+
426
+ await this.withTransaction(async () => {
427
+ await this.db.commonSourceTables(this.group_id).updateMany(
428
+ { _id: { $in: ids } },
429
+ {
430
+ $set: {
431
+ snapshot_done: true
432
+ },
433
+ $unset: {
434
+ snapshot_status: 1
435
+ }
436
+ },
437
+ { session }
438
+ );
439
+
440
+ if (no_checkpoint_before_lsn != null) {
441
+ await this.db.sync_rules.updateOne(
442
+ {
443
+ _id: this.group_id
444
+ },
445
+ {
446
+ $set: {
447
+ last_keepalive_ts: new Date()
448
+ },
449
+ $max: {
450
+ no_checkpoint_before: no_checkpoint_before_lsn
451
+ }
452
+ },
453
+ { session: this.session }
454
+ );
455
+ }
456
+ });
457
+ return tables.map((table) => {
458
+ const copy = table.clone();
459
+ copy.snapshotComplete = true;
460
+ return copy;
461
+ });
462
+ }
463
+
464
+ private async autoActivate(lsn: string): Promise<void> {
465
+ if (!this.needsActivation) {
466
+ return;
467
+ }
468
+
469
+ const session = this.session;
470
+ let activated = false;
471
+ await session.withTransaction(async () => {
472
+ const doc = (await this.db.sync_rules.findOne({ _id: this.group_id }, { session })) as SyncRuleDocumentV1;
473
+ if (doc && doc.state == storage.SyncRuleState.PROCESSING && doc.snapshot_done && doc.last_checkpoint != null) {
474
+ await this.db.sync_rules.updateOne(
475
+ {
476
+ _id: this.group_id
477
+ },
478
+ {
479
+ $set: {
480
+ state: storage.SyncRuleState.ACTIVE
481
+ }
482
+ },
483
+ { session }
484
+ );
485
+
486
+ await this.db.sync_rules.updateMany(
487
+ {
488
+ _id: { $ne: this.group_id },
489
+ state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
490
+ },
491
+ {
492
+ $set: {
493
+ state: storage.SyncRuleState.STOP
494
+ }
495
+ },
496
+ { session }
497
+ );
498
+ activated = true;
499
+ } else if (doc?.state != storage.SyncRuleState.PROCESSING) {
500
+ this.needsActivation = false;
501
+ }
502
+ });
503
+ if (activated) {
504
+ this.logger.info(`Activated new replication stream at ${lsn}`);
505
+ await this.db.notifyCheckpoint();
506
+ this.needsActivation = false;
507
+ }
508
+ }
32
509
  }
@@ -1,7 +1,7 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
2
  import { ReplicationAssertionError } from '@powersync/lib-services-framework';
3
3
  import { storage } from '@powersync/service-core';
4
- import { BucketDefinitionId } from '../BucketDefinitionMapping.js';
4
+ import { BucketDefinitionId } from '@powersync/service-sync-rules';
5
5
  import { SingleBucketStore } from '../common/SingleBucketStore.js';
6
6
  import { BucketStateDocumentBase, LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js';
7
7
  import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js';