@quereus/sync 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +187 -155
- package/dist/src/index.d.ts +3 -5
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3 -5
- package/dist/src/index.js.map +1 -1
- package/dist/src/metadata/keys.d.ts +8 -0
- package/dist/src/metadata/keys.d.ts.map +1 -1
- package/dist/src/metadata/keys.js +11 -0
- package/dist/src/metadata/keys.js.map +1 -1
- package/dist/src/sync/change-applicator.d.ts +44 -0
- package/dist/src/sync/change-applicator.d.ts.map +1 -0
- package/dist/src/sync/change-applicator.js +243 -0
- package/dist/src/sync/change-applicator.js.map +1 -0
- package/dist/src/sync/snapshot-stream.d.ts +26 -0
- package/dist/src/sync/snapshot-stream.d.ts.map +1 -0
- package/dist/src/sync/snapshot-stream.js +375 -0
- package/dist/src/sync/snapshot-stream.js.map +1 -0
- package/dist/src/sync/snapshot.d.ts +17 -0
- package/dist/src/sync/snapshot.d.ts.map +1 -0
- package/dist/src/sync/snapshot.js +166 -0
- package/dist/src/sync/snapshot.js.map +1 -0
- package/dist/src/sync/store-adapter.d.ts.map +1 -1
- package/dist/src/sync/store-adapter.js +3 -2
- package/dist/src/sync/store-adapter.js.map +1 -1
- package/dist/src/sync/sync-context.d.ts +47 -0
- package/dist/src/sync/sync-context.d.ts.map +1 -0
- package/dist/src/sync/sync-context.js +36 -0
- package/dist/src/sync/sync-context.js.map +1 -0
- package/dist/src/sync/sync-manager-impl.d.ts +21 -48
- package/dist/src/sync/sync-manager-impl.d.ts.map +1 -1
- package/dist/src/sync/sync-manager-impl.js +140 -988
- package/dist/src/sync/sync-manager-impl.js.map +1 -1
- package/package.json +4 -4
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* SyncManager implementation.
|
|
3
3
|
*
|
|
4
4
|
* Coordinates CRDT metadata tracking and sync operations.
|
|
5
|
+
* Delegates to focused sub-modules for snapshot, streaming, and change application.
|
|
5
6
|
*/
|
|
6
7
|
import { HLCManager, compareHLC } from '../clock/hlc.js';
|
|
7
8
|
import { generateSiteId, SITE_ID_KEY, serializeSiteIdentity, deserializeSiteIdentity, siteIdEquals, } from '../clock/site.js';
|
|
@@ -10,13 +11,16 @@ import { TombstoneStore, deserializeTombstone } from '../metadata/tombstones.js'
|
|
|
10
11
|
import { PeerStateStore } from '../metadata/peer-state.js';
|
|
11
12
|
import { SchemaMigrationStore, deserializeMigration } from '../metadata/schema-migration.js';
|
|
12
13
|
import { ChangeLogStore } from '../metadata/change-log.js';
|
|
13
|
-
import { SYNC_KEY_PREFIX, buildAllColumnVersionsScanBounds, buildAllTombstonesScanBounds, buildAllSchemaMigrationsScanBounds,
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
import { SYNC_KEY_PREFIX, buildAllColumnVersionsScanBounds, buildAllTombstonesScanBounds, buildAllSchemaMigrationsScanBounds, parseColumnVersionKey, parseTombstoneKey, parseSchemaMigrationKey, } from '../metadata/keys.js';
|
|
15
|
+
import { persistHLCState, persistHLCStateBatch, toError } from './sync-context.js';
|
|
16
|
+
import { applyChanges as applyChangesImpl } from './change-applicator.js';
|
|
17
|
+
import { getSnapshot as getSnapshotImpl, applySnapshot as applySnapshotImpl } from './snapshot.js';
|
|
18
|
+
import { getSnapshotStream as getSnapshotStreamImpl, applySnapshotStream as applySnapshotStreamImpl, getSnapshotCheckpoint as getSnapshotCheckpointImpl, resumeSnapshotStream as resumeSnapshotStreamImpl, } from './snapshot-stream.js';
|
|
18
19
|
/**
|
|
19
20
|
* Implementation of SyncManager.
|
|
21
|
+
*
|
|
22
|
+
* Acts as a coordinator/facade that delegates snapshot, streaming,
|
|
23
|
+
* and change application to focused sub-modules.
|
|
20
24
|
*/
|
|
21
25
|
export class SyncManagerImpl {
|
|
22
26
|
kv;
|
|
@@ -91,160 +95,151 @@ export class SyncManagerImpl {
|
|
|
91
95
|
storeEvents.onSchemaChange((event) => manager.handleSchemaChange(event));
|
|
92
96
|
return manager;
|
|
93
97
|
}
|
|
98
|
+
// ============================================================================
|
|
99
|
+
// Accessors
|
|
100
|
+
// ============================================================================
|
|
94
101
|
getSiteId() {
|
|
95
102
|
return this.hlcManager.getSiteId();
|
|
96
103
|
}
|
|
97
104
|
getCurrentHLC() {
|
|
98
105
|
return this.hlcManager.now();
|
|
99
106
|
}
|
|
107
|
+
// ============================================================================
|
|
108
|
+
// Event Handlers (local store changes)
|
|
109
|
+
// ============================================================================
|
|
100
110
|
/**
|
|
101
111
|
* Handle a data change event from the store.
|
|
102
112
|
* Records CRDT metadata for the change.
|
|
103
|
-
* Skips remote events to prevent duplicate recording.
|
|
104
113
|
*/
|
|
105
114
|
async handleDataChange(event) {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
pk,
|
|
130
|
-
hlc,
|
|
131
|
-
};
|
|
132
|
-
this.pendingChanges.push(change);
|
|
133
|
-
}
|
|
134
|
-
else {
|
|
135
|
-
// Insert or update: record column versions
|
|
136
|
-
if (newRow) {
|
|
137
|
-
await this.recordColumnVersions(batch, schemaName, tableName, pk, oldRow, newRow, hlc);
|
|
115
|
+
try {
|
|
116
|
+
if (event.remote)
|
|
117
|
+
return;
|
|
118
|
+
const hlc = this.hlcManager.tick();
|
|
119
|
+
const { schemaName, tableName, type, oldRow, newRow } = event;
|
|
120
|
+
const pk = event.key ?? event.pk;
|
|
121
|
+
if (!pk) {
|
|
122
|
+
console.warn(`[Sync] Missing primary key for ${schemaName}.${tableName} ${type} event — change not tracked`);
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
const batch = this.kv.batch();
|
|
126
|
+
if (type === 'delete') {
|
|
127
|
+
this.tombstones.setTombstoneBatch(batch, schemaName, tableName, pk, hlc);
|
|
128
|
+
this.changeLog.recordDeletionBatch(batch, hlc, schemaName, tableName, pk);
|
|
129
|
+
await this.columnVersions.deleteRowVersions(schemaName, tableName, pk);
|
|
130
|
+
const change = {
|
|
131
|
+
type: 'delete',
|
|
132
|
+
schema: schemaName,
|
|
133
|
+
table: tableName,
|
|
134
|
+
pk,
|
|
135
|
+
hlc,
|
|
136
|
+
};
|
|
137
|
+
this.pendingChanges.push(change);
|
|
138
138
|
}
|
|
139
|
+
else {
|
|
140
|
+
if (newRow) {
|
|
141
|
+
await this.recordColumnVersions(batch, schemaName, tableName, pk, oldRow, newRow, hlc);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
// Persist HLC state in batch (DRY: uses shared helper)
|
|
145
|
+
persistHLCStateBatch(this, batch);
|
|
146
|
+
await batch.write();
|
|
147
|
+
const changesToEmit = [...this.pendingChanges];
|
|
148
|
+
this.pendingChanges = [];
|
|
149
|
+
this.syncEvents.emitLocalChange({
|
|
150
|
+
transactionId: this.currentTransactionId || crypto.randomUUID(),
|
|
151
|
+
changes: changesToEmit,
|
|
152
|
+
pendingSync: true,
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
catch (error) {
|
|
156
|
+
console.error('[Sync] Error handling data change:', error);
|
|
157
|
+
this.syncEvents.emitSyncStateChange({
|
|
158
|
+
status: 'error',
|
|
159
|
+
error: toError(error),
|
|
160
|
+
});
|
|
139
161
|
}
|
|
140
|
-
// Persist HLC state in batch
|
|
141
|
-
const hlcState = this.hlcManager.getState();
|
|
142
|
-
const hlcBuffer = new Uint8Array(10);
|
|
143
|
-
const hlcView = new DataView(hlcBuffer.buffer);
|
|
144
|
-
hlcView.setBigUint64(0, hlcState.wallTime, false);
|
|
145
|
-
hlcView.setUint16(8, hlcState.counter, false);
|
|
146
|
-
batch.put(SYNC_KEY_PREFIX.HLC_STATE, hlcBuffer);
|
|
147
|
-
await batch.write();
|
|
148
|
-
// Emit local change event with current pending changes, then clear them
|
|
149
|
-
const changesToEmit = [...this.pendingChanges];
|
|
150
|
-
this.pendingChanges = [];
|
|
151
|
-
this.syncEvents.emitLocalChange({
|
|
152
|
-
transactionId: this.currentTransactionId || crypto.randomUUID(),
|
|
153
|
-
changes: changesToEmit,
|
|
154
|
-
pendingSync: true,
|
|
155
|
-
});
|
|
156
162
|
}
|
|
157
163
|
/**
|
|
158
164
|
* Handle a schema change event from the store.
|
|
159
165
|
* Records schema migrations for sync.
|
|
160
|
-
* Skips remote events to prevent duplicate recording.
|
|
161
166
|
*/
|
|
162
167
|
async handleSchemaChange(event) {
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
168
|
+
try {
|
|
169
|
+
if (event.remote)
|
|
170
|
+
return;
|
|
171
|
+
const hlc = this.hlcManager.tick();
|
|
172
|
+
const { type, objectType, schemaName, objectName, ddl } = event;
|
|
173
|
+
let migrationType;
|
|
174
|
+
if (objectType === 'table') {
|
|
175
|
+
switch (type) {
|
|
176
|
+
case 'create':
|
|
177
|
+
migrationType = 'create_table';
|
|
178
|
+
break;
|
|
179
|
+
case 'drop':
|
|
180
|
+
migrationType = 'drop_table';
|
|
181
|
+
break;
|
|
182
|
+
case 'alter':
|
|
183
|
+
migrationType = 'alter_column';
|
|
184
|
+
break;
|
|
185
|
+
default: return;
|
|
186
|
+
}
|
|
182
187
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
188
|
+
else if (objectType === 'index') {
|
|
189
|
+
switch (type) {
|
|
190
|
+
case 'create':
|
|
191
|
+
migrationType = 'add_index';
|
|
192
|
+
break;
|
|
193
|
+
case 'drop':
|
|
194
|
+
migrationType = 'drop_index';
|
|
195
|
+
break;
|
|
196
|
+
default: return;
|
|
197
|
+
}
|
|
193
198
|
}
|
|
199
|
+
else {
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
const currentVersion = await this.schemaMigrations.getCurrentVersion(schemaName, objectName);
|
|
203
|
+
const newVersion = currentVersion + 1;
|
|
204
|
+
await this.schemaMigrations.recordMigration(schemaName, objectName, {
|
|
205
|
+
type: migrationType,
|
|
206
|
+
ddl: ddl || '',
|
|
207
|
+
hlc,
|
|
208
|
+
schemaVersion: newVersion,
|
|
209
|
+
});
|
|
210
|
+
// Persist HLC state (DRY: uses shared helper)
|
|
211
|
+
await persistHLCState(this);
|
|
212
|
+
this.syncEvents.emitLocalChange({
|
|
213
|
+
transactionId: crypto.randomUUID(),
|
|
214
|
+
changes: [],
|
|
215
|
+
pendingSync: true,
|
|
216
|
+
});
|
|
194
217
|
}
|
|
195
|
-
|
|
196
|
-
|
|
218
|
+
catch (error) {
|
|
219
|
+
console.error('[Sync] Error handling schema change:', error);
|
|
220
|
+
this.syncEvents.emitSyncStateChange({
|
|
221
|
+
status: 'error',
|
|
222
|
+
error: toError(error),
|
|
223
|
+
});
|
|
197
224
|
}
|
|
198
|
-
// Get next schema version for this table
|
|
199
|
-
const currentVersion = await this.schemaMigrations.getCurrentVersion(schemaName, objectName);
|
|
200
|
-
const newVersion = currentVersion + 1;
|
|
201
|
-
// Record the migration
|
|
202
|
-
await this.schemaMigrations.recordMigration(schemaName, objectName, {
|
|
203
|
-
type: migrationType,
|
|
204
|
-
ddl: ddl || '',
|
|
205
|
-
hlc,
|
|
206
|
-
schemaVersion: newVersion,
|
|
207
|
-
});
|
|
208
|
-
// Persist HLC state
|
|
209
|
-
const hlcState = this.hlcManager.getState();
|
|
210
|
-
const hlcBuffer = new Uint8Array(10);
|
|
211
|
-
const hlcView = new DataView(hlcBuffer.buffer);
|
|
212
|
-
hlcView.setBigUint64(0, hlcState.wallTime, false);
|
|
213
|
-
hlcView.setUint16(8, hlcState.counter, false);
|
|
214
|
-
await this.kv.put(SYNC_KEY_PREFIX.HLC_STATE, hlcBuffer);
|
|
215
|
-
// Emit local change event for the schema migration
|
|
216
|
-
this.syncEvents.emitLocalChange({
|
|
217
|
-
transactionId: crypto.randomUUID(),
|
|
218
|
-
changes: [],
|
|
219
|
-
pendingSync: true,
|
|
220
|
-
});
|
|
221
225
|
}
|
|
222
226
|
async recordColumnVersions(batch, schemaName, tableName, pk, oldRow, newRow, hlc) {
|
|
223
|
-
// Try to get actual column names from schema
|
|
224
227
|
const tableSchema = this.getTableSchema?.(schemaName, tableName);
|
|
225
228
|
const columnNames = tableSchema?.columns?.map(c => c.name);
|
|
226
|
-
|
|
227
|
-
if (!tableSchema) {
|
|
229
|
+
if (!tableSchema && this.getTableSchema) {
|
|
228
230
|
console.warn(`[Sync] No table schema found for ${schemaName}.${tableName} - using fallback column names`);
|
|
229
231
|
}
|
|
230
|
-
// For each column that changed, record the new version
|
|
231
232
|
for (let i = 0; i < newRow.length; i++) {
|
|
232
233
|
const oldValue = oldRow?.[i];
|
|
233
234
|
const newValue = newRow[i];
|
|
234
|
-
// Only record if value changed (or it's an insert)
|
|
235
235
|
if (!oldRow || oldValue !== newValue) {
|
|
236
|
-
// Use actual column name if available, otherwise fall back to index-based
|
|
237
236
|
const column = columnNames?.[i] ?? `col_${i}`;
|
|
238
|
-
// Look up old column version to delete stale change log entry
|
|
239
237
|
const oldVersion = await this.columnVersions.getColumnVersion(schemaName, tableName, pk, column);
|
|
240
|
-
// Delete old change log entry if exists (keeps change log as a true secondary index)
|
|
241
238
|
if (oldVersion) {
|
|
242
239
|
this.changeLog.deleteEntryBatch(batch, oldVersion.hlc, 'column', schemaName, tableName, pk, column);
|
|
243
240
|
}
|
|
244
|
-
// Record new column version
|
|
245
241
|
const version = { hlc, value: newValue };
|
|
246
242
|
this.columnVersions.setColumnVersionBatch(batch, schemaName, tableName, pk, column, version);
|
|
247
|
-
// Record in change log for efficient delta queries
|
|
248
243
|
this.changeLog.recordColumnChangeBatch(batch, hlc, schemaName, tableName, pk, column);
|
|
249
244
|
const change = {
|
|
250
245
|
type: 'column',
|
|
@@ -259,27 +254,19 @@ export class SyncManagerImpl {
|
|
|
259
254
|
}
|
|
260
255
|
}
|
|
261
256
|
}
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
const view = new DataView(buffer.buffer);
|
|
266
|
-
view.setBigUint64(0, state.wallTime, false);
|
|
267
|
-
view.setUint16(8, state.counter, false);
|
|
268
|
-
await this.kv.put(SYNC_KEY_PREFIX.HLC_STATE, buffer);
|
|
269
|
-
}
|
|
257
|
+
// ============================================================================
|
|
258
|
+
// Delta Sync API
|
|
259
|
+
// ============================================================================
|
|
270
260
|
async getChangesSince(peerSiteId, sinceHLC) {
|
|
271
261
|
const changes = [];
|
|
272
262
|
if (sinceHLC) {
|
|
273
|
-
// Use change log for efficient delta query
|
|
274
263
|
for await (const logEntry of this.changeLog.getChangesSince(sinceHLC)) {
|
|
275
|
-
// Don't include changes from the requesting peer
|
|
276
264
|
if (siteIdEquals(logEntry.hlc.siteId, peerSiteId))
|
|
277
265
|
continue;
|
|
278
266
|
if (logEntry.entryType === 'column') {
|
|
279
|
-
// Look up the column version to get the value
|
|
280
267
|
const cv = await this.columnVersions.getColumnVersion(logEntry.schema, logEntry.table, logEntry.pk, logEntry.column);
|
|
281
268
|
if (!cv)
|
|
282
|
-
continue;
|
|
269
|
+
continue;
|
|
283
270
|
const columnChange = {
|
|
284
271
|
type: 'column',
|
|
285
272
|
schema: logEntry.schema,
|
|
@@ -292,10 +279,9 @@ export class SyncManagerImpl {
|
|
|
292
279
|
changes.push(columnChange);
|
|
293
280
|
}
|
|
294
281
|
else {
|
|
295
|
-
// Look up tombstone for the deletion
|
|
296
282
|
const tombstone = await this.tombstones.getTombstone(logEntry.schema, logEntry.table, logEntry.pk);
|
|
297
283
|
if (!tombstone)
|
|
298
|
-
continue;
|
|
284
|
+
continue;
|
|
299
285
|
const deletion = {
|
|
300
286
|
type: 'delete',
|
|
301
287
|
schema: logEntry.schema,
|
|
@@ -308,10 +294,9 @@ export class SyncManagerImpl {
|
|
|
308
294
|
}
|
|
309
295
|
}
|
|
310
296
|
else {
|
|
311
|
-
// No sinceHLC - need all changes (full scan fallback)
|
|
312
297
|
await this.collectAllChanges(peerSiteId, changes);
|
|
313
298
|
}
|
|
314
|
-
// Collect schema migrations
|
|
299
|
+
// Collect schema migrations
|
|
315
300
|
const schemaMigrations = [];
|
|
316
301
|
const smBounds = buildAllSchemaMigrationsScanBounds();
|
|
317
302
|
for await (const entry of this.kv.iterate(smBounds)) {
|
|
@@ -319,10 +304,8 @@ export class SyncManagerImpl {
|
|
|
319
304
|
if (!parsed)
|
|
320
305
|
continue;
|
|
321
306
|
const migration = deserializeMigration(entry.value);
|
|
322
|
-
// Filter by HLC if provided
|
|
323
307
|
if (sinceHLC && compareHLC(migration.hlc, sinceHLC) <= 0)
|
|
324
308
|
continue;
|
|
325
|
-
// Don't include changes from the requesting peer
|
|
326
309
|
if (siteIdEquals(migration.hlc.siteId, peerSiteId))
|
|
327
310
|
continue;
|
|
328
311
|
schemaMigrations.push({
|
|
@@ -334,14 +317,10 @@ export class SyncManagerImpl {
|
|
|
334
317
|
schemaVersion: migration.schemaVersion,
|
|
335
318
|
});
|
|
336
319
|
}
|
|
337
|
-
// If no changes, return empty array
|
|
338
320
|
if (changes.length === 0 && schemaMigrations.length === 0) {
|
|
339
321
|
return [];
|
|
340
322
|
}
|
|
341
|
-
// Changes from change log are already in HLC order
|
|
342
|
-
// Schema migrations need sorting
|
|
343
323
|
schemaMigrations.sort((a, b) => compareHLC(a.hlc, b.hlc));
|
|
344
|
-
// Batch changes up to config.batchSize
|
|
345
324
|
const result = [];
|
|
346
325
|
for (let i = 0; i < changes.length; i += this.config.batchSize) {
|
|
347
326
|
const batch = changes.slice(i, i + this.config.batchSize);
|
|
@@ -351,10 +330,9 @@ export class SyncManagerImpl {
|
|
|
351
330
|
transactionId: crypto.randomUUID(),
|
|
352
331
|
hlc: maxHLC,
|
|
353
332
|
changes: batch,
|
|
354
|
-
schemaMigrations: i === 0 ? schemaMigrations : [],
|
|
333
|
+
schemaMigrations: i === 0 ? schemaMigrations : [],
|
|
355
334
|
});
|
|
356
335
|
}
|
|
357
|
-
// If no data changes but we have schema migrations, create a changeset for them
|
|
358
336
|
if (result.length === 0 && schemaMigrations.length > 0) {
|
|
359
337
|
const maxHLC = schemaMigrations.reduce((max, m) => compareHLC(m.hlc, max) > 0 ? m.hlc : max, schemaMigrations[0].hlc);
|
|
360
338
|
result.push({
|
|
@@ -367,18 +345,13 @@ export class SyncManagerImpl {
|
|
|
367
345
|
}
|
|
368
346
|
return result;
|
|
369
347
|
}
|
|
370
|
-
/**
|
|
371
|
-
* Fallback: collect all changes when no sinceHLC is provided.
|
|
372
|
-
*/
|
|
373
348
|
async collectAllChanges(peerSiteId, changes) {
|
|
374
|
-
// Collect all column changes
|
|
375
349
|
const cvBounds = buildAllColumnVersionsScanBounds();
|
|
376
350
|
for await (const entry of this.kv.iterate(cvBounds)) {
|
|
377
351
|
const parsed = parseColumnVersionKey(entry.key);
|
|
378
352
|
if (!parsed)
|
|
379
353
|
continue;
|
|
380
354
|
const cv = deserializeColumnVersion(entry.value);
|
|
381
|
-
// Don't include changes from the requesting peer
|
|
382
355
|
if (siteIdEquals(cv.hlc.siteId, peerSiteId))
|
|
383
356
|
continue;
|
|
384
357
|
const columnChange = {
|
|
@@ -392,14 +365,12 @@ export class SyncManagerImpl {
|
|
|
392
365
|
};
|
|
393
366
|
changes.push(columnChange);
|
|
394
367
|
}
|
|
395
|
-
// Collect all deletions (tombstones)
|
|
396
368
|
const tbBounds = buildAllTombstonesScanBounds();
|
|
397
369
|
for await (const entry of this.kv.iterate(tbBounds)) {
|
|
398
370
|
const parsed = parseTombstoneKey(entry.key);
|
|
399
371
|
if (!parsed)
|
|
400
372
|
continue;
|
|
401
373
|
const tombstone = deserializeTombstone(entry.value);
|
|
402
|
-
// Don't include changes from the requesting peer
|
|
403
374
|
if (siteIdEquals(tombstone.hlc.siteId, peerSiteId))
|
|
404
375
|
continue;
|
|
405
376
|
const deletion = {
|
|
@@ -411,418 +382,39 @@ export class SyncManagerImpl {
|
|
|
411
382
|
};
|
|
412
383
|
changes.push(deletion);
|
|
413
384
|
}
|
|
414
|
-
// Sort by HLC for consistent ordering
|
|
415
385
|
changes.sort((a, b) => compareHLC(a.hlc, b.hlc));
|
|
416
386
|
}
|
|
387
|
+
// ============================================================================
|
|
388
|
+
// Delegated: Change Application
|
|
389
|
+
// ============================================================================
|
|
417
390
|
async applyChanges(changes) {
|
|
418
|
-
|
|
419
|
-
let skipped = 0;
|
|
420
|
-
let conflicts = 0;
|
|
421
|
-
// Collect changes to apply to the store (grouped by row for column merging)
|
|
422
|
-
const dataChangesToApply = [];
|
|
423
|
-
const schemaChangesToApply = [];
|
|
424
|
-
const appliedChanges = [];
|
|
425
|
-
// Track resolved changes for the commit phase (CRDT metadata)
|
|
426
|
-
const resolvedDataChanges = [];
|
|
427
|
-
// Track schema migrations that will be applied (for metadata commit)
|
|
428
|
-
const pendingSchemaMigrations = [];
|
|
429
|
-
// PHASE 1: Resolve all changes (no writes yet)
|
|
430
|
-
for (const changeSet of changes) {
|
|
431
|
-
// Update our clock with the remote HLC
|
|
432
|
-
this.hlcManager.receive(changeSet.hlc);
|
|
433
|
-
// Process schema migrations first (DDL before DML)
|
|
434
|
-
for (const migration of changeSet.schemaMigrations) {
|
|
435
|
-
// Use the incoming schemaVersion if provided, otherwise calculate next version
|
|
436
|
-
const schemaVersion = migration.schemaVersion ??
|
|
437
|
-
(await this.schemaMigrations.getCurrentVersion(migration.schema, migration.table)) + 1;
|
|
438
|
-
// Check if we've already recorded this migration
|
|
439
|
-
const existingMigration = await this.schemaMigrations.getMigration(migration.schema, migration.table, schemaVersion);
|
|
440
|
-
if (existingMigration) {
|
|
441
|
-
// Already have this migration version - skip
|
|
442
|
-
// (HLC comparison for first-writer-wins is done via checkConflict if needed)
|
|
443
|
-
if (compareHLC(migration.hlc, existingMigration.hlc) <= 0) {
|
|
444
|
-
skipped++;
|
|
445
|
-
continue;
|
|
446
|
-
}
|
|
447
|
-
}
|
|
448
|
-
// Queue schema change for application (don't record metadata yet)
|
|
449
|
-
schemaChangesToApply.push({
|
|
450
|
-
type: migration.type,
|
|
451
|
-
schema: migration.schema,
|
|
452
|
-
table: migration.table,
|
|
453
|
-
ddl: migration.ddl,
|
|
454
|
-
});
|
|
455
|
-
pendingSchemaMigrations.push({ migration, schemaVersion });
|
|
456
|
-
applied++;
|
|
457
|
-
}
|
|
458
|
-
// Resolve data changes (no metadata writes)
|
|
459
|
-
for (const change of changeSet.changes) {
|
|
460
|
-
const resolved = await this.resolveChange(change, changeSet.siteId);
|
|
461
|
-
if (resolved.outcome === 'applied') {
|
|
462
|
-
applied++;
|
|
463
|
-
appliedChanges.push({ change, siteId: changeSet.siteId });
|
|
464
|
-
resolvedDataChanges.push(resolved);
|
|
465
|
-
if (resolved.dataChange) {
|
|
466
|
-
dataChangesToApply.push(resolved.dataChange);
|
|
467
|
-
}
|
|
468
|
-
}
|
|
469
|
-
else if (resolved.outcome === 'skipped') {
|
|
470
|
-
skipped++;
|
|
471
|
-
}
|
|
472
|
-
else if (resolved.outcome === 'conflict') {
|
|
473
|
-
conflicts++;
|
|
474
|
-
}
|
|
475
|
-
}
|
|
476
|
-
}
|
|
477
|
-
// PHASE 2: Apply data and schema changes to the store via callback
|
|
478
|
-
// This happens BEFORE writing CRDT metadata for crash safety
|
|
479
|
-
if (this.applyToStore && (dataChangesToApply.length > 0 || schemaChangesToApply.length > 0)) {
|
|
480
|
-
await this.applyToStore(dataChangesToApply, schemaChangesToApply, { remote: true });
|
|
481
|
-
}
|
|
482
|
-
// PHASE 3: Commit CRDT metadata (after data is safely written)
|
|
483
|
-
// If crash occurs here, re-sync will re-apply same changes (idempotent)
|
|
484
|
-
await this.commitChangeMetadata(resolvedDataChanges);
|
|
485
|
-
// Commit schema migration metadata
|
|
486
|
-
for (const { migration, schemaVersion } of pendingSchemaMigrations) {
|
|
487
|
-
await this.schemaMigrations.recordMigration(migration.schema, migration.table, {
|
|
488
|
-
type: migration.type,
|
|
489
|
-
ddl: migration.ddl,
|
|
490
|
-
hlc: migration.hlc,
|
|
491
|
-
schemaVersion,
|
|
492
|
-
});
|
|
493
|
-
}
|
|
494
|
-
// Emit a single remote change event with all applied changes for UI reactivity
|
|
495
|
-
if (appliedChanges.length > 0) {
|
|
496
|
-
// Group by siteId to emit one event per originating site
|
|
497
|
-
const changesBySite = new Map();
|
|
498
|
-
for (const { change, siteId } of appliedChanges) {
|
|
499
|
-
const siteKey = Array.from(siteId).join(',');
|
|
500
|
-
const siteChanges = changesBySite.get(siteKey);
|
|
501
|
-
if (siteChanges) {
|
|
502
|
-
siteChanges.push(change);
|
|
503
|
-
}
|
|
504
|
-
else {
|
|
505
|
-
changesBySite.set(siteKey, [change]);
|
|
506
|
-
}
|
|
507
|
-
}
|
|
508
|
-
const appliedAt = this.hlcManager.now();
|
|
509
|
-
for (const [siteKey, siteChanges] of changesBySite) {
|
|
510
|
-
const siteIdBytes = new Uint8Array(siteKey.split(',').map(Number));
|
|
511
|
-
this.syncEvents.emitRemoteChange({
|
|
512
|
-
siteId: siteIdBytes,
|
|
513
|
-
transactionId: crypto.randomUUID(),
|
|
514
|
-
changes: siteChanges,
|
|
515
|
-
appliedAt,
|
|
516
|
-
});
|
|
517
|
-
}
|
|
518
|
-
}
|
|
519
|
-
await this.persistHLCState();
|
|
520
|
-
return {
|
|
521
|
-
applied,
|
|
522
|
-
skipped,
|
|
523
|
-
conflicts,
|
|
524
|
-
transactions: changes.length,
|
|
525
|
-
};
|
|
526
|
-
}
|
|
527
|
-
/**
|
|
528
|
-
* Resolve CRDT conflicts for a change WITHOUT writing metadata.
|
|
529
|
-
* This is phase 1 of the two-phase apply: resolve first, then after
|
|
530
|
-
* data is written to store, call commitChangeMetadata to persist CRDT state.
|
|
531
|
-
*
|
|
532
|
-
* Write order (for crash safety):
|
|
533
|
-
* 1. Resolve all changes (this method) - no writes
|
|
534
|
-
* 2. Apply data to store (applyToStore callback)
|
|
535
|
-
* 3. Commit CRDT metadata (commitChangeMetadata)
|
|
536
|
-
*
|
|
537
|
-
* If crash occurs after data but before metadata, re-sync will re-apply
|
|
538
|
-
* the same changes. Since CRDT operations are idempotent (same HLC → same
|
|
539
|
-
* LWW outcome), this is safe.
|
|
540
|
-
*/
|
|
541
|
-
async resolveChange(change, _remoteSiteId) {
|
|
542
|
-
// Skip changes that originated from ourselves (echo prevention).
|
|
543
|
-
// This can happen when a peer re-sends changes it received from us,
|
|
544
|
-
// or when changes propagate through a coordinator back to the originator.
|
|
545
|
-
if (siteIdEquals(change.hlc.siteId, this.getSiteId())) {
|
|
546
|
-
return { outcome: 'skipped', change };
|
|
547
|
-
}
|
|
548
|
-
if (change.type === 'delete') {
|
|
549
|
-
// Check if we should apply this deletion
|
|
550
|
-
const existingTombstone = await this.tombstones.getTombstone(change.schema, change.table, change.pk);
|
|
551
|
-
if (existingTombstone && compareHLC(change.hlc, existingTombstone.hlc) <= 0) {
|
|
552
|
-
return { outcome: 'skipped', change };
|
|
553
|
-
}
|
|
554
|
-
// Will apply - return the data change (metadata written in commit phase)
|
|
555
|
-
return {
|
|
556
|
-
outcome: 'applied',
|
|
557
|
-
change,
|
|
558
|
-
dataChange: {
|
|
559
|
-
type: 'delete',
|
|
560
|
-
schema: change.schema,
|
|
561
|
-
table: change.table,
|
|
562
|
-
pk: change.pk,
|
|
563
|
-
},
|
|
564
|
-
};
|
|
565
|
-
}
|
|
566
|
-
else {
|
|
567
|
-
// Column change
|
|
568
|
-
const shouldApply = await this.columnVersions.shouldApplyWrite(change.schema, change.table, change.pk, change.column, change.hlc);
|
|
569
|
-
if (!shouldApply) {
|
|
570
|
-
// Local version is newer - this is a conflict where local wins
|
|
571
|
-
const localVersion = await this.columnVersions.getColumnVersion(change.schema, change.table, change.pk, change.column);
|
|
572
|
-
if (localVersion) {
|
|
573
|
-
const conflictEvent = {
|
|
574
|
-
table: change.table,
|
|
575
|
-
pk: change.pk,
|
|
576
|
-
column: change.column,
|
|
577
|
-
localValue: localVersion.value,
|
|
578
|
-
remoteValue: change.value,
|
|
579
|
-
winner: 'local',
|
|
580
|
-
winningHLC: localVersion.hlc,
|
|
581
|
-
};
|
|
582
|
-
this.syncEvents.emitConflictResolved(conflictEvent);
|
|
583
|
-
}
|
|
584
|
-
return { outcome: 'conflict', change };
|
|
585
|
-
}
|
|
586
|
-
// Check for tombstone blocking
|
|
587
|
-
const isBlocked = await this.tombstones.isDeletedAndBlocking(change.schema, change.table, change.pk, change.hlc, this.config.allowResurrection);
|
|
588
|
-
if (isBlocked) {
|
|
589
|
-
return { outcome: 'skipped', change };
|
|
590
|
-
}
|
|
591
|
-
// Get old column version for change log cleanup (done in commit phase)
|
|
592
|
-
const oldColumnVersion = await this.columnVersions.getColumnVersion(change.schema, change.table, change.pk, change.column) ?? undefined;
|
|
593
|
-
// Will apply - return the data change (metadata written in commit phase)
|
|
594
|
-
return {
|
|
595
|
-
outcome: 'applied',
|
|
596
|
-
change,
|
|
597
|
-
oldColumnVersion,
|
|
598
|
-
dataChange: {
|
|
599
|
-
type: 'update', // Column changes are updates (or inserts handled by store)
|
|
600
|
-
schema: change.schema,
|
|
601
|
-
table: change.table,
|
|
602
|
-
pk: change.pk,
|
|
603
|
-
columns: { [change.column]: change.value },
|
|
604
|
-
},
|
|
605
|
-
};
|
|
606
|
-
}
|
|
607
|
-
}
|
|
608
|
-
/**
|
|
609
|
-
* Commit CRDT metadata for resolved changes.
|
|
610
|
-
* This is phase 2 of the two-phase apply: called AFTER data is written to store.
|
|
611
|
-
*/
|
|
612
|
-
async commitChangeMetadata(resolvedChanges) {
|
|
613
|
-
if (resolvedChanges.length === 0)
|
|
614
|
-
return;
|
|
615
|
-
const batch = this.kv.batch();
|
|
616
|
-
for (const resolved of resolvedChanges) {
|
|
617
|
-
if (resolved.outcome !== 'applied')
|
|
618
|
-
continue;
|
|
619
|
-
const change = resolved.change;
|
|
620
|
-
if (change.type === 'delete') {
|
|
621
|
-
// Record tombstone
|
|
622
|
-
this.tombstones.setTombstoneBatch(batch, change.schema, change.table, change.pk, change.hlc);
|
|
623
|
-
// Record in change log for delta sync
|
|
624
|
-
this.changeLog.recordDeletionBatch(batch, change.hlc, change.schema, change.table, change.pk);
|
|
625
|
-
// Note: column versions deletion is done outside the batch since it requires iteration
|
|
626
|
-
}
|
|
627
|
-
else {
|
|
628
|
-
// Column change
|
|
629
|
-
// Delete old change log entry if exists (keeps change log as a true secondary index)
|
|
630
|
-
if (resolved.oldColumnVersion) {
|
|
631
|
-
this.changeLog.deleteEntryBatch(batch, resolved.oldColumnVersion.hlc, 'column', change.schema, change.table, change.pk, change.column);
|
|
632
|
-
}
|
|
633
|
-
// Record the column version
|
|
634
|
-
this.columnVersions.setColumnVersionBatch(batch, change.schema, change.table, change.pk, change.column, { hlc: change.hlc, value: change.value });
|
|
635
|
-
// Record in change log for delta sync
|
|
636
|
-
this.changeLog.recordColumnChangeBatch(batch, change.hlc, change.schema, change.table, change.pk, change.column);
|
|
637
|
-
}
|
|
638
|
-
}
|
|
639
|
-
await batch.write();
|
|
640
|
-
// Handle column version deletions for delete operations (requires async iteration)
|
|
641
|
-
for (const resolved of resolvedChanges) {
|
|
642
|
-
if (resolved.outcome !== 'applied')
|
|
643
|
-
continue;
|
|
644
|
-
if (resolved.change.type === 'delete') {
|
|
645
|
-
await this.columnVersions.deleteRowVersions(resolved.change.schema, resolved.change.table, resolved.change.pk);
|
|
646
|
-
}
|
|
647
|
-
}
|
|
391
|
+
return applyChangesImpl(this, changes);
|
|
648
392
|
}
|
|
649
393
|
async canDeltaSync(peerSiteId, sinceHLC) {
|
|
650
394
|
const peerState = await this.peerStates.getPeerState(peerSiteId);
|
|
651
395
|
if (!peerState) {
|
|
652
|
-
// Never synced with this peer - need full snapshot
|
|
653
396
|
return false;
|
|
654
397
|
}
|
|
655
|
-
// Check if
|
|
398
|
+
// Check if tombstone TTL covers the requested time range
|
|
656
399
|
const now = Date.now();
|
|
657
400
|
const sinceTime = Number(sinceHLC.wallTime);
|
|
658
401
|
if (now - sinceTime > this.config.tombstoneTTL) {
|
|
659
|
-
// Too old - tombstones may have been pruned
|
|
660
402
|
return false;
|
|
661
403
|
}
|
|
662
404
|
return true;
|
|
663
405
|
}
|
|
406
|
+
// ============================================================================
|
|
407
|
+
// Delegated: Non-Streaming Snapshots
|
|
408
|
+
// ============================================================================
|
|
664
409
|
async getSnapshot() {
|
|
665
|
-
|
|
666
|
-
const cvBounds = buildAllColumnVersionsScanBounds();
|
|
667
|
-
for await (const entry of this.kv.iterate(cvBounds)) {
|
|
668
|
-
const parsed = parseColumnVersionKey(entry.key);
|
|
669
|
-
if (!parsed)
|
|
670
|
-
continue;
|
|
671
|
-
const cv = deserializeColumnVersion(entry.value);
|
|
672
|
-
const tableKey = `${parsed.schema}.${parsed.table}`;
|
|
673
|
-
const rowKey = encodePK(parsed.pk);
|
|
674
|
-
if (!tableData.has(tableKey)) {
|
|
675
|
-
tableData.set(tableKey, new Map());
|
|
676
|
-
}
|
|
677
|
-
const tableRows = tableData.get(tableKey);
|
|
678
|
-
if (!tableRows.has(rowKey)) {
|
|
679
|
-
tableRows.set(rowKey, new Map());
|
|
680
|
-
}
|
|
681
|
-
const rowVersions = tableRows.get(rowKey);
|
|
682
|
-
rowVersions.set(parsed.column, cv);
|
|
683
|
-
}
|
|
684
|
-
// Build table snapshots
|
|
685
|
-
const tables = [];
|
|
686
|
-
for (const [tableKey, rows] of tableData) {
|
|
687
|
-
const [schema, table] = tableKey.split('.');
|
|
688
|
-
const columnVersions = new Map();
|
|
689
|
-
const rowsArray = [];
|
|
690
|
-
for (const [rowKey, rowVersions] of rows) {
|
|
691
|
-
// Convert column map to array (row representation)
|
|
692
|
-
const row = Array.from(rowVersions.values()).map(cv => cv.value);
|
|
693
|
-
rowsArray.push(row);
|
|
694
|
-
// Add column versions with their values
|
|
695
|
-
for (const [column, cv] of rowVersions) {
|
|
696
|
-
const versionKey = `${rowKey}:${column}`;
|
|
697
|
-
columnVersions.set(versionKey, { hlc: cv.hlc, value: cv.value });
|
|
698
|
-
}
|
|
699
|
-
}
|
|
700
|
-
tables.push({
|
|
701
|
-
schema,
|
|
702
|
-
table,
|
|
703
|
-
rows: rowsArray,
|
|
704
|
-
columnVersions,
|
|
705
|
-
});
|
|
706
|
-
}
|
|
707
|
-
// Collect all schema migrations
|
|
708
|
-
const schemaMigrations = [];
|
|
709
|
-
const smBounds = buildAllSchemaMigrationsScanBounds();
|
|
710
|
-
for await (const entry of this.kv.iterate(smBounds)) {
|
|
711
|
-
const parsed = parseSchemaMigrationKey(entry.key);
|
|
712
|
-
if (!parsed)
|
|
713
|
-
continue;
|
|
714
|
-
const migration = deserializeMigration(entry.value);
|
|
715
|
-
schemaMigrations.push({
|
|
716
|
-
type: migration.type,
|
|
717
|
-
schema: parsed.schema,
|
|
718
|
-
table: parsed.table,
|
|
719
|
-
ddl: migration.ddl,
|
|
720
|
-
hlc: migration.hlc,
|
|
721
|
-
schemaVersion: migration.schemaVersion,
|
|
722
|
-
});
|
|
723
|
-
}
|
|
724
|
-
return {
|
|
725
|
-
siteId: this.getSiteId(),
|
|
726
|
-
hlc: this.getCurrentHLC(),
|
|
727
|
-
tables,
|
|
728
|
-
schemaMigrations,
|
|
729
|
-
};
|
|
410
|
+
return getSnapshotImpl(this);
|
|
730
411
|
}
|
|
731
412
|
async applySnapshot(snapshot) {
|
|
732
|
-
|
|
733
|
-
const dataChangesToApply = [];
|
|
734
|
-
const schemaChangesToApply = [];
|
|
735
|
-
// Build schema migrations for the store
|
|
736
|
-
for (const migration of snapshot.schemaMigrations) {
|
|
737
|
-
schemaChangesToApply.push({
|
|
738
|
-
type: migration.type,
|
|
739
|
-
schema: migration.schema,
|
|
740
|
-
table: migration.table,
|
|
741
|
-
ddl: migration.ddl,
|
|
742
|
-
});
|
|
743
|
-
}
|
|
744
|
-
// Build row data from column versions (grouped by pk)
|
|
745
|
-
for (const tableSnapshot of snapshot.tables) {
|
|
746
|
-
// Group column versions by pk to reconstruct rows
|
|
747
|
-
const rowsByPk = new Map();
|
|
748
|
-
for (const [versionKey, cvEntry] of tableSnapshot.columnVersions) {
|
|
749
|
-
const lastColon = versionKey.lastIndexOf(':');
|
|
750
|
-
if (lastColon === -1)
|
|
751
|
-
continue;
|
|
752
|
-
const rowKey = versionKey.slice(0, lastColon);
|
|
753
|
-
const column = versionKey.slice(lastColon + 1);
|
|
754
|
-
if (!rowsByPk.has(rowKey)) {
|
|
755
|
-
rowsByPk.set(rowKey, {});
|
|
756
|
-
}
|
|
757
|
-
rowsByPk.get(rowKey)[column] = cvEntry.value;
|
|
758
|
-
}
|
|
759
|
-
// Create data change for each row
|
|
760
|
-
for (const [rowKey, columns] of rowsByPk) {
|
|
761
|
-
const pk = JSON.parse(rowKey);
|
|
762
|
-
dataChangesToApply.push({
|
|
763
|
-
type: 'update', // Snapshot rows are upserts
|
|
764
|
-
schema: tableSnapshot.schema,
|
|
765
|
-
table: tableSnapshot.table,
|
|
766
|
-
pk,
|
|
767
|
-
columns,
|
|
768
|
-
});
|
|
769
|
-
}
|
|
770
|
-
}
|
|
771
|
-
// PHASE 2: Apply data to store via callback (before metadata)
|
|
772
|
-
if (this.applyToStore && (dataChangesToApply.length > 0 || schemaChangesToApply.length > 0)) {
|
|
773
|
-
await this.applyToStore(dataChangesToApply, schemaChangesToApply, { remote: true });
|
|
774
|
-
}
|
|
775
|
-
// PHASE 3: Clear existing CRDT metadata and apply new metadata
|
|
776
|
-
const clearBatch = this.kv.batch();
|
|
777
|
-
// Delete all existing column versions
|
|
778
|
-
const cvBounds = buildAllColumnVersionsScanBounds();
|
|
779
|
-
for await (const entry of this.kv.iterate(cvBounds)) {
|
|
780
|
-
clearBatch.delete(entry.key);
|
|
781
|
-
}
|
|
782
|
-
// Delete all existing tombstones
|
|
783
|
-
const tbBounds = buildAllTombstonesScanBounds();
|
|
784
|
-
for await (const entry of this.kv.iterate(tbBounds)) {
|
|
785
|
-
clearBatch.delete(entry.key);
|
|
786
|
-
}
|
|
787
|
-
// Delete all existing change log entries
|
|
788
|
-
const clBounds = buildAllChangeLogScanBounds();
|
|
789
|
-
for await (const entry of this.kv.iterate(clBounds)) {
|
|
790
|
-
clearBatch.delete(entry.key);
|
|
791
|
-
}
|
|
792
|
-
await clearBatch.write();
|
|
793
|
-
// Apply the snapshot's column versions and rebuild change log
|
|
794
|
-
const applyBatch = this.kv.batch();
|
|
795
|
-
for (const tableSnapshot of snapshot.tables) {
|
|
796
|
-
for (const [versionKey, cvEntry] of tableSnapshot.columnVersions) {
|
|
797
|
-
const lastColon = versionKey.lastIndexOf(':');
|
|
798
|
-
if (lastColon === -1)
|
|
799
|
-
continue;
|
|
800
|
-
const rowKey = versionKey.slice(0, lastColon);
|
|
801
|
-
const column = versionKey.slice(lastColon + 1);
|
|
802
|
-
const pk = JSON.parse(rowKey);
|
|
803
|
-
this.columnVersions.setColumnVersionBatch(applyBatch, tableSnapshot.schema, tableSnapshot.table, pk, column, { hlc: cvEntry.hlc, value: cvEntry.value });
|
|
804
|
-
// Rebuild change log entry
|
|
805
|
-
this.changeLog.recordColumnChangeBatch(applyBatch, cvEntry.hlc, tableSnapshot.schema, tableSnapshot.table, pk, column);
|
|
806
|
-
}
|
|
807
|
-
}
|
|
808
|
-
// Record schema migrations
|
|
809
|
-
for (const migration of snapshot.schemaMigrations) {
|
|
810
|
-
const schemaVersion = migration.schemaVersion ??
|
|
811
|
-
(await this.schemaMigrations.getCurrentVersion(migration.schema, migration.table)) + 1;
|
|
812
|
-
await this.schemaMigrations.recordMigration(migration.schema, migration.table, {
|
|
813
|
-
type: migration.type,
|
|
814
|
-
ddl: migration.ddl,
|
|
815
|
-
hlc: migration.hlc,
|
|
816
|
-
schemaVersion,
|
|
817
|
-
});
|
|
818
|
-
}
|
|
819
|
-
await applyBatch.write();
|
|
820
|
-
// Update our HLC to be at least as high as the snapshot
|
|
821
|
-
this.hlcManager.receive(snapshot.hlc);
|
|
822
|
-
await this.persistHLCState();
|
|
823
|
-
// Emit sync state change
|
|
824
|
-
this.syncEvents.emitSyncStateChange({ status: 'synced', lastSyncHLC: snapshot.hlc });
|
|
413
|
+
return applySnapshotImpl(this, snapshot);
|
|
825
414
|
}
|
|
415
|
+
// ============================================================================
|
|
416
|
+
// Peer State & Maintenance
|
|
417
|
+
// ============================================================================
|
|
826
418
|
async updatePeerSyncState(peerSiteId, hlc) {
|
|
827
419
|
await this.peerStates.setPeerState(peerSiteId, hlc);
|
|
828
420
|
}
|
|
@@ -845,463 +437,23 @@ export class SyncManagerImpl {
|
|
|
845
437
|
await batch.write();
|
|
846
438
|
return count;
|
|
847
439
|
}
|
|
848
|
-
/**
|
|
849
|
-
* Get the sync event emitter for UI integration.
|
|
850
|
-
*/
|
|
851
440
|
getEventEmitter() {
|
|
852
441
|
return this.syncEvents;
|
|
853
442
|
}
|
|
854
443
|
// ============================================================================
|
|
855
|
-
// Streaming Snapshot API
|
|
444
|
+
// Delegated: Streaming Snapshot API
|
|
856
445
|
// ============================================================================
|
|
857
|
-
async *getSnapshotStream(chunkSize
|
|
858
|
-
|
|
859
|
-
const siteId = this.getSiteId();
|
|
860
|
-
const hlc = this.getCurrentHLC();
|
|
861
|
-
// Count tables and migrations for header
|
|
862
|
-
const tableKeys = new Set();
|
|
863
|
-
const cvBounds = buildAllColumnVersionsScanBounds();
|
|
864
|
-
for await (const entry of this.kv.iterate(cvBounds)) {
|
|
865
|
-
const parsed = parseColumnVersionKey(entry.key);
|
|
866
|
-
if (parsed)
|
|
867
|
-
tableKeys.add(`${parsed.schema}.${parsed.table}`);
|
|
868
|
-
}
|
|
869
|
-
let migrationCount = 0;
|
|
870
|
-
const smBounds = buildAllSchemaMigrationsScanBounds();
|
|
871
|
-
for await (const _entry of this.kv.iterate(smBounds)) {
|
|
872
|
-
migrationCount++;
|
|
873
|
-
}
|
|
874
|
-
// Yield header
|
|
875
|
-
const header = {
|
|
876
|
-
type: 'header',
|
|
877
|
-
siteId,
|
|
878
|
-
hlc,
|
|
879
|
-
tableCount: tableKeys.size,
|
|
880
|
-
migrationCount,
|
|
881
|
-
snapshotId,
|
|
882
|
-
};
|
|
883
|
-
yield header;
|
|
884
|
-
// Stream each table
|
|
885
|
-
let totalEntries = 0;
|
|
886
|
-
for (const tableKey of tableKeys) {
|
|
887
|
-
const [schema, table] = tableKey.split('.');
|
|
888
|
-
// Estimate entries for this table
|
|
889
|
-
let tableEntryCount = 0;
|
|
890
|
-
const tableCvBounds = buildAllColumnVersionsScanBounds();
|
|
891
|
-
for await (const entry of this.kv.iterate(tableCvBounds)) {
|
|
892
|
-
const parsed = parseColumnVersionKey(entry.key);
|
|
893
|
-
if (parsed && parsed.schema === schema && parsed.table === table) {
|
|
894
|
-
tableEntryCount++;
|
|
895
|
-
}
|
|
896
|
-
}
|
|
897
|
-
// Yield table start
|
|
898
|
-
const tableStart = {
|
|
899
|
-
type: 'table-start',
|
|
900
|
-
schema,
|
|
901
|
-
table,
|
|
902
|
-
estimatedEntries: tableEntryCount,
|
|
903
|
-
};
|
|
904
|
-
yield tableStart;
|
|
905
|
-
// Stream column versions in chunks
|
|
906
|
-
let entries = [];
|
|
907
|
-
let entriesWritten = 0;
|
|
908
|
-
for await (const entry of this.kv.iterate(tableCvBounds)) {
|
|
909
|
-
const parsed = parseColumnVersionKey(entry.key);
|
|
910
|
-
if (!parsed || parsed.schema !== schema || parsed.table !== table)
|
|
911
|
-
continue;
|
|
912
|
-
const cv = deserializeColumnVersion(entry.value);
|
|
913
|
-
const versionKey = `${encodePK(parsed.pk)}:${parsed.column}`;
|
|
914
|
-
entries.push([versionKey, cv.hlc, cv.value]);
|
|
915
|
-
entriesWritten++;
|
|
916
|
-
if (entries.length >= chunkSize) {
|
|
917
|
-
const chunk = {
|
|
918
|
-
type: 'column-versions',
|
|
919
|
-
schema,
|
|
920
|
-
table,
|
|
921
|
-
entries,
|
|
922
|
-
};
|
|
923
|
-
yield chunk;
|
|
924
|
-
entries = [];
|
|
925
|
-
}
|
|
926
|
-
}
|
|
927
|
-
// Yield remaining entries
|
|
928
|
-
if (entries.length > 0) {
|
|
929
|
-
const chunk = {
|
|
930
|
-
type: 'column-versions',
|
|
931
|
-
schema,
|
|
932
|
-
table,
|
|
933
|
-
entries,
|
|
934
|
-
};
|
|
935
|
-
yield chunk;
|
|
936
|
-
}
|
|
937
|
-
// Yield table end
|
|
938
|
-
const tableEnd = {
|
|
939
|
-
type: 'table-end',
|
|
940
|
-
schema,
|
|
941
|
-
table,
|
|
942
|
-
entriesWritten,
|
|
943
|
-
};
|
|
944
|
-
yield tableEnd;
|
|
945
|
-
totalEntries += entriesWritten;
|
|
946
|
-
}
|
|
947
|
-
// Stream schema migrations
|
|
948
|
-
for await (const entry of this.kv.iterate(smBounds)) {
|
|
949
|
-
const parsed = parseSchemaMigrationKey(entry.key);
|
|
950
|
-
if (!parsed)
|
|
951
|
-
continue;
|
|
952
|
-
const migration = deserializeMigration(entry.value);
|
|
953
|
-
const migrationChunk = {
|
|
954
|
-
type: 'schema-migration',
|
|
955
|
-
migration: {
|
|
956
|
-
type: migration.type,
|
|
957
|
-
schema: parsed.schema,
|
|
958
|
-
table: parsed.table,
|
|
959
|
-
ddl: migration.ddl,
|
|
960
|
-
hlc: migration.hlc,
|
|
961
|
-
schemaVersion: migration.schemaVersion,
|
|
962
|
-
},
|
|
963
|
-
};
|
|
964
|
-
yield migrationChunk;
|
|
965
|
-
}
|
|
966
|
-
// Yield footer
|
|
967
|
-
const footer = {
|
|
968
|
-
type: 'footer',
|
|
969
|
-
snapshotId,
|
|
970
|
-
totalTables: tableKeys.size,
|
|
971
|
-
totalEntries,
|
|
972
|
-
totalMigrations: migrationCount,
|
|
973
|
-
};
|
|
974
|
-
yield footer;
|
|
446
|
+
async *getSnapshotStream(chunkSize) {
|
|
447
|
+
yield* getSnapshotStreamImpl(this, chunkSize);
|
|
975
448
|
}
|
|
976
449
|
async applySnapshotStream(chunks, onProgress) {
|
|
977
|
-
|
|
978
|
-
let snapshotHLC;
|
|
979
|
-
let totalTables = 0;
|
|
980
|
-
let totalEntries = 0;
|
|
981
|
-
let tablesProcessed = 0;
|
|
982
|
-
let entriesProcessed = 0;
|
|
983
|
-
let currentTable;
|
|
984
|
-
const completedTables = [];
|
|
985
|
-
// Pending data to apply to store (batched for efficiency)
|
|
986
|
-
let pendingDataChanges = [];
|
|
987
|
-
let pendingSchemaChanges = [];
|
|
988
|
-
const DATA_FLUSH_SIZE = 100;
|
|
989
|
-
// Helper to flush pending data changes to store
|
|
990
|
-
const flushDataToStore = async () => {
|
|
991
|
-
if (this.applyToStore && (pendingDataChanges.length > 0 || pendingSchemaChanges.length > 0)) {
|
|
992
|
-
await this.applyToStore(pendingDataChanges, pendingSchemaChanges, { remote: true });
|
|
993
|
-
pendingDataChanges = [];
|
|
994
|
-
pendingSchemaChanges = [];
|
|
995
|
-
}
|
|
996
|
-
};
|
|
997
|
-
// Clear existing metadata before applying
|
|
998
|
-
const clearBatch = this.kv.batch();
|
|
999
|
-
for await (const entry of this.kv.iterate(buildAllColumnVersionsScanBounds())) {
|
|
1000
|
-
clearBatch.delete(entry.key);
|
|
1001
|
-
}
|
|
1002
|
-
for await (const entry of this.kv.iterate(buildAllTombstonesScanBounds())) {
|
|
1003
|
-
clearBatch.delete(entry.key);
|
|
1004
|
-
}
|
|
1005
|
-
for await (const entry of this.kv.iterate(buildAllChangeLogScanBounds())) {
|
|
1006
|
-
clearBatch.delete(entry.key);
|
|
1007
|
-
}
|
|
1008
|
-
await clearBatch.write();
|
|
1009
|
-
// Process chunks - track column versions per row for grouping
|
|
1010
|
-
let batch = this.kv.batch();
|
|
1011
|
-
let batchSize = 0;
|
|
1012
|
-
const BATCH_FLUSH_SIZE = 1000;
|
|
1013
|
-
// Track current table's rows for data application
|
|
1014
|
-
let currentTableSchema;
|
|
1015
|
-
let currentTableName;
|
|
1016
|
-
const rowColumns = new Map();
|
|
1017
|
-
for await (const chunk of chunks) {
|
|
1018
|
-
switch (chunk.type) {
|
|
1019
|
-
case 'header':
|
|
1020
|
-
snapshotId = chunk.snapshotId;
|
|
1021
|
-
snapshotHLC = chunk.hlc;
|
|
1022
|
-
totalTables = chunk.tableCount;
|
|
1023
|
-
break;
|
|
1024
|
-
case 'table-start':
|
|
1025
|
-
currentTable = `${chunk.schema}.${chunk.table}`;
|
|
1026
|
-
currentTableSchema = chunk.schema;
|
|
1027
|
-
currentTableName = chunk.table;
|
|
1028
|
-
totalEntries += chunk.estimatedEntries;
|
|
1029
|
-
rowColumns.clear();
|
|
1030
|
-
break;
|
|
1031
|
-
case 'column-versions':
|
|
1032
|
-
for (const [versionKey, hlc, value] of chunk.entries) {
|
|
1033
|
-
const lastColon = versionKey.lastIndexOf(':');
|
|
1034
|
-
if (lastColon === -1)
|
|
1035
|
-
continue;
|
|
1036
|
-
const rowKey = versionKey.slice(0, lastColon);
|
|
1037
|
-
const column = versionKey.slice(lastColon + 1);
|
|
1038
|
-
const pk = JSON.parse(rowKey);
|
|
1039
|
-
// Track column for data application
|
|
1040
|
-
if (!rowColumns.has(rowKey)) {
|
|
1041
|
-
rowColumns.set(rowKey, {});
|
|
1042
|
-
}
|
|
1043
|
-
rowColumns.get(rowKey)[column] = value;
|
|
1044
|
-
// Write CRDT metadata
|
|
1045
|
-
this.columnVersions.setColumnVersionBatch(batch, chunk.schema, chunk.table, pk, column, { hlc, value });
|
|
1046
|
-
this.changeLog.recordColumnChangeBatch(batch, hlc, chunk.schema, chunk.table, pk, column);
|
|
1047
|
-
batchSize++;
|
|
1048
|
-
entriesProcessed++;
|
|
1049
|
-
if (batchSize >= BATCH_FLUSH_SIZE) {
|
|
1050
|
-
await batch.write();
|
|
1051
|
-
batch = this.kv.batch();
|
|
1052
|
-
batchSize = 0;
|
|
1053
|
-
// Save checkpoint with completed tables
|
|
1054
|
-
if (snapshotId && snapshotHLC) {
|
|
1055
|
-
await this.saveSnapshotCheckpoint({
|
|
1056
|
-
snapshotId,
|
|
1057
|
-
siteId: this.getSiteId(),
|
|
1058
|
-
hlc: snapshotHLC,
|
|
1059
|
-
lastTableIndex: tablesProcessed,
|
|
1060
|
-
lastEntryIndex: entriesProcessed,
|
|
1061
|
-
completedTables: [...completedTables],
|
|
1062
|
-
entriesProcessed,
|
|
1063
|
-
createdAt: Date.now(),
|
|
1064
|
-
});
|
|
1065
|
-
}
|
|
1066
|
-
}
|
|
1067
|
-
}
|
|
1068
|
-
if (onProgress && snapshotId) {
|
|
1069
|
-
onProgress({
|
|
1070
|
-
snapshotId,
|
|
1071
|
-
tablesProcessed,
|
|
1072
|
-
totalTables,
|
|
1073
|
-
entriesProcessed,
|
|
1074
|
-
totalEntries,
|
|
1075
|
-
currentTable,
|
|
1076
|
-
});
|
|
1077
|
-
}
|
|
1078
|
-
break;
|
|
1079
|
-
case 'table-end':
|
|
1080
|
-
// Flush accumulated rows to store
|
|
1081
|
-
if (currentTableSchema && currentTableName) {
|
|
1082
|
-
for (const [rowKey, columns] of rowColumns) {
|
|
1083
|
-
const pk = JSON.parse(rowKey);
|
|
1084
|
-
pendingDataChanges.push({
|
|
1085
|
-
type: 'update',
|
|
1086
|
-
schema: currentTableSchema,
|
|
1087
|
-
table: currentTableName,
|
|
1088
|
-
pk,
|
|
1089
|
-
columns,
|
|
1090
|
-
});
|
|
1091
|
-
if (pendingDataChanges.length >= DATA_FLUSH_SIZE) {
|
|
1092
|
-
await flushDataToStore();
|
|
1093
|
-
}
|
|
1094
|
-
}
|
|
1095
|
-
rowColumns.clear();
|
|
1096
|
-
}
|
|
1097
|
-
tablesProcessed++;
|
|
1098
|
-
if (currentTable) {
|
|
1099
|
-
completedTables.push(currentTable);
|
|
1100
|
-
}
|
|
1101
|
-
break;
|
|
1102
|
-
case 'schema-migration': {
|
|
1103
|
-
const migration = chunk.migration;
|
|
1104
|
-
pendingSchemaChanges.push({
|
|
1105
|
-
type: migration.type,
|
|
1106
|
-
schema: migration.schema,
|
|
1107
|
-
table: migration.table,
|
|
1108
|
-
ddl: migration.ddl,
|
|
1109
|
-
});
|
|
1110
|
-
// Record migration metadata
|
|
1111
|
-
const schemaVersion = migration.schemaVersion ??
|
|
1112
|
-
(await this.schemaMigrations.getCurrentVersion(migration.schema, migration.table)) + 1;
|
|
1113
|
-
await this.schemaMigrations.recordMigration(migration.schema, migration.table, {
|
|
1114
|
-
type: migration.type,
|
|
1115
|
-
ddl: migration.ddl,
|
|
1116
|
-
hlc: migration.hlc,
|
|
1117
|
-
schemaVersion,
|
|
1118
|
-
});
|
|
1119
|
-
break;
|
|
1120
|
-
}
|
|
1121
|
-
case 'footer':
|
|
1122
|
-
// Flush remaining data to store
|
|
1123
|
-
await flushDataToStore();
|
|
1124
|
-
// Flush remaining metadata batch
|
|
1125
|
-
if (batchSize > 0) {
|
|
1126
|
-
await batch.write();
|
|
1127
|
-
}
|
|
1128
|
-
// Update HLC
|
|
1129
|
-
if (snapshotHLC) {
|
|
1130
|
-
this.hlcManager.receive(snapshotHLC);
|
|
1131
|
-
await this.persistHLCState();
|
|
1132
|
-
}
|
|
1133
|
-
// Clear checkpoint
|
|
1134
|
-
if (snapshotId) {
|
|
1135
|
-
await this.clearSnapshotCheckpoint(snapshotId);
|
|
1136
|
-
}
|
|
1137
|
-
// Emit sync state change
|
|
1138
|
-
if (snapshotHLC) {
|
|
1139
|
-
this.syncEvents.emitSyncStateChange({ status: 'synced', lastSyncHLC: snapshotHLC });
|
|
1140
|
-
}
|
|
1141
|
-
break;
|
|
1142
|
-
}
|
|
1143
|
-
}
|
|
450
|
+
return applySnapshotStreamImpl(this, chunks, onProgress);
|
|
1144
451
|
}
|
|
1145
452
|
async getSnapshotCheckpoint(snapshotId) {
|
|
1146
|
-
|
|
1147
|
-
const data = await this.kv.get(key);
|
|
1148
|
-
if (!data)
|
|
1149
|
-
return undefined;
|
|
1150
|
-
const json = new TextDecoder().decode(data);
|
|
1151
|
-
const obj = JSON.parse(json);
|
|
1152
|
-
// Reconstruct HLC with proper types
|
|
1153
|
-
return {
|
|
1154
|
-
...obj,
|
|
1155
|
-
hlc: {
|
|
1156
|
-
wallTime: BigInt(obj.hlc.wallTime),
|
|
1157
|
-
counter: obj.hlc.counter,
|
|
1158
|
-
siteId: new Uint8Array(obj.hlc.siteId),
|
|
1159
|
-
},
|
|
1160
|
-
siteId: new Uint8Array(obj.siteId),
|
|
1161
|
-
};
|
|
1162
|
-
}
|
|
1163
|
-
async saveSnapshotCheckpoint(checkpoint) {
|
|
1164
|
-
const key = new TextEncoder().encode(`${CHECKPOINT_PREFIX}${checkpoint.snapshotId}`);
|
|
1165
|
-
const json = JSON.stringify({
|
|
1166
|
-
...checkpoint,
|
|
1167
|
-
hlc: {
|
|
1168
|
-
wallTime: checkpoint.hlc.wallTime.toString(),
|
|
1169
|
-
counter: checkpoint.hlc.counter,
|
|
1170
|
-
siteId: Array.from(checkpoint.hlc.siteId),
|
|
1171
|
-
},
|
|
1172
|
-
siteId: Array.from(checkpoint.siteId),
|
|
1173
|
-
});
|
|
1174
|
-
await this.kv.put(key, new TextEncoder().encode(json));
|
|
1175
|
-
}
|
|
1176
|
-
async clearSnapshotCheckpoint(snapshotId) {
|
|
1177
|
-
const key = new TextEncoder().encode(`${CHECKPOINT_PREFIX}${snapshotId}`);
|
|
1178
|
-
await this.kv.delete(key);
|
|
453
|
+
return getSnapshotCheckpointImpl(this, snapshotId);
|
|
1179
454
|
}
|
|
1180
455
|
async *resumeSnapshotStream(checkpoint) {
|
|
1181
|
-
|
|
1182
|
-
// Skip tables that have already been completed
|
|
1183
|
-
const completedSet = new Set(checkpoint.completedTables);
|
|
1184
|
-
const snapshotId = checkpoint.snapshotId;
|
|
1185
|
-
const siteId = checkpoint.siteId;
|
|
1186
|
-
const hlc = checkpoint.hlc;
|
|
1187
|
-
// Count tables and migrations for header
|
|
1188
|
-
const tableKeys = new Set();
|
|
1189
|
-
const cvBounds = buildAllColumnVersionsScanBounds();
|
|
1190
|
-
for await (const entry of this.kv.iterate(cvBounds)) {
|
|
1191
|
-
const parsed = parseColumnVersionKey(entry.key);
|
|
1192
|
-
if (parsed)
|
|
1193
|
-
tableKeys.add(`${parsed.schema}.${parsed.table}`);
|
|
1194
|
-
}
|
|
1195
|
-
let migrationCount = 0;
|
|
1196
|
-
const smBounds = buildAllSchemaMigrationsScanBounds();
|
|
1197
|
-
for await (const _entry of this.kv.iterate(smBounds)) {
|
|
1198
|
-
migrationCount++;
|
|
1199
|
-
}
|
|
1200
|
-
// Yield header (receiver needs to know this is a resume)
|
|
1201
|
-
const header = {
|
|
1202
|
-
type: 'header',
|
|
1203
|
-
siteId,
|
|
1204
|
-
hlc,
|
|
1205
|
-
tableCount: tableKeys.size,
|
|
1206
|
-
migrationCount,
|
|
1207
|
-
snapshotId,
|
|
1208
|
-
};
|
|
1209
|
-
yield header;
|
|
1210
|
-
// Stream each table, skipping completed ones
|
|
1211
|
-
let totalEntries = checkpoint.entriesProcessed;
|
|
1212
|
-
for (const tableKey of tableKeys) {
|
|
1213
|
-
// Skip already completed tables
|
|
1214
|
-
if (completedSet.has(tableKey))
|
|
1215
|
-
continue;
|
|
1216
|
-
const [schema, table] = tableKey.split('.');
|
|
1217
|
-
// Count entries for this table
|
|
1218
|
-
let tableEntryCount = 0;
|
|
1219
|
-
const tableCvBounds = buildAllColumnVersionsScanBounds();
|
|
1220
|
-
for await (const entry of this.kv.iterate(tableCvBounds)) {
|
|
1221
|
-
const parsed = parseColumnVersionKey(entry.key);
|
|
1222
|
-
if (parsed && parsed.schema === schema && parsed.table === table) {
|
|
1223
|
-
tableEntryCount++;
|
|
1224
|
-
}
|
|
1225
|
-
}
|
|
1226
|
-
// Yield table start
|
|
1227
|
-
const tableStart = {
|
|
1228
|
-
type: 'table-start',
|
|
1229
|
-
schema,
|
|
1230
|
-
table,
|
|
1231
|
-
estimatedEntries: tableEntryCount,
|
|
1232
|
-
};
|
|
1233
|
-
yield tableStart;
|
|
1234
|
-
// Stream column versions in chunks
|
|
1235
|
-
let entries = [];
|
|
1236
|
-
let entriesWritten = 0;
|
|
1237
|
-
const chunkSize = DEFAULT_SNAPSHOT_CHUNK_SIZE;
|
|
1238
|
-
for await (const entry of this.kv.iterate(tableCvBounds)) {
|
|
1239
|
-
const parsed = parseColumnVersionKey(entry.key);
|
|
1240
|
-
if (!parsed || parsed.schema !== schema || parsed.table !== table)
|
|
1241
|
-
continue;
|
|
1242
|
-
const cv = deserializeColumnVersion(entry.value);
|
|
1243
|
-
const versionKey = `${encodePK(parsed.pk)}:${parsed.column}`;
|
|
1244
|
-
entries.push([versionKey, cv.hlc, cv.value]);
|
|
1245
|
-
entriesWritten++;
|
|
1246
|
-
if (entries.length >= chunkSize) {
|
|
1247
|
-
const chunk = {
|
|
1248
|
-
type: 'column-versions',
|
|
1249
|
-
schema,
|
|
1250
|
-
table,
|
|
1251
|
-
entries,
|
|
1252
|
-
};
|
|
1253
|
-
yield chunk;
|
|
1254
|
-
entries = [];
|
|
1255
|
-
}
|
|
1256
|
-
}
|
|
1257
|
-
// Yield remaining entries
|
|
1258
|
-
if (entries.length > 0) {
|
|
1259
|
-
const chunk = {
|
|
1260
|
-
type: 'column-versions',
|
|
1261
|
-
schema,
|
|
1262
|
-
table,
|
|
1263
|
-
entries,
|
|
1264
|
-
};
|
|
1265
|
-
yield chunk;
|
|
1266
|
-
}
|
|
1267
|
-
// Yield table end
|
|
1268
|
-
const tableEnd = {
|
|
1269
|
-
type: 'table-end',
|
|
1270
|
-
schema,
|
|
1271
|
-
table,
|
|
1272
|
-
entriesWritten,
|
|
1273
|
-
};
|
|
1274
|
-
yield tableEnd;
|
|
1275
|
-
totalEntries += entriesWritten;
|
|
1276
|
-
}
|
|
1277
|
-
// Stream schema migrations
|
|
1278
|
-
for await (const entry of this.kv.iterate(smBounds)) {
|
|
1279
|
-
const parsed = parseSchemaMigrationKey(entry.key);
|
|
1280
|
-
if (!parsed)
|
|
1281
|
-
continue;
|
|
1282
|
-
const migration = deserializeMigration(entry.value);
|
|
1283
|
-
const migrationChunk = {
|
|
1284
|
-
type: 'schema-migration',
|
|
1285
|
-
migration: {
|
|
1286
|
-
type: migration.type,
|
|
1287
|
-
schema: parsed.schema,
|
|
1288
|
-
table: parsed.table,
|
|
1289
|
-
ddl: migration.ddl,
|
|
1290
|
-
hlc: migration.hlc,
|
|
1291
|
-
schemaVersion: migration.schemaVersion,
|
|
1292
|
-
},
|
|
1293
|
-
};
|
|
1294
|
-
yield migrationChunk;
|
|
1295
|
-
}
|
|
1296
|
-
// Yield footer
|
|
1297
|
-
const footer = {
|
|
1298
|
-
type: 'footer',
|
|
1299
|
-
snapshotId,
|
|
1300
|
-
totalTables: tableKeys.size,
|
|
1301
|
-
totalEntries,
|
|
1302
|
-
totalMigrations: migrationCount,
|
|
1303
|
-
};
|
|
1304
|
-
yield footer;
|
|
456
|
+
yield* resumeSnapshotStreamImpl(this, checkpoint);
|
|
1305
457
|
}
|
|
1306
458
|
}
|
|
1307
459
|
//# sourceMappingURL=sync-manager-impl.js.map
|