@quereus/sync 0.6.2 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/README.md +187 -155
  2. package/dist/src/index.d.ts +3 -5
  3. package/dist/src/index.d.ts.map +1 -1
  4. package/dist/src/index.js +3 -5
  5. package/dist/src/index.js.map +1 -1
  6. package/dist/src/metadata/keys.d.ts +8 -0
  7. package/dist/src/metadata/keys.d.ts.map +1 -1
  8. package/dist/src/metadata/keys.js +11 -0
  9. package/dist/src/metadata/keys.js.map +1 -1
  10. package/dist/src/sync/change-applicator.d.ts +44 -0
  11. package/dist/src/sync/change-applicator.d.ts.map +1 -0
  12. package/dist/src/sync/change-applicator.js +243 -0
  13. package/dist/src/sync/change-applicator.js.map +1 -0
  14. package/dist/src/sync/snapshot-stream.d.ts +26 -0
  15. package/dist/src/sync/snapshot-stream.d.ts.map +1 -0
  16. package/dist/src/sync/snapshot-stream.js +375 -0
  17. package/dist/src/sync/snapshot-stream.js.map +1 -0
  18. package/dist/src/sync/snapshot.d.ts +17 -0
  19. package/dist/src/sync/snapshot.d.ts.map +1 -0
  20. package/dist/src/sync/snapshot.js +166 -0
  21. package/dist/src/sync/snapshot.js.map +1 -0
  22. package/dist/src/sync/store-adapter.d.ts.map +1 -1
  23. package/dist/src/sync/store-adapter.js +3 -2
  24. package/dist/src/sync/store-adapter.js.map +1 -1
  25. package/dist/src/sync/sync-context.d.ts +47 -0
  26. package/dist/src/sync/sync-context.d.ts.map +1 -0
  27. package/dist/src/sync/sync-context.js +36 -0
  28. package/dist/src/sync/sync-context.js.map +1 -0
  29. package/dist/src/sync/sync-manager-impl.d.ts +21 -48
  30. package/dist/src/sync/sync-manager-impl.d.ts.map +1 -1
  31. package/dist/src/sync/sync-manager-impl.js +140 -988
  32. package/dist/src/sync/sync-manager-impl.js.map +1 -1
  33. package/package.json +4 -4
@@ -2,6 +2,7 @@
2
2
  * SyncManager implementation.
3
3
  *
4
4
  * Coordinates CRDT metadata tracking and sync operations.
5
+ * Delegates to focused sub-modules for snapshot, streaming, and change application.
5
6
  */
6
7
  import { HLCManager, compareHLC } from '../clock/hlc.js';
7
8
  import { generateSiteId, SITE_ID_KEY, serializeSiteIdentity, deserializeSiteIdentity, siteIdEquals, } from '../clock/site.js';
@@ -10,13 +11,16 @@ import { TombstoneStore, deserializeTombstone } from '../metadata/tombstones.js'
10
11
  import { PeerStateStore } from '../metadata/peer-state.js';
11
12
  import { SchemaMigrationStore, deserializeMigration } from '../metadata/schema-migration.js';
12
13
  import { ChangeLogStore } from '../metadata/change-log.js';
13
- import { SYNC_KEY_PREFIX, buildAllColumnVersionsScanBounds, buildAllTombstonesScanBounds, buildAllSchemaMigrationsScanBounds, buildAllChangeLogScanBounds, parseColumnVersionKey, parseTombstoneKey, parseSchemaMigrationKey, encodePK, } from '../metadata/keys.js';
14
- /** Default chunk size for streaming snapshots. */
15
- const DEFAULT_SNAPSHOT_CHUNK_SIZE = 1000;
16
- /** Key prefix for snapshot checkpoints. */
17
- const CHECKPOINT_PREFIX = 'sc:';
14
+ import { SYNC_KEY_PREFIX, buildAllColumnVersionsScanBounds, buildAllTombstonesScanBounds, buildAllSchemaMigrationsScanBounds, parseColumnVersionKey, parseTombstoneKey, parseSchemaMigrationKey, } from '../metadata/keys.js';
15
+ import { persistHLCState, persistHLCStateBatch, toError } from './sync-context.js';
16
+ import { applyChanges as applyChangesImpl } from './change-applicator.js';
17
+ import { getSnapshot as getSnapshotImpl, applySnapshot as applySnapshotImpl } from './snapshot.js';
18
+ import { getSnapshotStream as getSnapshotStreamImpl, applySnapshotStream as applySnapshotStreamImpl, getSnapshotCheckpoint as getSnapshotCheckpointImpl, resumeSnapshotStream as resumeSnapshotStreamImpl, } from './snapshot-stream.js';
18
19
  /**
19
20
  * Implementation of SyncManager.
21
+ *
22
+ * Acts as a coordinator/facade that delegates snapshot, streaming,
23
+ * and change application to focused sub-modules.
20
24
  */
21
25
  export class SyncManagerImpl {
22
26
  kv;
@@ -91,160 +95,151 @@ export class SyncManagerImpl {
91
95
  storeEvents.onSchemaChange((event) => manager.handleSchemaChange(event));
92
96
  return manager;
93
97
  }
98
+ // ============================================================================
99
+ // Accessors
100
+ // ============================================================================
94
101
  getSiteId() {
95
102
  return this.hlcManager.getSiteId();
96
103
  }
97
104
  getCurrentHLC() {
98
105
  return this.hlcManager.now();
99
106
  }
107
+ // ============================================================================
108
+ // Event Handlers (local store changes)
109
+ // ============================================================================
100
110
  /**
101
111
  * Handle a data change event from the store.
102
112
  * Records CRDT metadata for the change.
103
- * Skips remote events to prevent duplicate recording.
104
113
  */
105
114
  async handleDataChange(event) {
106
- // Skip events from remote sync - metadata already recorded by the originating replica
107
- if (event.remote)
108
- return;
109
- const hlc = this.hlcManager.tick();
110
- const { schemaName, tableName, type, oldRow, newRow } = event;
111
- // Support both 'key' and 'pk' property names
112
- const pk = event.key ?? event.pk;
113
- if (!pk) {
114
- // Cannot record change without primary key
115
- return;
116
- }
117
- const batch = this.kv.batch();
118
- if (type === 'delete') {
119
- // Record tombstone
120
- this.tombstones.setTombstoneBatch(batch, schemaName, tableName, pk, hlc);
121
- // Record in change log for efficient delta queries
122
- this.changeLog.recordDeletionBatch(batch, hlc, schemaName, tableName, pk);
123
- // Delete column versions for this row
124
- await this.columnVersions.deleteRowVersions(schemaName, tableName, pk);
125
- const change = {
126
- type: 'delete',
127
- schema: schemaName,
128
- table: tableName,
129
- pk,
130
- hlc,
131
- };
132
- this.pendingChanges.push(change);
133
- }
134
- else {
135
- // Insert or update: record column versions
136
- if (newRow) {
137
- await this.recordColumnVersions(batch, schemaName, tableName, pk, oldRow, newRow, hlc);
115
+ try {
116
+ if (event.remote)
117
+ return;
118
+ const hlc = this.hlcManager.tick();
119
+ const { schemaName, tableName, type, oldRow, newRow } = event;
120
+ const pk = event.key ?? event.pk;
121
+ if (!pk) {
122
+ console.warn(`[Sync] Missing primary key for ${schemaName}.${tableName} ${type} event — change not tracked`);
123
+ return;
124
+ }
125
+ const batch = this.kv.batch();
126
+ if (type === 'delete') {
127
+ this.tombstones.setTombstoneBatch(batch, schemaName, tableName, pk, hlc);
128
+ this.changeLog.recordDeletionBatch(batch, hlc, schemaName, tableName, pk);
129
+ await this.columnVersions.deleteRowVersions(schemaName, tableName, pk);
130
+ const change = {
131
+ type: 'delete',
132
+ schema: schemaName,
133
+ table: tableName,
134
+ pk,
135
+ hlc,
136
+ };
137
+ this.pendingChanges.push(change);
138
138
  }
139
+ else {
140
+ if (newRow) {
141
+ await this.recordColumnVersions(batch, schemaName, tableName, pk, oldRow, newRow, hlc);
142
+ }
143
+ }
144
+ // Persist HLC state in batch (DRY: uses shared helper)
145
+ persistHLCStateBatch(this, batch);
146
+ await batch.write();
147
+ const changesToEmit = [...this.pendingChanges];
148
+ this.pendingChanges = [];
149
+ this.syncEvents.emitLocalChange({
150
+ transactionId: this.currentTransactionId || crypto.randomUUID(),
151
+ changes: changesToEmit,
152
+ pendingSync: true,
153
+ });
154
+ }
155
+ catch (error) {
156
+ console.error('[Sync] Error handling data change:', error);
157
+ this.syncEvents.emitSyncStateChange({
158
+ status: 'error',
159
+ error: toError(error),
160
+ });
139
161
  }
140
- // Persist HLC state in batch
141
- const hlcState = this.hlcManager.getState();
142
- const hlcBuffer = new Uint8Array(10);
143
- const hlcView = new DataView(hlcBuffer.buffer);
144
- hlcView.setBigUint64(0, hlcState.wallTime, false);
145
- hlcView.setUint16(8, hlcState.counter, false);
146
- batch.put(SYNC_KEY_PREFIX.HLC_STATE, hlcBuffer);
147
- await batch.write();
148
- // Emit local change event with current pending changes, then clear them
149
- const changesToEmit = [...this.pendingChanges];
150
- this.pendingChanges = [];
151
- this.syncEvents.emitLocalChange({
152
- transactionId: this.currentTransactionId || crypto.randomUUID(),
153
- changes: changesToEmit,
154
- pendingSync: true,
155
- });
156
162
  }
157
163
  /**
158
164
  * Handle a schema change event from the store.
159
165
  * Records schema migrations for sync.
160
- * Skips remote events to prevent duplicate recording.
161
166
  */
162
167
  async handleSchemaChange(event) {
163
- // Skip events from remote sync - metadata already recorded by the originating replica
164
- if (event.remote)
165
- return;
166
- const hlc = this.hlcManager.tick();
167
- const { type, objectType, schemaName, objectName, ddl } = event;
168
- // Map store event type to migration type
169
- let migrationType;
170
- if (objectType === 'table') {
171
- switch (type) {
172
- case 'create':
173
- migrationType = 'create_table';
174
- break;
175
- case 'drop':
176
- migrationType = 'drop_table';
177
- break;
178
- case 'alter':
179
- migrationType = 'alter_column';
180
- break;
181
- default: return; // Unknown type
168
+ try {
169
+ if (event.remote)
170
+ return;
171
+ const hlc = this.hlcManager.tick();
172
+ const { type, objectType, schemaName, objectName, ddl } = event;
173
+ let migrationType;
174
+ if (objectType === 'table') {
175
+ switch (type) {
176
+ case 'create':
177
+ migrationType = 'create_table';
178
+ break;
179
+ case 'drop':
180
+ migrationType = 'drop_table';
181
+ break;
182
+ case 'alter':
183
+ migrationType = 'alter_column';
184
+ break;
185
+ default: return;
186
+ }
182
187
  }
183
- }
184
- else if (objectType === 'index') {
185
- switch (type) {
186
- case 'create':
187
- migrationType = 'add_index';
188
- break;
189
- case 'drop':
190
- migrationType = 'drop_index';
191
- break;
192
- default: return; // Unknown type
188
+ else if (objectType === 'index') {
189
+ switch (type) {
190
+ case 'create':
191
+ migrationType = 'add_index';
192
+ break;
193
+ case 'drop':
194
+ migrationType = 'drop_index';
195
+ break;
196
+ default: return;
197
+ }
193
198
  }
199
+ else {
200
+ return;
201
+ }
202
+ const currentVersion = await this.schemaMigrations.getCurrentVersion(schemaName, objectName);
203
+ const newVersion = currentVersion + 1;
204
+ await this.schemaMigrations.recordMigration(schemaName, objectName, {
205
+ type: migrationType,
206
+ ddl: ddl || '',
207
+ hlc,
208
+ schemaVersion: newVersion,
209
+ });
210
+ // Persist HLC state (DRY: uses shared helper)
211
+ await persistHLCState(this);
212
+ this.syncEvents.emitLocalChange({
213
+ transactionId: crypto.randomUUID(),
214
+ changes: [],
215
+ pendingSync: true,
216
+ });
194
217
  }
195
- else {
196
- return; // Unknown object type
218
+ catch (error) {
219
+ console.error('[Sync] Error handling schema change:', error);
220
+ this.syncEvents.emitSyncStateChange({
221
+ status: 'error',
222
+ error: toError(error),
223
+ });
197
224
  }
198
- // Get next schema version for this table
199
- const currentVersion = await this.schemaMigrations.getCurrentVersion(schemaName, objectName);
200
- const newVersion = currentVersion + 1;
201
- // Record the migration
202
- await this.schemaMigrations.recordMigration(schemaName, objectName, {
203
- type: migrationType,
204
- ddl: ddl || '',
205
- hlc,
206
- schemaVersion: newVersion,
207
- });
208
- // Persist HLC state
209
- const hlcState = this.hlcManager.getState();
210
- const hlcBuffer = new Uint8Array(10);
211
- const hlcView = new DataView(hlcBuffer.buffer);
212
- hlcView.setBigUint64(0, hlcState.wallTime, false);
213
- hlcView.setUint16(8, hlcState.counter, false);
214
- await this.kv.put(SYNC_KEY_PREFIX.HLC_STATE, hlcBuffer);
215
- // Emit local change event for the schema migration
216
- this.syncEvents.emitLocalChange({
217
- transactionId: crypto.randomUUID(),
218
- changes: [],
219
- pendingSync: true,
220
- });
221
225
  }
222
226
  async recordColumnVersions(batch, schemaName, tableName, pk, oldRow, newRow, hlc) {
223
- // Try to get actual column names from schema
224
227
  const tableSchema = this.getTableSchema?.(schemaName, tableName);
225
228
  const columnNames = tableSchema?.columns?.map(c => c.name);
226
- // Debug logging for column name resolution
227
- if (!tableSchema) {
229
+ if (!tableSchema && this.getTableSchema) {
228
230
  console.warn(`[Sync] No table schema found for ${schemaName}.${tableName} - using fallback column names`);
229
231
  }
230
- // For each column that changed, record the new version
231
232
  for (let i = 0; i < newRow.length; i++) {
232
233
  const oldValue = oldRow?.[i];
233
234
  const newValue = newRow[i];
234
- // Only record if value changed (or it's an insert)
235
235
  if (!oldRow || oldValue !== newValue) {
236
- // Use actual column name if available, otherwise fall back to index-based
237
236
  const column = columnNames?.[i] ?? `col_${i}`;
238
- // Look up old column version to delete stale change log entry
239
237
  const oldVersion = await this.columnVersions.getColumnVersion(schemaName, tableName, pk, column);
240
- // Delete old change log entry if exists (keeps change log as a true secondary index)
241
238
  if (oldVersion) {
242
239
  this.changeLog.deleteEntryBatch(batch, oldVersion.hlc, 'column', schemaName, tableName, pk, column);
243
240
  }
244
- // Record new column version
245
241
  const version = { hlc, value: newValue };
246
242
  this.columnVersions.setColumnVersionBatch(batch, schemaName, tableName, pk, column, version);
247
- // Record in change log for efficient delta queries
248
243
  this.changeLog.recordColumnChangeBatch(batch, hlc, schemaName, tableName, pk, column);
249
244
  const change = {
250
245
  type: 'column',
@@ -259,27 +254,19 @@ export class SyncManagerImpl {
259
254
  }
260
255
  }
261
256
  }
262
- async persistHLCState() {
263
- const state = this.hlcManager.getState();
264
- const buffer = new Uint8Array(10);
265
- const view = new DataView(buffer.buffer);
266
- view.setBigUint64(0, state.wallTime, false);
267
- view.setUint16(8, state.counter, false);
268
- await this.kv.put(SYNC_KEY_PREFIX.HLC_STATE, buffer);
269
- }
257
+ // ============================================================================
258
+ // Delta Sync API
259
+ // ============================================================================
270
260
  async getChangesSince(peerSiteId, sinceHLC) {
271
261
  const changes = [];
272
262
  if (sinceHLC) {
273
- // Use change log for efficient delta query
274
263
  for await (const logEntry of this.changeLog.getChangesSince(sinceHLC)) {
275
- // Don't include changes from the requesting peer
276
264
  if (siteIdEquals(logEntry.hlc.siteId, peerSiteId))
277
265
  continue;
278
266
  if (logEntry.entryType === 'column') {
279
- // Look up the column version to get the value
280
267
  const cv = await this.columnVersions.getColumnVersion(logEntry.schema, logEntry.table, logEntry.pk, logEntry.column);
281
268
  if (!cv)
282
- continue; // May have been superseded
269
+ continue;
283
270
  const columnChange = {
284
271
  type: 'column',
285
272
  schema: logEntry.schema,
@@ -292,10 +279,9 @@ export class SyncManagerImpl {
292
279
  changes.push(columnChange);
293
280
  }
294
281
  else {
295
- // Look up tombstone for the deletion
296
282
  const tombstone = await this.tombstones.getTombstone(logEntry.schema, logEntry.table, logEntry.pk);
297
283
  if (!tombstone)
298
- continue; // May have been pruned
284
+ continue;
299
285
  const deletion = {
300
286
  type: 'delete',
301
287
  schema: logEntry.schema,
@@ -308,10 +294,9 @@ export class SyncManagerImpl {
308
294
  }
309
295
  }
310
296
  else {
311
- // No sinceHLC - need all changes (full scan fallback)
312
297
  await this.collectAllChanges(peerSiteId, changes);
313
298
  }
314
- // Collect schema migrations since sinceHLC
299
+ // Collect schema migrations
315
300
  const schemaMigrations = [];
316
301
  const smBounds = buildAllSchemaMigrationsScanBounds();
317
302
  for await (const entry of this.kv.iterate(smBounds)) {
@@ -319,10 +304,8 @@ export class SyncManagerImpl {
319
304
  if (!parsed)
320
305
  continue;
321
306
  const migration = deserializeMigration(entry.value);
322
- // Filter by HLC if provided
323
307
  if (sinceHLC && compareHLC(migration.hlc, sinceHLC) <= 0)
324
308
  continue;
325
- // Don't include changes from the requesting peer
326
309
  if (siteIdEquals(migration.hlc.siteId, peerSiteId))
327
310
  continue;
328
311
  schemaMigrations.push({
@@ -334,14 +317,10 @@ export class SyncManagerImpl {
334
317
  schemaVersion: migration.schemaVersion,
335
318
  });
336
319
  }
337
- // If no changes, return empty array
338
320
  if (changes.length === 0 && schemaMigrations.length === 0) {
339
321
  return [];
340
322
  }
341
- // Changes from change log are already in HLC order
342
- // Schema migrations need sorting
343
323
  schemaMigrations.sort((a, b) => compareHLC(a.hlc, b.hlc));
344
- // Batch changes up to config.batchSize
345
324
  const result = [];
346
325
  for (let i = 0; i < changes.length; i += this.config.batchSize) {
347
326
  const batch = changes.slice(i, i + this.config.batchSize);
@@ -351,10 +330,9 @@ export class SyncManagerImpl {
351
330
  transactionId: crypto.randomUUID(),
352
331
  hlc: maxHLC,
353
332
  changes: batch,
354
- schemaMigrations: i === 0 ? schemaMigrations : [], // Only include schema migrations in first batch
333
+ schemaMigrations: i === 0 ? schemaMigrations : [],
355
334
  });
356
335
  }
357
- // If no data changes but we have schema migrations, create a changeset for them
358
336
  if (result.length === 0 && schemaMigrations.length > 0) {
359
337
  const maxHLC = schemaMigrations.reduce((max, m) => compareHLC(m.hlc, max) > 0 ? m.hlc : max, schemaMigrations[0].hlc);
360
338
  result.push({
@@ -367,18 +345,13 @@ export class SyncManagerImpl {
367
345
  }
368
346
  return result;
369
347
  }
370
- /**
371
- * Fallback: collect all changes when no sinceHLC is provided.
372
- */
373
348
  async collectAllChanges(peerSiteId, changes) {
374
- // Collect all column changes
375
349
  const cvBounds = buildAllColumnVersionsScanBounds();
376
350
  for await (const entry of this.kv.iterate(cvBounds)) {
377
351
  const parsed = parseColumnVersionKey(entry.key);
378
352
  if (!parsed)
379
353
  continue;
380
354
  const cv = deserializeColumnVersion(entry.value);
381
- // Don't include changes from the requesting peer
382
355
  if (siteIdEquals(cv.hlc.siteId, peerSiteId))
383
356
  continue;
384
357
  const columnChange = {
@@ -392,14 +365,12 @@ export class SyncManagerImpl {
392
365
  };
393
366
  changes.push(columnChange);
394
367
  }
395
- // Collect all deletions (tombstones)
396
368
  const tbBounds = buildAllTombstonesScanBounds();
397
369
  for await (const entry of this.kv.iterate(tbBounds)) {
398
370
  const parsed = parseTombstoneKey(entry.key);
399
371
  if (!parsed)
400
372
  continue;
401
373
  const tombstone = deserializeTombstone(entry.value);
402
- // Don't include changes from the requesting peer
403
374
  if (siteIdEquals(tombstone.hlc.siteId, peerSiteId))
404
375
  continue;
405
376
  const deletion = {
@@ -411,418 +382,39 @@ export class SyncManagerImpl {
411
382
  };
412
383
  changes.push(deletion);
413
384
  }
414
- // Sort by HLC for consistent ordering
415
385
  changes.sort((a, b) => compareHLC(a.hlc, b.hlc));
416
386
  }
387
+ // ============================================================================
388
+ // Delegated: Change Application
389
+ // ============================================================================
417
390
  async applyChanges(changes) {
418
- let applied = 0;
419
- let skipped = 0;
420
- let conflicts = 0;
421
- // Collect changes to apply to the store (grouped by row for column merging)
422
- const dataChangesToApply = [];
423
- const schemaChangesToApply = [];
424
- const appliedChanges = [];
425
- // Track resolved changes for the commit phase (CRDT metadata)
426
- const resolvedDataChanges = [];
427
- // Track schema migrations that will be applied (for metadata commit)
428
- const pendingSchemaMigrations = [];
429
- // PHASE 1: Resolve all changes (no writes yet)
430
- for (const changeSet of changes) {
431
- // Update our clock with the remote HLC
432
- this.hlcManager.receive(changeSet.hlc);
433
- // Process schema migrations first (DDL before DML)
434
- for (const migration of changeSet.schemaMigrations) {
435
- // Use the incoming schemaVersion if provided, otherwise calculate next version
436
- const schemaVersion = migration.schemaVersion ??
437
- (await this.schemaMigrations.getCurrentVersion(migration.schema, migration.table)) + 1;
438
- // Check if we've already recorded this migration
439
- const existingMigration = await this.schemaMigrations.getMigration(migration.schema, migration.table, schemaVersion);
440
- if (existingMigration) {
441
- // Already have this migration version - skip
442
- // (HLC comparison for first-writer-wins is done via checkConflict if needed)
443
- if (compareHLC(migration.hlc, existingMigration.hlc) <= 0) {
444
- skipped++;
445
- continue;
446
- }
447
- }
448
- // Queue schema change for application (don't record metadata yet)
449
- schemaChangesToApply.push({
450
- type: migration.type,
451
- schema: migration.schema,
452
- table: migration.table,
453
- ddl: migration.ddl,
454
- });
455
- pendingSchemaMigrations.push({ migration, schemaVersion });
456
- applied++;
457
- }
458
- // Resolve data changes (no metadata writes)
459
- for (const change of changeSet.changes) {
460
- const resolved = await this.resolveChange(change, changeSet.siteId);
461
- if (resolved.outcome === 'applied') {
462
- applied++;
463
- appliedChanges.push({ change, siteId: changeSet.siteId });
464
- resolvedDataChanges.push(resolved);
465
- if (resolved.dataChange) {
466
- dataChangesToApply.push(resolved.dataChange);
467
- }
468
- }
469
- else if (resolved.outcome === 'skipped') {
470
- skipped++;
471
- }
472
- else if (resolved.outcome === 'conflict') {
473
- conflicts++;
474
- }
475
- }
476
- }
477
- // PHASE 2: Apply data and schema changes to the store via callback
478
- // This happens BEFORE writing CRDT metadata for crash safety
479
- if (this.applyToStore && (dataChangesToApply.length > 0 || schemaChangesToApply.length > 0)) {
480
- await this.applyToStore(dataChangesToApply, schemaChangesToApply, { remote: true });
481
- }
482
- // PHASE 3: Commit CRDT metadata (after data is safely written)
483
- // If crash occurs here, re-sync will re-apply same changes (idempotent)
484
- await this.commitChangeMetadata(resolvedDataChanges);
485
- // Commit schema migration metadata
486
- for (const { migration, schemaVersion } of pendingSchemaMigrations) {
487
- await this.schemaMigrations.recordMigration(migration.schema, migration.table, {
488
- type: migration.type,
489
- ddl: migration.ddl,
490
- hlc: migration.hlc,
491
- schemaVersion,
492
- });
493
- }
494
- // Emit a single remote change event with all applied changes for UI reactivity
495
- if (appliedChanges.length > 0) {
496
- // Group by siteId to emit one event per originating site
497
- const changesBySite = new Map();
498
- for (const { change, siteId } of appliedChanges) {
499
- const siteKey = Array.from(siteId).join(',');
500
- const siteChanges = changesBySite.get(siteKey);
501
- if (siteChanges) {
502
- siteChanges.push(change);
503
- }
504
- else {
505
- changesBySite.set(siteKey, [change]);
506
- }
507
- }
508
- const appliedAt = this.hlcManager.now();
509
- for (const [siteKey, siteChanges] of changesBySite) {
510
- const siteIdBytes = new Uint8Array(siteKey.split(',').map(Number));
511
- this.syncEvents.emitRemoteChange({
512
- siteId: siteIdBytes,
513
- transactionId: crypto.randomUUID(),
514
- changes: siteChanges,
515
- appliedAt,
516
- });
517
- }
518
- }
519
- await this.persistHLCState();
520
- return {
521
- applied,
522
- skipped,
523
- conflicts,
524
- transactions: changes.length,
525
- };
526
- }
527
- /**
528
- * Resolve CRDT conflicts for a change WITHOUT writing metadata.
529
- * This is phase 1 of the two-phase apply: resolve first, then after
530
- * data is written to store, call commitChangeMetadata to persist CRDT state.
531
- *
532
- * Write order (for crash safety):
533
- * 1. Resolve all changes (this method) - no writes
534
- * 2. Apply data to store (applyToStore callback)
535
- * 3. Commit CRDT metadata (commitChangeMetadata)
536
- *
537
- * If crash occurs after data but before metadata, re-sync will re-apply
538
- * the same changes. Since CRDT operations are idempotent (same HLC → same
539
- * LWW outcome), this is safe.
540
- */
541
- async resolveChange(change, _remoteSiteId) {
542
- // Skip changes that originated from ourselves (echo prevention).
543
- // This can happen when a peer re-sends changes it received from us,
544
- // or when changes propagate through a coordinator back to the originator.
545
- if (siteIdEquals(change.hlc.siteId, this.getSiteId())) {
546
- return { outcome: 'skipped', change };
547
- }
548
- if (change.type === 'delete') {
549
- // Check if we should apply this deletion
550
- const existingTombstone = await this.tombstones.getTombstone(change.schema, change.table, change.pk);
551
- if (existingTombstone && compareHLC(change.hlc, existingTombstone.hlc) <= 0) {
552
- return { outcome: 'skipped', change };
553
- }
554
- // Will apply - return the data change (metadata written in commit phase)
555
- return {
556
- outcome: 'applied',
557
- change,
558
- dataChange: {
559
- type: 'delete',
560
- schema: change.schema,
561
- table: change.table,
562
- pk: change.pk,
563
- },
564
- };
565
- }
566
- else {
567
- // Column change
568
- const shouldApply = await this.columnVersions.shouldApplyWrite(change.schema, change.table, change.pk, change.column, change.hlc);
569
- if (!shouldApply) {
570
- // Local version is newer - this is a conflict where local wins
571
- const localVersion = await this.columnVersions.getColumnVersion(change.schema, change.table, change.pk, change.column);
572
- if (localVersion) {
573
- const conflictEvent = {
574
- table: change.table,
575
- pk: change.pk,
576
- column: change.column,
577
- localValue: localVersion.value,
578
- remoteValue: change.value,
579
- winner: 'local',
580
- winningHLC: localVersion.hlc,
581
- };
582
- this.syncEvents.emitConflictResolved(conflictEvent);
583
- }
584
- return { outcome: 'conflict', change };
585
- }
586
- // Check for tombstone blocking
587
- const isBlocked = await this.tombstones.isDeletedAndBlocking(change.schema, change.table, change.pk, change.hlc, this.config.allowResurrection);
588
- if (isBlocked) {
589
- return { outcome: 'skipped', change };
590
- }
591
- // Get old column version for change log cleanup (done in commit phase)
592
- const oldColumnVersion = await this.columnVersions.getColumnVersion(change.schema, change.table, change.pk, change.column) ?? undefined;
593
- // Will apply - return the data change (metadata written in commit phase)
594
- return {
595
- outcome: 'applied',
596
- change,
597
- oldColumnVersion,
598
- dataChange: {
599
- type: 'update', // Column changes are updates (or inserts handled by store)
600
- schema: change.schema,
601
- table: change.table,
602
- pk: change.pk,
603
- columns: { [change.column]: change.value },
604
- },
605
- };
606
- }
607
- }
608
- /**
609
- * Commit CRDT metadata for resolved changes.
610
- * This is phase 2 of the two-phase apply: called AFTER data is written to store.
611
- */
612
- async commitChangeMetadata(resolvedChanges) {
613
- if (resolvedChanges.length === 0)
614
- return;
615
- const batch = this.kv.batch();
616
- for (const resolved of resolvedChanges) {
617
- if (resolved.outcome !== 'applied')
618
- continue;
619
- const change = resolved.change;
620
- if (change.type === 'delete') {
621
- // Record tombstone
622
- this.tombstones.setTombstoneBatch(batch, change.schema, change.table, change.pk, change.hlc);
623
- // Record in change log for delta sync
624
- this.changeLog.recordDeletionBatch(batch, change.hlc, change.schema, change.table, change.pk);
625
- // Note: column versions deletion is done outside the batch since it requires iteration
626
- }
627
- else {
628
- // Column change
629
- // Delete old change log entry if exists (keeps change log as a true secondary index)
630
- if (resolved.oldColumnVersion) {
631
- this.changeLog.deleteEntryBatch(batch, resolved.oldColumnVersion.hlc, 'column', change.schema, change.table, change.pk, change.column);
632
- }
633
- // Record the column version
634
- this.columnVersions.setColumnVersionBatch(batch, change.schema, change.table, change.pk, change.column, { hlc: change.hlc, value: change.value });
635
- // Record in change log for delta sync
636
- this.changeLog.recordColumnChangeBatch(batch, change.hlc, change.schema, change.table, change.pk, change.column);
637
- }
638
- }
639
- await batch.write();
640
- // Handle column version deletions for delete operations (requires async iteration)
641
- for (const resolved of resolvedChanges) {
642
- if (resolved.outcome !== 'applied')
643
- continue;
644
- if (resolved.change.type === 'delete') {
645
- await this.columnVersions.deleteRowVersions(resolved.change.schema, resolved.change.table, resolved.change.pk);
646
- }
647
- }
391
+ return applyChangesImpl(this, changes);
648
392
  }
649
393
  async canDeltaSync(peerSiteId, sinceHLC) {
650
394
  const peerState = await this.peerStates.getPeerState(peerSiteId);
651
395
  if (!peerState) {
652
- // Never synced with this peer - need full snapshot
653
396
  return false;
654
397
  }
655
- // Check if the sinceHLC is within tombstone TTL
398
+ // Check if tombstone TTL covers the requested time range
656
399
  const now = Date.now();
657
400
  const sinceTime = Number(sinceHLC.wallTime);
658
401
  if (now - sinceTime > this.config.tombstoneTTL) {
659
- // Too old - tombstones may have been pruned
660
402
  return false;
661
403
  }
662
404
  return true;
663
405
  }
406
+ // ============================================================================
407
+ // Delegated: Non-Streaming Snapshots
408
+ // ============================================================================
664
409
  async getSnapshot() {
665
- const tableData = new Map();
666
- const cvBounds = buildAllColumnVersionsScanBounds();
667
- for await (const entry of this.kv.iterate(cvBounds)) {
668
- const parsed = parseColumnVersionKey(entry.key);
669
- if (!parsed)
670
- continue;
671
- const cv = deserializeColumnVersion(entry.value);
672
- const tableKey = `${parsed.schema}.${parsed.table}`;
673
- const rowKey = encodePK(parsed.pk);
674
- if (!tableData.has(tableKey)) {
675
- tableData.set(tableKey, new Map());
676
- }
677
- const tableRows = tableData.get(tableKey);
678
- if (!tableRows.has(rowKey)) {
679
- tableRows.set(rowKey, new Map());
680
- }
681
- const rowVersions = tableRows.get(rowKey);
682
- rowVersions.set(parsed.column, cv);
683
- }
684
- // Build table snapshots
685
- const tables = [];
686
- for (const [tableKey, rows] of tableData) {
687
- const [schema, table] = tableKey.split('.');
688
- const columnVersions = new Map();
689
- const rowsArray = [];
690
- for (const [rowKey, rowVersions] of rows) {
691
- // Convert column map to array (row representation)
692
- const row = Array.from(rowVersions.values()).map(cv => cv.value);
693
- rowsArray.push(row);
694
- // Add column versions with their values
695
- for (const [column, cv] of rowVersions) {
696
- const versionKey = `${rowKey}:${column}`;
697
- columnVersions.set(versionKey, { hlc: cv.hlc, value: cv.value });
698
- }
699
- }
700
- tables.push({
701
- schema,
702
- table,
703
- rows: rowsArray,
704
- columnVersions,
705
- });
706
- }
707
- // Collect all schema migrations
708
- const schemaMigrations = [];
709
- const smBounds = buildAllSchemaMigrationsScanBounds();
710
- for await (const entry of this.kv.iterate(smBounds)) {
711
- const parsed = parseSchemaMigrationKey(entry.key);
712
- if (!parsed)
713
- continue;
714
- const migration = deserializeMigration(entry.value);
715
- schemaMigrations.push({
716
- type: migration.type,
717
- schema: parsed.schema,
718
- table: parsed.table,
719
- ddl: migration.ddl,
720
- hlc: migration.hlc,
721
- schemaVersion: migration.schemaVersion,
722
- });
723
- }
724
- return {
725
- siteId: this.getSiteId(),
726
- hlc: this.getCurrentHLC(),
727
- tables,
728
- schemaMigrations,
729
- };
410
+ return getSnapshotImpl(this);
730
411
  }
731
412
  async applySnapshot(snapshot) {
732
- // PHASE 1: Build data changes from snapshot (before any writes)
733
- const dataChangesToApply = [];
734
- const schemaChangesToApply = [];
735
- // Build schema migrations for the store
736
- for (const migration of snapshot.schemaMigrations) {
737
- schemaChangesToApply.push({
738
- type: migration.type,
739
- schema: migration.schema,
740
- table: migration.table,
741
- ddl: migration.ddl,
742
- });
743
- }
744
- // Build row data from column versions (grouped by pk)
745
- for (const tableSnapshot of snapshot.tables) {
746
- // Group column versions by pk to reconstruct rows
747
- const rowsByPk = new Map();
748
- for (const [versionKey, cvEntry] of tableSnapshot.columnVersions) {
749
- const lastColon = versionKey.lastIndexOf(':');
750
- if (lastColon === -1)
751
- continue;
752
- const rowKey = versionKey.slice(0, lastColon);
753
- const column = versionKey.slice(lastColon + 1);
754
- if (!rowsByPk.has(rowKey)) {
755
- rowsByPk.set(rowKey, {});
756
- }
757
- rowsByPk.get(rowKey)[column] = cvEntry.value;
758
- }
759
- // Create data change for each row
760
- for (const [rowKey, columns] of rowsByPk) {
761
- const pk = JSON.parse(rowKey);
762
- dataChangesToApply.push({
763
- type: 'update', // Snapshot rows are upserts
764
- schema: tableSnapshot.schema,
765
- table: tableSnapshot.table,
766
- pk,
767
- columns,
768
- });
769
- }
770
- }
771
- // PHASE 2: Apply data to store via callback (before metadata)
772
- if (this.applyToStore && (dataChangesToApply.length > 0 || schemaChangesToApply.length > 0)) {
773
- await this.applyToStore(dataChangesToApply, schemaChangesToApply, { remote: true });
774
- }
775
- // PHASE 3: Clear existing CRDT metadata and apply new metadata
776
- const clearBatch = this.kv.batch();
777
- // Delete all existing column versions
778
- const cvBounds = buildAllColumnVersionsScanBounds();
779
- for await (const entry of this.kv.iterate(cvBounds)) {
780
- clearBatch.delete(entry.key);
781
- }
782
- // Delete all existing tombstones
783
- const tbBounds = buildAllTombstonesScanBounds();
784
- for await (const entry of this.kv.iterate(tbBounds)) {
785
- clearBatch.delete(entry.key);
786
- }
787
- // Delete all existing change log entries
788
- const clBounds = buildAllChangeLogScanBounds();
789
- for await (const entry of this.kv.iterate(clBounds)) {
790
- clearBatch.delete(entry.key);
791
- }
792
- await clearBatch.write();
793
- // Apply the snapshot's column versions and rebuild change log
794
- const applyBatch = this.kv.batch();
795
- for (const tableSnapshot of snapshot.tables) {
796
- for (const [versionKey, cvEntry] of tableSnapshot.columnVersions) {
797
- const lastColon = versionKey.lastIndexOf(':');
798
- if (lastColon === -1)
799
- continue;
800
- const rowKey = versionKey.slice(0, lastColon);
801
- const column = versionKey.slice(lastColon + 1);
802
- const pk = JSON.parse(rowKey);
803
- this.columnVersions.setColumnVersionBatch(applyBatch, tableSnapshot.schema, tableSnapshot.table, pk, column, { hlc: cvEntry.hlc, value: cvEntry.value });
804
- // Rebuild change log entry
805
- this.changeLog.recordColumnChangeBatch(applyBatch, cvEntry.hlc, tableSnapshot.schema, tableSnapshot.table, pk, column);
806
- }
807
- }
808
- // Record schema migrations
809
- for (const migration of snapshot.schemaMigrations) {
810
- const schemaVersion = migration.schemaVersion ??
811
- (await this.schemaMigrations.getCurrentVersion(migration.schema, migration.table)) + 1;
812
- await this.schemaMigrations.recordMigration(migration.schema, migration.table, {
813
- type: migration.type,
814
- ddl: migration.ddl,
815
- hlc: migration.hlc,
816
- schemaVersion,
817
- });
818
- }
819
- await applyBatch.write();
820
- // Update our HLC to be at least as high as the snapshot
821
- this.hlcManager.receive(snapshot.hlc);
822
- await this.persistHLCState();
823
- // Emit sync state change
824
- this.syncEvents.emitSyncStateChange({ status: 'synced', lastSyncHLC: snapshot.hlc });
413
+ return applySnapshotImpl(this, snapshot);
825
414
  }
415
+ // ============================================================================
416
+ // Peer State & Maintenance
417
+ // ============================================================================
826
418
  async updatePeerSyncState(peerSiteId, hlc) {
827
419
  await this.peerStates.setPeerState(peerSiteId, hlc);
828
420
  }
@@ -845,463 +437,23 @@ export class SyncManagerImpl {
845
437
  await batch.write();
846
438
  return count;
847
439
  }
848
- /**
849
- * Get the sync event emitter for UI integration.
850
- */
851
440
  getEventEmitter() {
852
441
  return this.syncEvents;
853
442
  }
854
443
  // ============================================================================
855
- // Streaming Snapshot API
444
+ // Delegated: Streaming Snapshot API
856
445
  // ============================================================================
857
- async *getSnapshotStream(chunkSize = DEFAULT_SNAPSHOT_CHUNK_SIZE) {
858
- const snapshotId = crypto.randomUUID();
859
- const siteId = this.getSiteId();
860
- const hlc = this.getCurrentHLC();
861
- // Count tables and migrations for header
862
- const tableKeys = new Set();
863
- const cvBounds = buildAllColumnVersionsScanBounds();
864
- for await (const entry of this.kv.iterate(cvBounds)) {
865
- const parsed = parseColumnVersionKey(entry.key);
866
- if (parsed)
867
- tableKeys.add(`${parsed.schema}.${parsed.table}`);
868
- }
869
- let migrationCount = 0;
870
- const smBounds = buildAllSchemaMigrationsScanBounds();
871
- for await (const _entry of this.kv.iterate(smBounds)) {
872
- migrationCount++;
873
- }
874
- // Yield header
875
- const header = {
876
- type: 'header',
877
- siteId,
878
- hlc,
879
- tableCount: tableKeys.size,
880
- migrationCount,
881
- snapshotId,
882
- };
883
- yield header;
884
- // Stream each table
885
- let totalEntries = 0;
886
- for (const tableKey of tableKeys) {
887
- const [schema, table] = tableKey.split('.');
888
- // Estimate entries for this table
889
- let tableEntryCount = 0;
890
- const tableCvBounds = buildAllColumnVersionsScanBounds();
891
- for await (const entry of this.kv.iterate(tableCvBounds)) {
892
- const parsed = parseColumnVersionKey(entry.key);
893
- if (parsed && parsed.schema === schema && parsed.table === table) {
894
- tableEntryCount++;
895
- }
896
- }
897
- // Yield table start
898
- const tableStart = {
899
- type: 'table-start',
900
- schema,
901
- table,
902
- estimatedEntries: tableEntryCount,
903
- };
904
- yield tableStart;
905
- // Stream column versions in chunks
906
- let entries = [];
907
- let entriesWritten = 0;
908
- for await (const entry of this.kv.iterate(tableCvBounds)) {
909
- const parsed = parseColumnVersionKey(entry.key);
910
- if (!parsed || parsed.schema !== schema || parsed.table !== table)
911
- continue;
912
- const cv = deserializeColumnVersion(entry.value);
913
- const versionKey = `${encodePK(parsed.pk)}:${parsed.column}`;
914
- entries.push([versionKey, cv.hlc, cv.value]);
915
- entriesWritten++;
916
- if (entries.length >= chunkSize) {
917
- const chunk = {
918
- type: 'column-versions',
919
- schema,
920
- table,
921
- entries,
922
- };
923
- yield chunk;
924
- entries = [];
925
- }
926
- }
927
- // Yield remaining entries
928
- if (entries.length > 0) {
929
- const chunk = {
930
- type: 'column-versions',
931
- schema,
932
- table,
933
- entries,
934
- };
935
- yield chunk;
936
- }
937
- // Yield table end
938
- const tableEnd = {
939
- type: 'table-end',
940
- schema,
941
- table,
942
- entriesWritten,
943
- };
944
- yield tableEnd;
945
- totalEntries += entriesWritten;
946
- }
947
- // Stream schema migrations
948
- for await (const entry of this.kv.iterate(smBounds)) {
949
- const parsed = parseSchemaMigrationKey(entry.key);
950
- if (!parsed)
951
- continue;
952
- const migration = deserializeMigration(entry.value);
953
- const migrationChunk = {
954
- type: 'schema-migration',
955
- migration: {
956
- type: migration.type,
957
- schema: parsed.schema,
958
- table: parsed.table,
959
- ddl: migration.ddl,
960
- hlc: migration.hlc,
961
- schemaVersion: migration.schemaVersion,
962
- },
963
- };
964
- yield migrationChunk;
965
- }
966
- // Yield footer
967
- const footer = {
968
- type: 'footer',
969
- snapshotId,
970
- totalTables: tableKeys.size,
971
- totalEntries,
972
- totalMigrations: migrationCount,
973
- };
974
- yield footer;
446
+ async *getSnapshotStream(chunkSize) {
447
+ yield* getSnapshotStreamImpl(this, chunkSize);
975
448
  }
976
449
  async applySnapshotStream(chunks, onProgress) {
977
- let snapshotId;
978
- let snapshotHLC;
979
- let totalTables = 0;
980
- let totalEntries = 0;
981
- let tablesProcessed = 0;
982
- let entriesProcessed = 0;
983
- let currentTable;
984
- const completedTables = [];
985
- // Pending data to apply to store (batched for efficiency)
986
- let pendingDataChanges = [];
987
- let pendingSchemaChanges = [];
988
- const DATA_FLUSH_SIZE = 100;
989
- // Helper to flush pending data changes to store
990
- const flushDataToStore = async () => {
991
- if (this.applyToStore && (pendingDataChanges.length > 0 || pendingSchemaChanges.length > 0)) {
992
- await this.applyToStore(pendingDataChanges, pendingSchemaChanges, { remote: true });
993
- pendingDataChanges = [];
994
- pendingSchemaChanges = [];
995
- }
996
- };
997
- // Clear existing metadata before applying
998
- const clearBatch = this.kv.batch();
999
- for await (const entry of this.kv.iterate(buildAllColumnVersionsScanBounds())) {
1000
- clearBatch.delete(entry.key);
1001
- }
1002
- for await (const entry of this.kv.iterate(buildAllTombstonesScanBounds())) {
1003
- clearBatch.delete(entry.key);
1004
- }
1005
- for await (const entry of this.kv.iterate(buildAllChangeLogScanBounds())) {
1006
- clearBatch.delete(entry.key);
1007
- }
1008
- await clearBatch.write();
1009
- // Process chunks - track column versions per row for grouping
1010
- let batch = this.kv.batch();
1011
- let batchSize = 0;
1012
- const BATCH_FLUSH_SIZE = 1000;
1013
- // Track current table's rows for data application
1014
- let currentTableSchema;
1015
- let currentTableName;
1016
- const rowColumns = new Map();
1017
- for await (const chunk of chunks) {
1018
- switch (chunk.type) {
1019
- case 'header':
1020
- snapshotId = chunk.snapshotId;
1021
- snapshotHLC = chunk.hlc;
1022
- totalTables = chunk.tableCount;
1023
- break;
1024
- case 'table-start':
1025
- currentTable = `${chunk.schema}.${chunk.table}`;
1026
- currentTableSchema = chunk.schema;
1027
- currentTableName = chunk.table;
1028
- totalEntries += chunk.estimatedEntries;
1029
- rowColumns.clear();
1030
- break;
1031
- case 'column-versions':
1032
- for (const [versionKey, hlc, value] of chunk.entries) {
1033
- const lastColon = versionKey.lastIndexOf(':');
1034
- if (lastColon === -1)
1035
- continue;
1036
- const rowKey = versionKey.slice(0, lastColon);
1037
- const column = versionKey.slice(lastColon + 1);
1038
- const pk = JSON.parse(rowKey);
1039
- // Track column for data application
1040
- if (!rowColumns.has(rowKey)) {
1041
- rowColumns.set(rowKey, {});
1042
- }
1043
- rowColumns.get(rowKey)[column] = value;
1044
- // Write CRDT metadata
1045
- this.columnVersions.setColumnVersionBatch(batch, chunk.schema, chunk.table, pk, column, { hlc, value });
1046
- this.changeLog.recordColumnChangeBatch(batch, hlc, chunk.schema, chunk.table, pk, column);
1047
- batchSize++;
1048
- entriesProcessed++;
1049
- if (batchSize >= BATCH_FLUSH_SIZE) {
1050
- await batch.write();
1051
- batch = this.kv.batch();
1052
- batchSize = 0;
1053
- // Save checkpoint with completed tables
1054
- if (snapshotId && snapshotHLC) {
1055
- await this.saveSnapshotCheckpoint({
1056
- snapshotId,
1057
- siteId: this.getSiteId(),
1058
- hlc: snapshotHLC,
1059
- lastTableIndex: tablesProcessed,
1060
- lastEntryIndex: entriesProcessed,
1061
- completedTables: [...completedTables],
1062
- entriesProcessed,
1063
- createdAt: Date.now(),
1064
- });
1065
- }
1066
- }
1067
- }
1068
- if (onProgress && snapshotId) {
1069
- onProgress({
1070
- snapshotId,
1071
- tablesProcessed,
1072
- totalTables,
1073
- entriesProcessed,
1074
- totalEntries,
1075
- currentTable,
1076
- });
1077
- }
1078
- break;
1079
- case 'table-end':
1080
- // Flush accumulated rows to store
1081
- if (currentTableSchema && currentTableName) {
1082
- for (const [rowKey, columns] of rowColumns) {
1083
- const pk = JSON.parse(rowKey);
1084
- pendingDataChanges.push({
1085
- type: 'update',
1086
- schema: currentTableSchema,
1087
- table: currentTableName,
1088
- pk,
1089
- columns,
1090
- });
1091
- if (pendingDataChanges.length >= DATA_FLUSH_SIZE) {
1092
- await flushDataToStore();
1093
- }
1094
- }
1095
- rowColumns.clear();
1096
- }
1097
- tablesProcessed++;
1098
- if (currentTable) {
1099
- completedTables.push(currentTable);
1100
- }
1101
- break;
1102
- case 'schema-migration': {
1103
- const migration = chunk.migration;
1104
- pendingSchemaChanges.push({
1105
- type: migration.type,
1106
- schema: migration.schema,
1107
- table: migration.table,
1108
- ddl: migration.ddl,
1109
- });
1110
- // Record migration metadata
1111
- const schemaVersion = migration.schemaVersion ??
1112
- (await this.schemaMigrations.getCurrentVersion(migration.schema, migration.table)) + 1;
1113
- await this.schemaMigrations.recordMigration(migration.schema, migration.table, {
1114
- type: migration.type,
1115
- ddl: migration.ddl,
1116
- hlc: migration.hlc,
1117
- schemaVersion,
1118
- });
1119
- break;
1120
- }
1121
- case 'footer':
1122
- // Flush remaining data to store
1123
- await flushDataToStore();
1124
- // Flush remaining metadata batch
1125
- if (batchSize > 0) {
1126
- await batch.write();
1127
- }
1128
- // Update HLC
1129
- if (snapshotHLC) {
1130
- this.hlcManager.receive(snapshotHLC);
1131
- await this.persistHLCState();
1132
- }
1133
- // Clear checkpoint
1134
- if (snapshotId) {
1135
- await this.clearSnapshotCheckpoint(snapshotId);
1136
- }
1137
- // Emit sync state change
1138
- if (snapshotHLC) {
1139
- this.syncEvents.emitSyncStateChange({ status: 'synced', lastSyncHLC: snapshotHLC });
1140
- }
1141
- break;
1142
- }
1143
- }
450
+ return applySnapshotStreamImpl(this, chunks, onProgress);
1144
451
  }
1145
452
  async getSnapshotCheckpoint(snapshotId) {
1146
- const key = new TextEncoder().encode(`${CHECKPOINT_PREFIX}${snapshotId}`);
1147
- const data = await this.kv.get(key);
1148
- if (!data)
1149
- return undefined;
1150
- const json = new TextDecoder().decode(data);
1151
- const obj = JSON.parse(json);
1152
- // Reconstruct HLC with proper types
1153
- return {
1154
- ...obj,
1155
- hlc: {
1156
- wallTime: BigInt(obj.hlc.wallTime),
1157
- counter: obj.hlc.counter,
1158
- siteId: new Uint8Array(obj.hlc.siteId),
1159
- },
1160
- siteId: new Uint8Array(obj.siteId),
1161
- };
1162
- }
1163
- async saveSnapshotCheckpoint(checkpoint) {
1164
- const key = new TextEncoder().encode(`${CHECKPOINT_PREFIX}${checkpoint.snapshotId}`);
1165
- const json = JSON.stringify({
1166
- ...checkpoint,
1167
- hlc: {
1168
- wallTime: checkpoint.hlc.wallTime.toString(),
1169
- counter: checkpoint.hlc.counter,
1170
- siteId: Array.from(checkpoint.hlc.siteId),
1171
- },
1172
- siteId: Array.from(checkpoint.siteId),
1173
- });
1174
- await this.kv.put(key, new TextEncoder().encode(json));
1175
- }
1176
- async clearSnapshotCheckpoint(snapshotId) {
1177
- const key = new TextEncoder().encode(`${CHECKPOINT_PREFIX}${snapshotId}`);
1178
- await this.kv.delete(key);
453
+ return getSnapshotCheckpointImpl(this, snapshotId);
1179
454
  }
1180
455
  async *resumeSnapshotStream(checkpoint) {
1181
- // Resume streaming from checkpoint position
1182
- // Skip tables that have already been completed
1183
- const completedSet = new Set(checkpoint.completedTables);
1184
- const snapshotId = checkpoint.snapshotId;
1185
- const siteId = checkpoint.siteId;
1186
- const hlc = checkpoint.hlc;
1187
- // Count tables and migrations for header
1188
- const tableKeys = new Set();
1189
- const cvBounds = buildAllColumnVersionsScanBounds();
1190
- for await (const entry of this.kv.iterate(cvBounds)) {
1191
- const parsed = parseColumnVersionKey(entry.key);
1192
- if (parsed)
1193
- tableKeys.add(`${parsed.schema}.${parsed.table}`);
1194
- }
1195
- let migrationCount = 0;
1196
- const smBounds = buildAllSchemaMigrationsScanBounds();
1197
- for await (const _entry of this.kv.iterate(smBounds)) {
1198
- migrationCount++;
1199
- }
1200
- // Yield header (receiver needs to know this is a resume)
1201
- const header = {
1202
- type: 'header',
1203
- siteId,
1204
- hlc,
1205
- tableCount: tableKeys.size,
1206
- migrationCount,
1207
- snapshotId,
1208
- };
1209
- yield header;
1210
- // Stream each table, skipping completed ones
1211
- let totalEntries = checkpoint.entriesProcessed;
1212
- for (const tableKey of tableKeys) {
1213
- // Skip already completed tables
1214
- if (completedSet.has(tableKey))
1215
- continue;
1216
- const [schema, table] = tableKey.split('.');
1217
- // Count entries for this table
1218
- let tableEntryCount = 0;
1219
- const tableCvBounds = buildAllColumnVersionsScanBounds();
1220
- for await (const entry of this.kv.iterate(tableCvBounds)) {
1221
- const parsed = parseColumnVersionKey(entry.key);
1222
- if (parsed && parsed.schema === schema && parsed.table === table) {
1223
- tableEntryCount++;
1224
- }
1225
- }
1226
- // Yield table start
1227
- const tableStart = {
1228
- type: 'table-start',
1229
- schema,
1230
- table,
1231
- estimatedEntries: tableEntryCount,
1232
- };
1233
- yield tableStart;
1234
- // Stream column versions in chunks
1235
- let entries = [];
1236
- let entriesWritten = 0;
1237
- const chunkSize = DEFAULT_SNAPSHOT_CHUNK_SIZE;
1238
- for await (const entry of this.kv.iterate(tableCvBounds)) {
1239
- const parsed = parseColumnVersionKey(entry.key);
1240
- if (!parsed || parsed.schema !== schema || parsed.table !== table)
1241
- continue;
1242
- const cv = deserializeColumnVersion(entry.value);
1243
- const versionKey = `${encodePK(parsed.pk)}:${parsed.column}`;
1244
- entries.push([versionKey, cv.hlc, cv.value]);
1245
- entriesWritten++;
1246
- if (entries.length >= chunkSize) {
1247
- const chunk = {
1248
- type: 'column-versions',
1249
- schema,
1250
- table,
1251
- entries,
1252
- };
1253
- yield chunk;
1254
- entries = [];
1255
- }
1256
- }
1257
- // Yield remaining entries
1258
- if (entries.length > 0) {
1259
- const chunk = {
1260
- type: 'column-versions',
1261
- schema,
1262
- table,
1263
- entries,
1264
- };
1265
- yield chunk;
1266
- }
1267
- // Yield table end
1268
- const tableEnd = {
1269
- type: 'table-end',
1270
- schema,
1271
- table,
1272
- entriesWritten,
1273
- };
1274
- yield tableEnd;
1275
- totalEntries += entriesWritten;
1276
- }
1277
- // Stream schema migrations
1278
- for await (const entry of this.kv.iterate(smBounds)) {
1279
- const parsed = parseSchemaMigrationKey(entry.key);
1280
- if (!parsed)
1281
- continue;
1282
- const migration = deserializeMigration(entry.value);
1283
- const migrationChunk = {
1284
- type: 'schema-migration',
1285
- migration: {
1286
- type: migration.type,
1287
- schema: parsed.schema,
1288
- table: parsed.table,
1289
- ddl: migration.ddl,
1290
- hlc: migration.hlc,
1291
- schemaVersion: migration.schemaVersion,
1292
- },
1293
- };
1294
- yield migrationChunk;
1295
- }
1296
- // Yield footer
1297
- const footer = {
1298
- type: 'footer',
1299
- snapshotId,
1300
- totalTables: tableKeys.size,
1301
- totalEntries,
1302
- totalMigrations: migrationCount,
1303
- };
1304
- yield footer;
456
+ yield* resumeSnapshotStreamImpl(this, checkpoint);
1305
457
  }
1306
458
  }
1307
459
  //# sourceMappingURL=sync-manager-impl.js.map