@syncular/server 0.0.1-73 → 0.0.1-88

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/blobs/adapters/database.d.ts.map +1 -1
  2. package/dist/blobs/adapters/database.js +25 -3
  3. package/dist/blobs/adapters/database.js.map +1 -1
  4. package/dist/blobs/index.js +5 -5
  5. package/dist/dialect/base.js +1 -1
  6. package/dist/dialect/index.js +3 -3
  7. package/dist/helpers/index.js +4 -4
  8. package/dist/index.js +16 -16
  9. package/dist/proxy/handler.d.ts.map +1 -1
  10. package/dist/proxy/handler.js +7 -4
  11. package/dist/proxy/handler.js.map +1 -1
  12. package/dist/proxy/index.js +3 -3
  13. package/dist/proxy/mutation-detector.d.ts +4 -0
  14. package/dist/proxy/mutation-detector.d.ts.map +1 -1
  15. package/dist/proxy/mutation-detector.js +209 -24
  16. package/dist/proxy/mutation-detector.js.map +1 -1
  17. package/dist/pull.d.ts +1 -1
  18. package/dist/pull.d.ts.map +1 -1
  19. package/dist/pull.js +445 -322
  20. package/dist/pull.js.map +1 -1
  21. package/dist/push.d.ts +1 -1
  22. package/dist/push.d.ts.map +1 -1
  23. package/dist/push.js +341 -260
  24. package/dist/push.js.map +1 -1
  25. package/dist/realtime/index.js +1 -1
  26. package/dist/shapes/index.js +3 -3
  27. package/dist/snapshot-chunks/adapters/s3.js +1 -1
  28. package/dist/snapshot-chunks/db-metadata.d.ts.map +1 -1
  29. package/dist/snapshot-chunks/db-metadata.js +47 -41
  30. package/dist/snapshot-chunks/db-metadata.js.map +1 -1
  31. package/dist/snapshot-chunks/index.js +3 -3
  32. package/dist/subscriptions/index.js +1 -1
  33. package/package.json +1 -1
  34. package/src/blobs/adapters/database.test.ts +67 -0
  35. package/src/blobs/adapters/database.ts +34 -9
  36. package/src/proxy/handler.test.ts +120 -0
  37. package/src/proxy/handler.ts +9 -2
  38. package/src/proxy/mutation-detector.test.ts +71 -0
  39. package/src/proxy/mutation-detector.ts +227 -29
  40. package/src/pull.ts +609 -418
  41. package/src/push.ts +473 -349
  42. package/src/snapshot-chunks/db-metadata.test.ts +100 -0
  43. package/src/snapshot-chunks/db-metadata.ts +68 -48
package/src/pull.ts CHANGED
@@ -1,15 +1,19 @@
1
1
  import { createHash, randomUUID } from 'node:crypto';
2
2
  import { promisify } from 'node:util';
3
3
  import { gzip, gzipSync } from 'node:zlib';
4
- import type {
5
- ScopeValues,
6
- SyncBootstrapState,
7
- SyncChange,
8
- SyncCommit,
9
- SyncPullRequest,
10
- SyncPullResponse,
11
- SyncPullSubscriptionResponse,
12
- SyncSnapshot,
4
+ import {
5
+ captureSyncException,
6
+ countSyncMetric,
7
+ distributionSyncMetric,
8
+ type ScopeValues,
9
+ type SyncBootstrapState,
10
+ type SyncChange,
11
+ type SyncCommit,
12
+ type SyncPullRequest,
13
+ type SyncPullResponse,
14
+ type SyncPullSubscriptionResponse,
15
+ type SyncSnapshot,
16
+ startSyncSpan,
13
17
  } from '@syncular/core';
14
18
  import type { Kysely } from 'kysely';
15
19
  import type { ServerSyncDialect } from './dialect/types';
@@ -96,6 +100,106 @@ function mergeScopes(subscriptions: { scopes: ScopeValues }[]): ScopeValues {
96
100
  return merged;
97
101
  }
98
102
 
103
+ interface PullResponseStats {
104
+ subscriptionCount: number;
105
+ activeSubscriptionCount: number;
106
+ revokedSubscriptionCount: number;
107
+ bootstrapSubscriptionCount: number;
108
+ commitCount: number;
109
+ changeCount: number;
110
+ snapshotPageCount: number;
111
+ }
112
+
113
+ function summarizePullResponse(response: SyncPullResponse): PullResponseStats {
114
+ const subscriptions = response.subscriptions ?? [];
115
+ let activeSubscriptionCount = 0;
116
+ let revokedSubscriptionCount = 0;
117
+ let bootstrapSubscriptionCount = 0;
118
+ let commitCount = 0;
119
+ let changeCount = 0;
120
+ let snapshotPageCount = 0;
121
+
122
+ for (const sub of subscriptions) {
123
+ if (sub.status === 'revoked') {
124
+ revokedSubscriptionCount += 1;
125
+ } else {
126
+ activeSubscriptionCount += 1;
127
+ }
128
+
129
+ if (sub.bootstrap) {
130
+ bootstrapSubscriptionCount += 1;
131
+ }
132
+
133
+ const commits = sub.commits ?? [];
134
+ commitCount += commits.length;
135
+ for (const commit of commits) {
136
+ changeCount += commit.changes?.length ?? 0;
137
+ }
138
+
139
+ snapshotPageCount += sub.snapshots?.length ?? 0;
140
+ }
141
+
142
+ return {
143
+ subscriptionCount: subscriptions.length,
144
+ activeSubscriptionCount,
145
+ revokedSubscriptionCount,
146
+ bootstrapSubscriptionCount,
147
+ commitCount,
148
+ changeCount,
149
+ snapshotPageCount,
150
+ };
151
+ }
152
+
153
+ function recordPullMetrics(args: {
154
+ status: string;
155
+ dedupeRows: boolean;
156
+ durationMs: number;
157
+ stats: PullResponseStats;
158
+ }): void {
159
+ const { status, dedupeRows, durationMs, stats } = args;
160
+ const attributes = {
161
+ status,
162
+ dedupe_rows: dedupeRows,
163
+ };
164
+
165
+ countSyncMetric('sync.server.pull.requests', 1, { attributes });
166
+ distributionSyncMetric('sync.server.pull.duration_ms', durationMs, {
167
+ unit: 'millisecond',
168
+ attributes,
169
+ });
170
+ distributionSyncMetric(
171
+ 'sync.server.pull.subscriptions',
172
+ stats.subscriptionCount,
173
+ { attributes }
174
+ );
175
+ distributionSyncMetric(
176
+ 'sync.server.pull.active_subscriptions',
177
+ stats.activeSubscriptionCount,
178
+ { attributes }
179
+ );
180
+ distributionSyncMetric(
181
+ 'sync.server.pull.revoked_subscriptions',
182
+ stats.revokedSubscriptionCount,
183
+ { attributes }
184
+ );
185
+ distributionSyncMetric(
186
+ 'sync.server.pull.bootstrap_subscriptions',
187
+ stats.bootstrapSubscriptionCount,
188
+ { attributes }
189
+ );
190
+ distributionSyncMetric('sync.server.pull.commits', stats.commitCount, {
191
+ attributes,
192
+ });
193
+ distributionSyncMetric('sync.server.pull.changes', stats.changeCount, {
194
+ attributes,
195
+ });
196
+ distributionSyncMetric(
197
+ 'sync.server.pull.snapshot_pages',
198
+ stats.snapshotPageCount,
199
+ { attributes }
200
+ );
201
+ }
202
+
99
203
  export async function pull<DB extends SyncCoreDb>(args: {
100
204
  db: Kysely<DB>;
101
205
  dialect: ServerSyncDialect;
@@ -113,439 +217,526 @@ export async function pull<DB extends SyncCoreDb>(args: {
113
217
  const { request, dialect } = args;
114
218
  const db = args.db;
115
219
  const partitionId = args.partitionId ?? 'default';
116
-
117
- // Validate and sanitize request limits
118
- const limitCommits = sanitizeLimit(request.limitCommits, 50, 1, 500);
119
- const limitSnapshotRows = sanitizeLimit(
120
- request.limitSnapshotRows,
121
- 1000,
122
- 1,
123
- 5000
124
- );
125
- const maxSnapshotPages = sanitizeLimit(request.maxSnapshotPages, 1, 1, 50);
126
- const dedupeRows = request.dedupeRows === true;
127
-
128
- // Resolve effective scopes for each subscription
129
- const resolved = await resolveEffectiveScopesForSubscriptions({
130
- db,
131
- actorId: args.actorId,
132
- subscriptions: request.subscriptions ?? [],
133
- shapes: args.shapes,
134
- });
135
-
136
- return dialect.executeInTransaction(db, async (trx) => {
137
- await dialect.setRepeatableRead(trx);
138
-
139
- const maxCommitSeq = await dialect.readMaxCommitSeq(trx, { partitionId });
140
- const minCommitSeq = await dialect.readMinCommitSeq(trx, { partitionId });
141
-
142
- const subResponses: SyncPullSubscriptionResponse[] = [];
143
- const activeSubscriptions: { scopes: ScopeValues }[] = [];
144
- const nextCursors: number[] = [];
145
-
146
- for (const sub of resolved) {
147
- const cursor = Math.max(-1, sub.cursor ?? -1);
148
- // Validate shape exists (throws if not registered)
149
- args.shapes.getOrThrow(sub.shape);
150
-
151
- if (sub.status === 'revoked' || Object.keys(sub.scopes).length === 0) {
152
- subResponses.push({
153
- id: sub.id,
154
- status: 'revoked',
155
- scopes: {},
156
- bootstrap: false,
157
- nextCursor: cursor,
158
- commits: [],
220
+ const requestedSubscriptionCount = Array.isArray(request.subscriptions)
221
+ ? request.subscriptions.length
222
+ : 0;
223
+ const startedAtMs = Date.now();
224
+
225
+ return startSyncSpan(
226
+ {
227
+ name: 'sync.server.pull',
228
+ op: 'sync.pull',
229
+ attributes: {
230
+ requested_subscription_count: requestedSubscriptionCount,
231
+ dedupe_rows: request.dedupeRows === true,
232
+ },
233
+ },
234
+ async (span) => {
235
+ try {
236
+ // Validate and sanitize request limits
237
+ const limitCommits = sanitizeLimit(request.limitCommits, 50, 1, 500);
238
+ const limitSnapshotRows = sanitizeLimit(
239
+ request.limitSnapshotRows,
240
+ 1000,
241
+ 1,
242
+ 5000
243
+ );
244
+ const maxSnapshotPages = sanitizeLimit(
245
+ request.maxSnapshotPages,
246
+ 1,
247
+ 1,
248
+ 50
249
+ );
250
+ const dedupeRows = request.dedupeRows === true;
251
+
252
+ // Resolve effective scopes for each subscription
253
+ const resolved = await resolveEffectiveScopesForSubscriptions({
254
+ db,
255
+ actorId: args.actorId,
256
+ subscriptions: request.subscriptions ?? [],
257
+ shapes: args.shapes,
159
258
  });
160
- continue;
161
- }
162
-
163
- const effectiveScopes = sub.scopes;
164
- activeSubscriptions.push({ scopes: effectiveScopes });
165
-
166
- const needsBootstrap =
167
- sub.bootstrapState != null ||
168
- cursor < 0 ||
169
- cursor > maxCommitSeq ||
170
- (minCommitSeq > 0 && cursor < minCommitSeq - 1);
171
-
172
- if (needsBootstrap) {
173
- const tables = args.shapes
174
- .getBootstrapOrderFor(sub.shape)
175
- .map((handler) => handler.table);
176
-
177
- const initState: SyncBootstrapState = {
178
- asOfCommitSeq: maxCommitSeq,
179
- tables,
180
- tableIndex: 0,
181
- rowCursor: null,
182
- };
183
-
184
- const requestedState = sub.bootstrapState ?? null;
185
- const state =
186
- requestedState &&
187
- typeof requestedState.asOfCommitSeq === 'number' &&
188
- Array.isArray(requestedState.tables) &&
189
- typeof requestedState.tableIndex === 'number'
190
- ? (requestedState as SyncBootstrapState)
191
- : initState;
192
-
193
- // If the bootstrap state's asOfCommitSeq is no longer catch-up-able, restart bootstrap.
194
- const effectiveState =
195
- state.asOfCommitSeq < minCommitSeq - 1 ? initState : state;
196
-
197
- const tableName = effectiveState.tables[effectiveState.tableIndex];
198
-
199
- // No tables (or ran past the end): treat bootstrap as complete.
200
- if (!tableName) {
201
- subResponses.push({
202
- id: sub.id,
203
- status: 'active',
204
- scopes: effectiveScopes,
205
- bootstrap: true,
206
- bootstrapState: null,
207
- nextCursor: effectiveState.asOfCommitSeq,
208
- commits: [],
209
- snapshots: [],
210
- });
211
- nextCursors.push(effectiveState.asOfCommitSeq);
212
- continue;
213
- }
214
-
215
- const snapshots: SyncSnapshot[] = [];
216
- let nextState: SyncBootstrapState | null = effectiveState;
217
-
218
- for (let pageIndex = 0; pageIndex < maxSnapshotPages; pageIndex++) {
219
- if (!nextState) break;
220
259
 
221
- const nextTableName = nextState.tables[nextState.tableIndex];
222
- if (!nextTableName) {
223
- nextState = null;
224
- break;
225
- }
226
-
227
- const tableHandler = args.shapes.getOrThrow(nextTableName);
228
- const isFirstPage = nextState.rowCursor == null;
260
+ const result = await dialect.executeInTransaction(db, async (trx) => {
261
+ await dialect.setRepeatableRead(trx);
229
262
 
230
- const page = await tableHandler.snapshot(
231
- {
232
- db: trx,
233
- actorId: args.actorId,
234
- scopeValues: effectiveScopes,
235
- cursor: nextState.rowCursor,
236
- limit: limitSnapshotRows,
237
- },
238
- sub.params
239
- );
240
-
241
- const isLastPage = page.nextCursor == null;
242
-
243
- // Always use NDJSON+gzip for bootstrap snapshots
244
- const ttlMs = tableHandler.snapshotChunkTtlMs ?? 24 * 60 * 60 * 1000; // 24h
245
- const nowIso = new Date().toISOString();
246
-
247
- // Use scope hash for caching
248
- const cacheKey = `${partitionId}:${scopesToCacheKey(effectiveScopes)}`;
249
- const cached = await readSnapshotChunkRefByPageKey(trx, {
263
+ const maxCommitSeq = await dialect.readMaxCommitSeq(trx, {
264
+ partitionId,
265
+ });
266
+ const minCommitSeq = await dialect.readMinCommitSeq(trx, {
250
267
  partitionId,
251
- scopeKey: cacheKey,
252
- scope: nextTableName,
253
- asOfCommitSeq: effectiveState.asOfCommitSeq,
254
- rowCursor: nextState.rowCursor,
255
- rowLimit: limitSnapshotRows,
256
- encoding: 'ndjson',
257
- compression: 'gzip',
258
- nowIso,
259
268
  });
260
269
 
261
- let chunkRef = cached;
262
-
263
- if (!chunkRef) {
264
- const lines: string[] = [];
265
- for (const r of page.rows ?? []) {
266
- const s = JSON.stringify(r);
267
- lines.push(s === undefined ? 'null' : s);
270
+ const subResponses: SyncPullSubscriptionResponse[] = [];
271
+ const activeSubscriptions: { scopes: ScopeValues }[] = [];
272
+ const nextCursors: number[] = [];
273
+
274
+ for (const sub of resolved) {
275
+ const cursor = Math.max(-1, sub.cursor ?? -1);
276
+ // Validate shape exists (throws if not registered)
277
+ args.shapes.getOrThrow(sub.shape);
278
+
279
+ if (
280
+ sub.status === 'revoked' ||
281
+ Object.keys(sub.scopes).length === 0
282
+ ) {
283
+ subResponses.push({
284
+ id: sub.id,
285
+ status: 'revoked',
286
+ scopes: {},
287
+ bootstrap: false,
288
+ nextCursor: cursor,
289
+ commits: [],
290
+ });
291
+ continue;
268
292
  }
269
- const ndjson = lines.length > 0 ? `${lines.join('\n')}\n` : '';
270
- const gz = await compressSnapshotNdjson(ndjson);
271
- const sha256 = createHash('sha256').update(ndjson).digest('hex');
272
- const expiresAt = new Date(
273
- Date.now() + Math.max(1000, ttlMs)
274
- ).toISOString();
275
-
276
- // Use external chunk storage if available, otherwise fall back to inline
277
- if (args.chunkStorage) {
278
- chunkRef = await args.chunkStorage.storeChunk({
279
- partitionId,
280
- scopeKey: cacheKey,
281
- scope: nextTableName,
282
- asOfCommitSeq: effectiveState.asOfCommitSeq,
283
- rowCursor: nextState.rowCursor ?? null,
284
- rowLimit: limitSnapshotRows,
285
- encoding: 'ndjson',
286
- compression: 'gzip',
287
- sha256,
288
- body: gz,
289
- expiresAt,
293
+
294
+ const effectiveScopes = sub.scopes;
295
+ activeSubscriptions.push({ scopes: effectiveScopes });
296
+
297
+ const needsBootstrap =
298
+ sub.bootstrapState != null ||
299
+ cursor < 0 ||
300
+ cursor > maxCommitSeq ||
301
+ (minCommitSeq > 0 && cursor < minCommitSeq - 1);
302
+
303
+ if (needsBootstrap) {
304
+ const tables = args.shapes
305
+ .getBootstrapOrderFor(sub.shape)
306
+ .map((handler) => handler.table);
307
+
308
+ const initState: SyncBootstrapState = {
309
+ asOfCommitSeq: maxCommitSeq,
310
+ tables,
311
+ tableIndex: 0,
312
+ rowCursor: null,
313
+ };
314
+
315
+ const requestedState = sub.bootstrapState ?? null;
316
+ const state =
317
+ requestedState &&
318
+ typeof requestedState.asOfCommitSeq === 'number' &&
319
+ Array.isArray(requestedState.tables) &&
320
+ typeof requestedState.tableIndex === 'number'
321
+ ? (requestedState as SyncBootstrapState)
322
+ : initState;
323
+
324
+ // If the bootstrap state's asOfCommitSeq is no longer catch-up-able, restart bootstrap.
325
+ const effectiveState =
326
+ state.asOfCommitSeq < minCommitSeq - 1 ? initState : state;
327
+
328
+ const tableName =
329
+ effectiveState.tables[effectiveState.tableIndex];
330
+
331
+ // No tables (or ran past the end): treat bootstrap as complete.
332
+ if (!tableName) {
333
+ subResponses.push({
334
+ id: sub.id,
335
+ status: 'active',
336
+ scopes: effectiveScopes,
337
+ bootstrap: true,
338
+ bootstrapState: null,
339
+ nextCursor: effectiveState.asOfCommitSeq,
340
+ commits: [],
341
+ snapshots: [],
342
+ });
343
+ nextCursors.push(effectiveState.asOfCommitSeq);
344
+ continue;
345
+ }
346
+
347
+ const snapshots: SyncSnapshot[] = [];
348
+ let nextState: SyncBootstrapState | null = effectiveState;
349
+
350
+ for (
351
+ let pageIndex = 0;
352
+ pageIndex < maxSnapshotPages;
353
+ pageIndex++
354
+ ) {
355
+ if (!nextState) break;
356
+
357
+ const nextTableName = nextState.tables[nextState.tableIndex];
358
+ if (!nextTableName) {
359
+ nextState = null;
360
+ break;
361
+ }
362
+
363
+ const tableHandler = args.shapes.getOrThrow(nextTableName);
364
+ const isFirstPage = nextState.rowCursor == null;
365
+
366
+ const page = await tableHandler.snapshot(
367
+ {
368
+ db: trx,
369
+ actorId: args.actorId,
370
+ scopeValues: effectiveScopes,
371
+ cursor: nextState.rowCursor,
372
+ limit: limitSnapshotRows,
373
+ },
374
+ sub.params
375
+ );
376
+
377
+ const isLastPage = page.nextCursor == null;
378
+
379
+ // Always use NDJSON+gzip for bootstrap snapshots
380
+ const ttlMs =
381
+ tableHandler.snapshotChunkTtlMs ?? 24 * 60 * 60 * 1000; // 24h
382
+ const nowIso = new Date().toISOString();
383
+
384
+ // Use scope hash for caching
385
+ const cacheKey = `${partitionId}:${scopesToCacheKey(effectiveScopes)}`;
386
+ const cached = await readSnapshotChunkRefByPageKey(trx, {
387
+ partitionId,
388
+ scopeKey: cacheKey,
389
+ scope: nextTableName,
390
+ asOfCommitSeq: effectiveState.asOfCommitSeq,
391
+ rowCursor: nextState.rowCursor,
392
+ rowLimit: limitSnapshotRows,
393
+ encoding: 'ndjson',
394
+ compression: 'gzip',
395
+ nowIso,
396
+ });
397
+
398
+ let chunkRef = cached;
399
+
400
+ if (!chunkRef) {
401
+ const lines: string[] = [];
402
+ for (const r of page.rows ?? []) {
403
+ const s = JSON.stringify(r);
404
+ lines.push(s === undefined ? 'null' : s);
405
+ }
406
+ const ndjson =
407
+ lines.length > 0 ? `${lines.join('\n')}\n` : '';
408
+ const gz = await compressSnapshotNdjson(ndjson);
409
+ const sha256 = createHash('sha256')
410
+ .update(ndjson)
411
+ .digest('hex');
412
+ const expiresAt = new Date(
413
+ Date.now() + Math.max(1000, ttlMs)
414
+ ).toISOString();
415
+
416
+ // Use external chunk storage if available, otherwise fall back to inline
417
+ if (args.chunkStorage) {
418
+ chunkRef = await args.chunkStorage.storeChunk({
419
+ partitionId,
420
+ scopeKey: cacheKey,
421
+ scope: nextTableName,
422
+ asOfCommitSeq: effectiveState.asOfCommitSeq,
423
+ rowCursor: nextState.rowCursor ?? null,
424
+ rowLimit: limitSnapshotRows,
425
+ encoding: 'ndjson',
426
+ compression: 'gzip',
427
+ sha256,
428
+ body: gz,
429
+ expiresAt,
430
+ });
431
+ } else {
432
+ const chunkId = randomUUID();
433
+ chunkRef = await insertSnapshotChunk(trx, {
434
+ chunkId,
435
+ partitionId,
436
+ scopeKey: cacheKey,
437
+ scope: nextTableName,
438
+ asOfCommitSeq: effectiveState.asOfCommitSeq,
439
+ rowCursor: nextState.rowCursor,
440
+ rowLimit: limitSnapshotRows,
441
+ encoding: 'ndjson',
442
+ compression: 'gzip',
443
+ sha256,
444
+ body: gz,
445
+ expiresAt,
446
+ });
447
+ }
448
+ }
449
+
450
+ snapshots.push({
451
+ table: nextTableName,
452
+ rows: [],
453
+ chunks: [chunkRef],
454
+ isFirstPage,
455
+ isLastPage,
456
+ });
457
+
458
+ if (page.nextCursor != null) {
459
+ nextState = { ...nextState, rowCursor: page.nextCursor };
460
+ continue;
461
+ }
462
+
463
+ if (nextState.tableIndex + 1 < nextState.tables.length) {
464
+ nextState = {
465
+ ...nextState,
466
+ tableIndex: nextState.tableIndex + 1,
467
+ rowCursor: null,
468
+ };
469
+ continue;
470
+ }
471
+
472
+ nextState = null;
473
+ break;
474
+ }
475
+
476
+ subResponses.push({
477
+ id: sub.id,
478
+ status: 'active',
479
+ scopes: effectiveScopes,
480
+ bootstrap: true,
481
+ bootstrapState: nextState,
482
+ nextCursor: effectiveState.asOfCommitSeq,
483
+ commits: [],
484
+ snapshots,
290
485
  });
486
+ nextCursors.push(effectiveState.asOfCommitSeq);
487
+ continue;
488
+ }
489
+
490
+ // Incremental pull for this subscription
491
+ // Read the commit window for this table up-front so the subscription cursor
492
+ // can advance past commits that don't match the requested scopes.
493
+ const scannedCommitSeqs = await dialect.readCommitSeqsForPull(trx, {
494
+ partitionId,
495
+ cursor,
496
+ limitCommits,
497
+ tables: [sub.shape],
498
+ });
499
+ const maxScannedCommitSeq =
500
+ scannedCommitSeqs.length > 0
501
+ ? scannedCommitSeqs[scannedCommitSeqs.length - 1]!
502
+ : cursor;
503
+
504
+ // Use streaming when available to reduce memory pressure for large pulls
505
+ const pullRowStream = dialect.streamIncrementalPullRows
506
+ ? dialect.streamIncrementalPullRows(trx, {
507
+ partitionId,
508
+ table: sub.shape,
509
+ scopes: effectiveScopes,
510
+ cursor,
511
+ limitCommits,
512
+ })
513
+ : null;
514
+
515
+ // Collect rows and compute nextCursor in a single pass
516
+ const incrementalRows: Array<{
517
+ commit_seq: number;
518
+ actor_id: string;
519
+ created_at: string;
520
+ change_id: number;
521
+ table: string;
522
+ row_id: string;
523
+ op: 'upsert' | 'delete';
524
+ row_json: unknown | null;
525
+ row_version: number | null;
526
+ scopes: Record<string, string | string[]>;
527
+ }> = [];
528
+
529
+ let nextCursor = cursor;
530
+
531
+ if (pullRowStream) {
532
+ // Streaming path: process rows as they arrive
533
+ for await (const row of pullRowStream) {
534
+ incrementalRows.push(row);
535
+ nextCursor = Math.max(nextCursor, row.commit_seq);
536
+ }
291
537
  } else {
292
- const chunkId = randomUUID();
293
- chunkRef = await insertSnapshotChunk(trx, {
294
- chunkId,
538
+ // Non-streaming fallback: load all rows at once
539
+ const rows = await dialect.readIncrementalPullRows(trx, {
295
540
  partitionId,
296
- scopeKey: cacheKey,
297
- scope: nextTableName,
298
- asOfCommitSeq: effectiveState.asOfCommitSeq,
299
- rowCursor: nextState.rowCursor,
300
- rowLimit: limitSnapshotRows,
301
- encoding: 'ndjson',
302
- compression: 'gzip',
303
- sha256,
304
- body: gz,
305
- expiresAt,
541
+ table: sub.shape,
542
+ scopes: effectiveScopes,
543
+ cursor,
544
+ limitCommits,
306
545
  });
546
+ incrementalRows.push(...rows);
547
+ for (const r of incrementalRows) {
548
+ nextCursor = Math.max(nextCursor, r.commit_seq);
549
+ }
307
550
  }
308
- }
309
-
310
- snapshots.push({
311
- table: nextTableName,
312
- rows: [],
313
- chunks: [chunkRef],
314
- isFirstPage,
315
- isLastPage,
316
- });
317
-
318
- if (page.nextCursor != null) {
319
- nextState = { ...nextState, rowCursor: page.nextCursor };
320
- continue;
321
- }
322
551
 
323
- if (nextState.tableIndex + 1 < nextState.tables.length) {
324
- nextState = {
325
- ...nextState,
326
- tableIndex: nextState.tableIndex + 1,
327
- rowCursor: null,
328
- };
329
- continue;
330
- }
331
-
332
- nextState = null;
333
- break;
334
- }
335
-
336
- subResponses.push({
337
- id: sub.id,
338
- status: 'active',
339
- scopes: effectiveScopes,
340
- bootstrap: true,
341
- bootstrapState: nextState,
342
- nextCursor: effectiveState.asOfCommitSeq,
343
- commits: [],
344
- snapshots,
345
- });
346
- nextCursors.push(effectiveState.asOfCommitSeq);
347
- continue;
348
- }
552
+ nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
349
553
 
350
- // Incremental pull for this subscription
351
- // Read the commit window for this table up-front so the subscription cursor
352
- // can advance past commits that don't match the requested scopes.
353
- const scannedCommitSeqs = await dialect.readCommitSeqsForPull(trx, {
354
- partitionId,
355
- cursor,
356
- limitCommits,
357
- tables: [sub.shape],
358
- });
359
- const maxScannedCommitSeq =
360
- scannedCommitSeqs.length > 0
361
- ? scannedCommitSeqs[scannedCommitSeqs.length - 1]!
362
- : cursor;
363
-
364
- // Use streaming when available to reduce memory pressure for large pulls
365
- const pullRowStream = dialect.streamIncrementalPullRows
366
- ? dialect.streamIncrementalPullRows(trx, {
367
- partitionId,
368
- table: sub.shape,
369
- scopes: effectiveScopes,
370
- cursor,
371
- limitCommits,
372
- })
373
- : null;
374
-
375
- // Collect rows and compute nextCursor in a single pass
376
- const incrementalRows: Array<{
377
- commit_seq: number;
378
- actor_id: string;
379
- created_at: string;
380
- change_id: number;
381
- table: string;
382
- row_id: string;
383
- op: 'upsert' | 'delete';
384
- row_json: unknown | null;
385
- row_version: number | null;
386
- scopes: Record<string, string | string[]>;
387
- }> = [];
388
-
389
- let nextCursor = cursor;
390
-
391
- if (pullRowStream) {
392
- // Streaming path: process rows as they arrive
393
- for await (const row of pullRowStream) {
394
- incrementalRows.push(row);
395
- nextCursor = Math.max(nextCursor, row.commit_seq);
396
- }
397
- } else {
398
- // Non-streaming fallback: load all rows at once
399
- const rows = await dialect.readIncrementalPullRows(trx, {
400
- partitionId,
401
- table: sub.shape,
402
- scopes: effectiveScopes,
403
- cursor,
404
- limitCommits,
405
- });
406
- incrementalRows.push(...rows);
407
- for (const r of incrementalRows) {
408
- nextCursor = Math.max(nextCursor, r.commit_seq);
409
- }
410
- }
554
+ if (incrementalRows.length === 0) {
555
+ subResponses.push({
556
+ id: sub.id,
557
+ status: 'active',
558
+ scopes: effectiveScopes,
559
+ bootstrap: false,
560
+ nextCursor,
561
+ commits: [],
562
+ });
563
+ nextCursors.push(nextCursor);
564
+ continue;
565
+ }
411
566
 
412
- nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
567
+ if (dedupeRows) {
568
+ const latestByRowKey = new Map<
569
+ string,
570
+ {
571
+ commitSeq: number;
572
+ createdAt: string;
573
+ actorId: string;
574
+ changeId: number;
575
+ change: SyncChange;
576
+ }
577
+ >();
578
+
579
+ for (const r of incrementalRows) {
580
+ const rowKey = `${r.table}\u0000${r.row_id}`;
581
+ const change: SyncChange = {
582
+ table: r.table,
583
+ row_id: r.row_id,
584
+ op: r.op,
585
+ row_json: r.row_json,
586
+ row_version: r.row_version,
587
+ scopes: dialect.dbToScopes(r.scopes),
588
+ };
589
+
590
+ latestByRowKey.set(rowKey, {
591
+ commitSeq: r.commit_seq,
592
+ createdAt: r.created_at,
593
+ actorId: r.actor_id,
594
+ changeId: r.change_id,
595
+ change,
596
+ });
597
+ }
598
+
599
+ const latest = Array.from(latestByRowKey.values()).sort(
600
+ (a, b) => a.commitSeq - b.commitSeq || a.changeId - b.changeId
601
+ );
602
+
603
+ const commitsBySeq = new Map<number, SyncCommit>();
604
+ for (const item of latest) {
605
+ let commit = commitsBySeq.get(item.commitSeq);
606
+ if (!commit) {
607
+ commit = {
608
+ commitSeq: item.commitSeq,
609
+ createdAt: item.createdAt,
610
+ actorId: item.actorId,
611
+ changes: [],
612
+ };
613
+ commitsBySeq.set(item.commitSeq, commit);
614
+ }
615
+ commit.changes.push(item.change);
616
+ }
617
+
618
+ const commits = Array.from(commitsBySeq.values()).sort(
619
+ (a, b) => a.commitSeq - b.commitSeq
620
+ );
621
+
622
+ subResponses.push({
623
+ id: sub.id,
624
+ status: 'active',
625
+ scopes: effectiveScopes,
626
+ bootstrap: false,
627
+ nextCursor,
628
+ commits,
629
+ });
630
+ nextCursors.push(nextCursor);
631
+ continue;
632
+ }
413
633
 
414
- if (incrementalRows.length === 0) {
415
- subResponses.push({
416
- id: sub.id,
417
- status: 'active',
418
- scopes: effectiveScopes,
419
- bootstrap: false,
420
- nextCursor,
421
- commits: [],
422
- });
423
- nextCursors.push(nextCursor);
424
- continue;
425
- }
634
+ const commitsBySeq = new Map<number, SyncCommit>();
635
+ const commitSeqs: number[] = [];
636
+
637
+ for (const r of incrementalRows) {
638
+ const seq = r.commit_seq;
639
+ let commit = commitsBySeq.get(seq);
640
+ if (!commit) {
641
+ commit = {
642
+ commitSeq: seq,
643
+ createdAt: r.created_at,
644
+ actorId: r.actor_id,
645
+ changes: [],
646
+ };
647
+ commitsBySeq.set(seq, commit);
648
+ commitSeqs.push(seq);
649
+ }
650
+
651
+ const change: SyncChange = {
652
+ table: r.table,
653
+ row_id: r.row_id,
654
+ op: r.op,
655
+ row_json: r.row_json,
656
+ row_version: r.row_version,
657
+ scopes: dialect.dbToScopes(r.scopes),
658
+ };
659
+ commit.changes.push(change);
660
+ }
426
661
 
427
- if (dedupeRows) {
428
- const latestByRowKey = new Map<
429
- string,
430
- {
431
- commitSeq: number;
432
- createdAt: string;
433
- actorId: string;
434
- changeId: number;
435
- change: SyncChange;
662
+ const commits: SyncCommit[] = commitSeqs
663
+ .map((seq) => commitsBySeq.get(seq))
664
+ .filter((c): c is SyncCommit => !!c)
665
+ .filter((c) => c.changes.length > 0);
666
+
667
+ subResponses.push({
668
+ id: sub.id,
669
+ status: 'active',
670
+ scopes: effectiveScopes,
671
+ bootstrap: false,
672
+ nextCursor,
673
+ commits,
674
+ });
675
+ nextCursors.push(nextCursor);
436
676
  }
437
- >();
438
-
439
- for (const r of incrementalRows) {
440
- const rowKey = `${r.table}\u0000${r.row_id}`;
441
- const change: SyncChange = {
442
- table: r.table,
443
- row_id: r.row_id,
444
- op: r.op,
445
- row_json: r.row_json,
446
- row_version: r.row_version,
447
- scopes: dialect.dbToScopes(r.scopes),
448
- };
449
677
 
450
- latestByRowKey.set(rowKey, {
451
- commitSeq: r.commit_seq,
452
- createdAt: r.created_at,
453
- actorId: r.actor_id,
454
- changeId: r.change_id,
455
- change,
456
- });
457
- }
678
+ const effectiveScopes = mergeScopes(activeSubscriptions);
679
+ const clientCursor =
680
+ nextCursors.length > 0 ? Math.min(...nextCursors) : maxCommitSeq;
458
681
 
459
- const latest = Array.from(latestByRowKey.values()).sort(
460
- (a, b) => a.commitSeq - b.commitSeq || a.changeId - b.changeId
461
- );
462
-
463
- const commitsBySeq = new Map<number, SyncCommit>();
464
- for (const item of latest) {
465
- let commit = commitsBySeq.get(item.commitSeq);
466
- if (!commit) {
467
- commit = {
468
- commitSeq: item.commitSeq,
469
- createdAt: item.createdAt,
470
- actorId: item.actorId,
471
- changes: [],
472
- };
473
- commitsBySeq.set(item.commitSeq, commit);
474
- }
475
- commit.changes.push(item.change);
476
- }
682
+ return {
683
+ response: {
684
+ ok: true as const,
685
+ subscriptions: subResponses,
686
+ },
687
+ effectiveScopes,
688
+ clientCursor,
689
+ };
690
+ });
477
691
 
478
- const commits = Array.from(commitsBySeq.values()).sort(
479
- (a, b) => a.commitSeq - b.commitSeq
480
- );
692
+ const durationMs = Math.max(0, Date.now() - startedAtMs);
693
+ const stats = summarizePullResponse(result.response);
694
+
695
+ span.setAttribute('status', 'ok');
696
+ span.setAttribute('duration_ms', durationMs);
697
+ span.setAttribute('subscription_count', stats.subscriptionCount);
698
+ span.setAttribute('commit_count', stats.commitCount);
699
+ span.setAttribute('change_count', stats.changeCount);
700
+ span.setAttribute('snapshot_page_count', stats.snapshotPageCount);
701
+ span.setStatus('ok');
702
+
703
+ recordPullMetrics({
704
+ status: 'ok',
705
+ dedupeRows,
706
+ durationMs,
707
+ stats,
708
+ });
481
709
 
482
- subResponses.push({
483
- id: sub.id,
484
- status: 'active',
485
- scopes: effectiveScopes,
486
- bootstrap: false,
487
- nextCursor,
488
- commits,
710
+ return result;
711
+ } catch (error) {
712
+ const durationMs = Math.max(0, Date.now() - startedAtMs);
713
+
714
+ span.setAttribute('status', 'error');
715
+ span.setAttribute('duration_ms', durationMs);
716
+ span.setStatus('error');
717
+
718
+ recordPullMetrics({
719
+ status: 'error',
720
+ dedupeRows: request.dedupeRows === true,
721
+ durationMs,
722
+ stats: {
723
+ subscriptionCount: 0,
724
+ activeSubscriptionCount: 0,
725
+ revokedSubscriptionCount: 0,
726
+ bootstrapSubscriptionCount: 0,
727
+ commitCount: 0,
728
+ changeCount: 0,
729
+ snapshotPageCount: 0,
730
+ },
489
731
  });
490
- nextCursors.push(nextCursor);
491
- continue;
492
- }
493
732
 
494
- const commitsBySeq = new Map<number, SyncCommit>();
495
- const commitSeqs: number[] = [];
496
-
497
- for (const r of incrementalRows) {
498
- const seq = r.commit_seq;
499
- let commit = commitsBySeq.get(seq);
500
- if (!commit) {
501
- commit = {
502
- commitSeq: seq,
503
- createdAt: r.created_at,
504
- actorId: r.actor_id,
505
- changes: [],
506
- };
507
- commitsBySeq.set(seq, commit);
508
- commitSeqs.push(seq);
509
- }
510
-
511
- const change: SyncChange = {
512
- table: r.table,
513
- row_id: r.row_id,
514
- op: r.op,
515
- row_json: r.row_json,
516
- row_version: r.row_version,
517
- scopes: dialect.dbToScopes(r.scopes),
518
- };
519
- commit.changes.push(change);
733
+ captureSyncException(error, {
734
+ event: 'sync.server.pull',
735
+ requestedSubscriptionCount,
736
+ dedupeRows: request.dedupeRows === true,
737
+ });
738
+ throw error;
520
739
  }
521
-
522
- const commits: SyncCommit[] = commitSeqs
523
- .map((seq) => commitsBySeq.get(seq))
524
- .filter((c): c is SyncCommit => !!c)
525
- .filter((c) => c.changes.length > 0);
526
-
527
- subResponses.push({
528
- id: sub.id,
529
- status: 'active',
530
- scopes: effectiveScopes,
531
- bootstrap: false,
532
- nextCursor,
533
- commits,
534
- });
535
- nextCursors.push(nextCursor);
536
740
  }
537
-
538
- const effectiveScopes = mergeScopes(activeSubscriptions);
539
- const clientCursor =
540
- nextCursors.length > 0 ? Math.min(...nextCursors) : maxCommitSeq;
541
-
542
- return {
543
- response: {
544
- ok: true as const,
545
- subscriptions: subResponses,
546
- },
547
- effectiveScopes,
548
- clientCursor,
549
- };
550
- });
741
+ );
551
742
  }