@syncular/server 0.0.1-73 → 0.0.1-88

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/blobs/adapters/database.d.ts.map +1 -1
  2. package/dist/blobs/adapters/database.js +25 -3
  3. package/dist/blobs/adapters/database.js.map +1 -1
  4. package/dist/blobs/index.js +5 -5
  5. package/dist/dialect/base.js +1 -1
  6. package/dist/dialect/index.js +3 -3
  7. package/dist/helpers/index.js +4 -4
  8. package/dist/index.js +16 -16
  9. package/dist/proxy/handler.d.ts.map +1 -1
  10. package/dist/proxy/handler.js +7 -4
  11. package/dist/proxy/handler.js.map +1 -1
  12. package/dist/proxy/index.js +3 -3
  13. package/dist/proxy/mutation-detector.d.ts +4 -0
  14. package/dist/proxy/mutation-detector.d.ts.map +1 -1
  15. package/dist/proxy/mutation-detector.js +209 -24
  16. package/dist/proxy/mutation-detector.js.map +1 -1
  17. package/dist/pull.d.ts +1 -1
  18. package/dist/pull.d.ts.map +1 -1
  19. package/dist/pull.js +445 -322
  20. package/dist/pull.js.map +1 -1
  21. package/dist/push.d.ts +1 -1
  22. package/dist/push.d.ts.map +1 -1
  23. package/dist/push.js +341 -260
  24. package/dist/push.js.map +1 -1
  25. package/dist/realtime/index.js +1 -1
  26. package/dist/shapes/index.js +3 -3
  27. package/dist/snapshot-chunks/adapters/s3.js +1 -1
  28. package/dist/snapshot-chunks/db-metadata.d.ts.map +1 -1
  29. package/dist/snapshot-chunks/db-metadata.js +47 -41
  30. package/dist/snapshot-chunks/db-metadata.js.map +1 -1
  31. package/dist/snapshot-chunks/index.js +3 -3
  32. package/dist/subscriptions/index.js +1 -1
  33. package/package.json +1 -1
  34. package/src/blobs/adapters/database.test.ts +67 -0
  35. package/src/blobs/adapters/database.ts +34 -9
  36. package/src/proxy/handler.test.ts +120 -0
  37. package/src/proxy/handler.ts +9 -2
  38. package/src/proxy/mutation-detector.test.ts +71 -0
  39. package/src/proxy/mutation-detector.ts +227 -29
  40. package/src/pull.ts +609 -418
  41. package/src/push.ts +473 -349
  42. package/src/snapshot-chunks/db-metadata.test.ts +100 -0
  43. package/src/snapshot-chunks/db-metadata.ts +68 -48
package/dist/pull.js CHANGED
@@ -1,8 +1,9 @@
1
1
  import { createHash, randomUUID } from 'node:crypto';
2
2
  import { promisify } from 'node:util';
3
3
  import { gzip, gzipSync } from 'node:zlib';
4
- import { insertSnapshotChunk, readSnapshotChunkRefByPageKey, } from './snapshot-chunks';
5
- import { resolveEffectiveScopesForSubscriptions } from './subscriptions/resolve';
4
+ import { captureSyncException, countSyncMetric, distributionSyncMetric, startSyncSpan, } from '@syncular/core';
5
+ import { insertSnapshotChunk, readSnapshotChunkRefByPageKey, } from './snapshot-chunks.js';
6
+ import { resolveEffectiveScopesForSubscriptions } from './subscriptions/resolve.js';
6
7
  const gzipAsync = promisify(gzip);
7
8
  const ASYNC_GZIP_MIN_BYTES = 64 * 1024;
8
9
  async function compressSnapshotNdjson(ndjson) {
@@ -59,152 +60,187 @@ function mergeScopes(subscriptions) {
59
60
  }
60
61
  return merged;
61
62
  }
63
+ function summarizePullResponse(response) {
64
+ const subscriptions = response.subscriptions ?? [];
65
+ let activeSubscriptionCount = 0;
66
+ let revokedSubscriptionCount = 0;
67
+ let bootstrapSubscriptionCount = 0;
68
+ let commitCount = 0;
69
+ let changeCount = 0;
70
+ let snapshotPageCount = 0;
71
+ for (const sub of subscriptions) {
72
+ if (sub.status === 'revoked') {
73
+ revokedSubscriptionCount += 1;
74
+ }
75
+ else {
76
+ activeSubscriptionCount += 1;
77
+ }
78
+ if (sub.bootstrap) {
79
+ bootstrapSubscriptionCount += 1;
80
+ }
81
+ const commits = sub.commits ?? [];
82
+ commitCount += commits.length;
83
+ for (const commit of commits) {
84
+ changeCount += commit.changes?.length ?? 0;
85
+ }
86
+ snapshotPageCount += sub.snapshots?.length ?? 0;
87
+ }
88
+ return {
89
+ subscriptionCount: subscriptions.length,
90
+ activeSubscriptionCount,
91
+ revokedSubscriptionCount,
92
+ bootstrapSubscriptionCount,
93
+ commitCount,
94
+ changeCount,
95
+ snapshotPageCount,
96
+ };
97
+ }
98
+ function recordPullMetrics(args) {
99
+ const { status, dedupeRows, durationMs, stats } = args;
100
+ const attributes = {
101
+ status,
102
+ dedupe_rows: dedupeRows,
103
+ };
104
+ countSyncMetric('sync.server.pull.requests', 1, { attributes });
105
+ distributionSyncMetric('sync.server.pull.duration_ms', durationMs, {
106
+ unit: 'millisecond',
107
+ attributes,
108
+ });
109
+ distributionSyncMetric('sync.server.pull.subscriptions', stats.subscriptionCount, { attributes });
110
+ distributionSyncMetric('sync.server.pull.active_subscriptions', stats.activeSubscriptionCount, { attributes });
111
+ distributionSyncMetric('sync.server.pull.revoked_subscriptions', stats.revokedSubscriptionCount, { attributes });
112
+ distributionSyncMetric('sync.server.pull.bootstrap_subscriptions', stats.bootstrapSubscriptionCount, { attributes });
113
+ distributionSyncMetric('sync.server.pull.commits', stats.commitCount, {
114
+ attributes,
115
+ });
116
+ distributionSyncMetric('sync.server.pull.changes', stats.changeCount, {
117
+ attributes,
118
+ });
119
+ distributionSyncMetric('sync.server.pull.snapshot_pages', stats.snapshotPageCount, { attributes });
120
+ }
62
121
  export async function pull(args) {
63
122
  const { request, dialect } = args;
64
123
  const db = args.db;
65
124
  const partitionId = args.partitionId ?? 'default';
66
- // Validate and sanitize request limits
67
- const limitCommits = sanitizeLimit(request.limitCommits, 50, 1, 500);
68
- const limitSnapshotRows = sanitizeLimit(request.limitSnapshotRows, 1000, 1, 5000);
69
- const maxSnapshotPages = sanitizeLimit(request.maxSnapshotPages, 1, 1, 50);
70
- const dedupeRows = request.dedupeRows === true;
71
- // Resolve effective scopes for each subscription
72
- const resolved = await resolveEffectiveScopesForSubscriptions({
73
- db,
74
- actorId: args.actorId,
75
- subscriptions: request.subscriptions ?? [],
76
- shapes: args.shapes,
77
- });
78
- return dialect.executeInTransaction(db, async (trx) => {
79
- await dialect.setRepeatableRead(trx);
80
- const maxCommitSeq = await dialect.readMaxCommitSeq(trx, { partitionId });
81
- const minCommitSeq = await dialect.readMinCommitSeq(trx, { partitionId });
82
- const subResponses = [];
83
- const activeSubscriptions = [];
84
- const nextCursors = [];
85
- for (const sub of resolved) {
86
- const cursor = Math.max(-1, sub.cursor ?? -1);
87
- // Validate shape exists (throws if not registered)
88
- args.shapes.getOrThrow(sub.shape);
89
- if (sub.status === 'revoked' || Object.keys(sub.scopes).length === 0) {
90
- subResponses.push({
91
- id: sub.id,
92
- status: 'revoked',
93
- scopes: {},
94
- bootstrap: false,
95
- nextCursor: cursor,
96
- commits: [],
125
+ const requestedSubscriptionCount = Array.isArray(request.subscriptions)
126
+ ? request.subscriptions.length
127
+ : 0;
128
+ const startedAtMs = Date.now();
129
+ return startSyncSpan({
130
+ name: 'sync.server.pull',
131
+ op: 'sync.pull',
132
+ attributes: {
133
+ requested_subscription_count: requestedSubscriptionCount,
134
+ dedupe_rows: request.dedupeRows === true,
135
+ },
136
+ }, async (span) => {
137
+ try {
138
+ // Validate and sanitize request limits
139
+ const limitCommits = sanitizeLimit(request.limitCommits, 50, 1, 500);
140
+ const limitSnapshotRows = sanitizeLimit(request.limitSnapshotRows, 1000, 1, 5000);
141
+ const maxSnapshotPages = sanitizeLimit(request.maxSnapshotPages, 1, 1, 50);
142
+ const dedupeRows = request.dedupeRows === true;
143
+ // Resolve effective scopes for each subscription
144
+ const resolved = await resolveEffectiveScopesForSubscriptions({
145
+ db,
146
+ actorId: args.actorId,
147
+ subscriptions: request.subscriptions ?? [],
148
+ shapes: args.shapes,
149
+ });
150
+ const result = await dialect.executeInTransaction(db, async (trx) => {
151
+ await dialect.setRepeatableRead(trx);
152
+ const maxCommitSeq = await dialect.readMaxCommitSeq(trx, {
153
+ partitionId,
97
154
  });
98
- continue;
99
- }
100
- const effectiveScopes = sub.scopes;
101
- activeSubscriptions.push({ scopes: effectiveScopes });
102
- const needsBootstrap = sub.bootstrapState != null ||
103
- cursor < 0 ||
104
- cursor > maxCommitSeq ||
105
- (minCommitSeq > 0 && cursor < minCommitSeq - 1);
106
- if (needsBootstrap) {
107
- const tables = args.shapes
108
- .getBootstrapOrderFor(sub.shape)
109
- .map((handler) => handler.table);
110
- const initState = {
111
- asOfCommitSeq: maxCommitSeq,
112
- tables,
113
- tableIndex: 0,
114
- rowCursor: null,
115
- };
116
- const requestedState = sub.bootstrapState ?? null;
117
- const state = requestedState &&
118
- typeof requestedState.asOfCommitSeq === 'number' &&
119
- Array.isArray(requestedState.tables) &&
120
- typeof requestedState.tableIndex === 'number'
121
- ? requestedState
122
- : initState;
123
- // If the bootstrap state's asOfCommitSeq is no longer catch-up-able, restart bootstrap.
124
- const effectiveState = state.asOfCommitSeq < minCommitSeq - 1 ? initState : state;
125
- const tableName = effectiveState.tables[effectiveState.tableIndex];
126
- // No tables (or ran past the end): treat bootstrap as complete.
127
- if (!tableName) {
128
- subResponses.push({
129
- id: sub.id,
130
- status: 'active',
131
- scopes: effectiveScopes,
132
- bootstrap: true,
133
- bootstrapState: null,
134
- nextCursor: effectiveState.asOfCommitSeq,
135
- commits: [],
136
- snapshots: [],
137
- });
138
- nextCursors.push(effectiveState.asOfCommitSeq);
139
- continue;
140
- }
141
- const snapshots = [];
142
- let nextState = effectiveState;
143
- for (let pageIndex = 0; pageIndex < maxSnapshotPages; pageIndex++) {
144
- if (!nextState)
145
- break;
146
- const nextTableName = nextState.tables[nextState.tableIndex];
147
- if (!nextTableName) {
148
- nextState = null;
149
- break;
155
+ const minCommitSeq = await dialect.readMinCommitSeq(trx, {
156
+ partitionId,
157
+ });
158
+ const subResponses = [];
159
+ const activeSubscriptions = [];
160
+ const nextCursors = [];
161
+ for (const sub of resolved) {
162
+ const cursor = Math.max(-1, sub.cursor ?? -1);
163
+ // Validate shape exists (throws if not registered)
164
+ args.shapes.getOrThrow(sub.shape);
165
+ if (sub.status === 'revoked' ||
166
+ Object.keys(sub.scopes).length === 0) {
167
+ subResponses.push({
168
+ id: sub.id,
169
+ status: 'revoked',
170
+ scopes: {},
171
+ bootstrap: false,
172
+ nextCursor: cursor,
173
+ commits: [],
174
+ });
175
+ continue;
150
176
  }
151
- const tableHandler = args.shapes.getOrThrow(nextTableName);
152
- const isFirstPage = nextState.rowCursor == null;
153
- const page = await tableHandler.snapshot({
154
- db: trx,
155
- actorId: args.actorId,
156
- scopeValues: effectiveScopes,
157
- cursor: nextState.rowCursor,
158
- limit: limitSnapshotRows,
159
- }, sub.params);
160
- const isLastPage = page.nextCursor == null;
161
- // Always use NDJSON+gzip for bootstrap snapshots
162
- const ttlMs = tableHandler.snapshotChunkTtlMs ?? 24 * 60 * 60 * 1000; // 24h
163
- const nowIso = new Date().toISOString();
164
- // Use scope hash for caching
165
- const cacheKey = `${partitionId}:${scopesToCacheKey(effectiveScopes)}`;
166
- const cached = await readSnapshotChunkRefByPageKey(trx, {
167
- partitionId,
168
- scopeKey: cacheKey,
169
- scope: nextTableName,
170
- asOfCommitSeq: effectiveState.asOfCommitSeq,
171
- rowCursor: nextState.rowCursor,
172
- rowLimit: limitSnapshotRows,
173
- encoding: 'ndjson',
174
- compression: 'gzip',
175
- nowIso,
176
- });
177
- let chunkRef = cached;
178
- if (!chunkRef) {
179
- const lines = [];
180
- for (const r of page.rows ?? []) {
181
- const s = JSON.stringify(r);
182
- lines.push(s === undefined ? 'null' : s);
183
- }
184
- const ndjson = lines.length > 0 ? `${lines.join('\n')}\n` : '';
185
- const gz = await compressSnapshotNdjson(ndjson);
186
- const sha256 = createHash('sha256').update(ndjson).digest('hex');
187
- const expiresAt = new Date(Date.now() + Math.max(1000, ttlMs)).toISOString();
188
- // Use external chunk storage if available, otherwise fall back to inline
189
- if (args.chunkStorage) {
190
- chunkRef = await args.chunkStorage.storeChunk({
191
- partitionId,
192
- scopeKey: cacheKey,
193
- scope: nextTableName,
194
- asOfCommitSeq: effectiveState.asOfCommitSeq,
195
- rowCursor: nextState.rowCursor ?? null,
196
- rowLimit: limitSnapshotRows,
197
- encoding: 'ndjson',
198
- compression: 'gzip',
199
- sha256,
200
- body: gz,
201
- expiresAt,
177
+ const effectiveScopes = sub.scopes;
178
+ activeSubscriptions.push({ scopes: effectiveScopes });
179
+ const needsBootstrap = sub.bootstrapState != null ||
180
+ cursor < 0 ||
181
+ cursor > maxCommitSeq ||
182
+ (minCommitSeq > 0 && cursor < minCommitSeq - 1);
183
+ if (needsBootstrap) {
184
+ const tables = args.shapes
185
+ .getBootstrapOrderFor(sub.shape)
186
+ .map((handler) => handler.table);
187
+ const initState = {
188
+ asOfCommitSeq: maxCommitSeq,
189
+ tables,
190
+ tableIndex: 0,
191
+ rowCursor: null,
192
+ };
193
+ const requestedState = sub.bootstrapState ?? null;
194
+ const state = requestedState &&
195
+ typeof requestedState.asOfCommitSeq === 'number' &&
196
+ Array.isArray(requestedState.tables) &&
197
+ typeof requestedState.tableIndex === 'number'
198
+ ? requestedState
199
+ : initState;
200
+ // If the bootstrap state's asOfCommitSeq is no longer catch-up-able, restart bootstrap.
201
+ const effectiveState = state.asOfCommitSeq < minCommitSeq - 1 ? initState : state;
202
+ const tableName = effectiveState.tables[effectiveState.tableIndex];
203
+ // No tables (or ran past the end): treat bootstrap as complete.
204
+ if (!tableName) {
205
+ subResponses.push({
206
+ id: sub.id,
207
+ status: 'active',
208
+ scopes: effectiveScopes,
209
+ bootstrap: true,
210
+ bootstrapState: null,
211
+ nextCursor: effectiveState.asOfCommitSeq,
212
+ commits: [],
213
+ snapshots: [],
202
214
  });
215
+ nextCursors.push(effectiveState.asOfCommitSeq);
216
+ continue;
203
217
  }
204
- else {
205
- const chunkId = randomUUID();
206
- chunkRef = await insertSnapshotChunk(trx, {
207
- chunkId,
218
+ const snapshots = [];
219
+ let nextState = effectiveState;
220
+ for (let pageIndex = 0; pageIndex < maxSnapshotPages; pageIndex++) {
221
+ if (!nextState)
222
+ break;
223
+ const nextTableName = nextState.tables[nextState.tableIndex];
224
+ if (!nextTableName) {
225
+ nextState = null;
226
+ break;
227
+ }
228
+ const tableHandler = args.shapes.getOrThrow(nextTableName);
229
+ const isFirstPage = nextState.rowCursor == null;
230
+ const page = await tableHandler.snapshot({
231
+ db: trx,
232
+ actorId: args.actorId,
233
+ scopeValues: effectiveScopes,
234
+ cursor: nextState.rowCursor,
235
+ limit: limitSnapshotRows,
236
+ }, sub.params);
237
+ const isLastPage = page.nextCursor == null;
238
+ // Always use NDJSON+gzip for bootstrap snapshots
239
+ const ttlMs = tableHandler.snapshotChunkTtlMs ?? 24 * 60 * 60 * 1000; // 24h
240
+ const nowIso = new Date().toISOString();
241
+ // Use scope hash for caching
242
+ const cacheKey = `${partitionId}:${scopesToCacheKey(effectiveScopes)}`;
243
+ const cached = await readSnapshotChunkRefByPageKey(trx, {
208
244
  partitionId,
209
245
  scopeKey: cacheKey,
210
246
  scope: nextTableName,
@@ -213,202 +249,289 @@ export async function pull(args) {
213
249
  rowLimit: limitSnapshotRows,
214
250
  encoding: 'ndjson',
215
251
  compression: 'gzip',
216
- sha256,
217
- body: gz,
218
- expiresAt,
252
+ nowIso,
219
253
  });
254
+ let chunkRef = cached;
255
+ if (!chunkRef) {
256
+ const lines = [];
257
+ for (const r of page.rows ?? []) {
258
+ const s = JSON.stringify(r);
259
+ lines.push(s === undefined ? 'null' : s);
260
+ }
261
+ const ndjson = lines.length > 0 ? `${lines.join('\n')}\n` : '';
262
+ const gz = await compressSnapshotNdjson(ndjson);
263
+ const sha256 = createHash('sha256')
264
+ .update(ndjson)
265
+ .digest('hex');
266
+ const expiresAt = new Date(Date.now() + Math.max(1000, ttlMs)).toISOString();
267
+ // Use external chunk storage if available, otherwise fall back to inline
268
+ if (args.chunkStorage) {
269
+ chunkRef = await args.chunkStorage.storeChunk({
270
+ partitionId,
271
+ scopeKey: cacheKey,
272
+ scope: nextTableName,
273
+ asOfCommitSeq: effectiveState.asOfCommitSeq,
274
+ rowCursor: nextState.rowCursor ?? null,
275
+ rowLimit: limitSnapshotRows,
276
+ encoding: 'ndjson',
277
+ compression: 'gzip',
278
+ sha256,
279
+ body: gz,
280
+ expiresAt,
281
+ });
282
+ }
283
+ else {
284
+ const chunkId = randomUUID();
285
+ chunkRef = await insertSnapshotChunk(trx, {
286
+ chunkId,
287
+ partitionId,
288
+ scopeKey: cacheKey,
289
+ scope: nextTableName,
290
+ asOfCommitSeq: effectiveState.asOfCommitSeq,
291
+ rowCursor: nextState.rowCursor,
292
+ rowLimit: limitSnapshotRows,
293
+ encoding: 'ndjson',
294
+ compression: 'gzip',
295
+ sha256,
296
+ body: gz,
297
+ expiresAt,
298
+ });
299
+ }
300
+ }
301
+ snapshots.push({
302
+ table: nextTableName,
303
+ rows: [],
304
+ chunks: [chunkRef],
305
+ isFirstPage,
306
+ isLastPage,
307
+ });
308
+ if (page.nextCursor != null) {
309
+ nextState = { ...nextState, rowCursor: page.nextCursor };
310
+ continue;
311
+ }
312
+ if (nextState.tableIndex + 1 < nextState.tables.length) {
313
+ nextState = {
314
+ ...nextState,
315
+ tableIndex: nextState.tableIndex + 1,
316
+ rowCursor: null,
317
+ };
318
+ continue;
319
+ }
320
+ nextState = null;
321
+ break;
220
322
  }
323
+ subResponses.push({
324
+ id: sub.id,
325
+ status: 'active',
326
+ scopes: effectiveScopes,
327
+ bootstrap: true,
328
+ bootstrapState: nextState,
329
+ nextCursor: effectiveState.asOfCommitSeq,
330
+ commits: [],
331
+ snapshots,
332
+ });
333
+ nextCursors.push(effectiveState.asOfCommitSeq);
334
+ continue;
221
335
  }
222
- snapshots.push({
223
- table: nextTableName,
224
- rows: [],
225
- chunks: [chunkRef],
226
- isFirstPage,
227
- isLastPage,
336
+ // Incremental pull for this subscription
337
+ // Read the commit window for this table up-front so the subscription cursor
338
+ // can advance past commits that don't match the requested scopes.
339
+ const scannedCommitSeqs = await dialect.readCommitSeqsForPull(trx, {
340
+ partitionId,
341
+ cursor,
342
+ limitCommits,
343
+ tables: [sub.shape],
228
344
  });
229
- if (page.nextCursor != null) {
230
- nextState = { ...nextState, rowCursor: page.nextCursor };
345
+ const maxScannedCommitSeq = scannedCommitSeqs.length > 0
346
+ ? scannedCommitSeqs[scannedCommitSeqs.length - 1]
347
+ : cursor;
348
+ // Use streaming when available to reduce memory pressure for large pulls
349
+ const pullRowStream = dialect.streamIncrementalPullRows
350
+ ? dialect.streamIncrementalPullRows(trx, {
351
+ partitionId,
352
+ table: sub.shape,
353
+ scopes: effectiveScopes,
354
+ cursor,
355
+ limitCommits,
356
+ })
357
+ : null;
358
+ // Collect rows and compute nextCursor in a single pass
359
+ const incrementalRows = [];
360
+ let nextCursor = cursor;
361
+ if (pullRowStream) {
362
+ // Streaming path: process rows as they arrive
363
+ for await (const row of pullRowStream) {
364
+ incrementalRows.push(row);
365
+ nextCursor = Math.max(nextCursor, row.commit_seq);
366
+ }
367
+ }
368
+ else {
369
+ // Non-streaming fallback: load all rows at once
370
+ const rows = await dialect.readIncrementalPullRows(trx, {
371
+ partitionId,
372
+ table: sub.shape,
373
+ scopes: effectiveScopes,
374
+ cursor,
375
+ limitCommits,
376
+ });
377
+ incrementalRows.push(...rows);
378
+ for (const r of incrementalRows) {
379
+ nextCursor = Math.max(nextCursor, r.commit_seq);
380
+ }
381
+ }
382
+ nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
383
+ if (incrementalRows.length === 0) {
384
+ subResponses.push({
385
+ id: sub.id,
386
+ status: 'active',
387
+ scopes: effectiveScopes,
388
+ bootstrap: false,
389
+ nextCursor,
390
+ commits: [],
391
+ });
392
+ nextCursors.push(nextCursor);
231
393
  continue;
232
394
  }
233
- if (nextState.tableIndex + 1 < nextState.tables.length) {
234
- nextState = {
235
- ...nextState,
236
- tableIndex: nextState.tableIndex + 1,
237
- rowCursor: null,
238
- };
395
+ if (dedupeRows) {
396
+ const latestByRowKey = new Map();
397
+ for (const r of incrementalRows) {
398
+ const rowKey = `${r.table}\u0000${r.row_id}`;
399
+ const change = {
400
+ table: r.table,
401
+ row_id: r.row_id,
402
+ op: r.op,
403
+ row_json: r.row_json,
404
+ row_version: r.row_version,
405
+ scopes: dialect.dbToScopes(r.scopes),
406
+ };
407
+ latestByRowKey.set(rowKey, {
408
+ commitSeq: r.commit_seq,
409
+ createdAt: r.created_at,
410
+ actorId: r.actor_id,
411
+ changeId: r.change_id,
412
+ change,
413
+ });
414
+ }
415
+ const latest = Array.from(latestByRowKey.values()).sort((a, b) => a.commitSeq - b.commitSeq || a.changeId - b.changeId);
416
+ const commitsBySeq = new Map();
417
+ for (const item of latest) {
418
+ let commit = commitsBySeq.get(item.commitSeq);
419
+ if (!commit) {
420
+ commit = {
421
+ commitSeq: item.commitSeq,
422
+ createdAt: item.createdAt,
423
+ actorId: item.actorId,
424
+ changes: [],
425
+ };
426
+ commitsBySeq.set(item.commitSeq, commit);
427
+ }
428
+ commit.changes.push(item.change);
429
+ }
430
+ const commits = Array.from(commitsBySeq.values()).sort((a, b) => a.commitSeq - b.commitSeq);
431
+ subResponses.push({
432
+ id: sub.id,
433
+ status: 'active',
434
+ scopes: effectiveScopes,
435
+ bootstrap: false,
436
+ nextCursor,
437
+ commits,
438
+ });
439
+ nextCursors.push(nextCursor);
239
440
  continue;
240
441
  }
241
- nextState = null;
242
- break;
243
- }
244
- subResponses.push({
245
- id: sub.id,
246
- status: 'active',
247
- scopes: effectiveScopes,
248
- bootstrap: true,
249
- bootstrapState: nextState,
250
- nextCursor: effectiveState.asOfCommitSeq,
251
- commits: [],
252
- snapshots,
253
- });
254
- nextCursors.push(effectiveState.asOfCommitSeq);
255
- continue;
256
- }
257
- // Incremental pull for this subscription
258
- // Read the commit window for this table up-front so the subscription cursor
259
- // can advance past commits that don't match the requested scopes.
260
- const scannedCommitSeqs = await dialect.readCommitSeqsForPull(trx, {
261
- partitionId,
262
- cursor,
263
- limitCommits,
264
- tables: [sub.shape],
265
- });
266
- const maxScannedCommitSeq = scannedCommitSeqs.length > 0
267
- ? scannedCommitSeqs[scannedCommitSeqs.length - 1]
268
- : cursor;
269
- // Use streaming when available to reduce memory pressure for large pulls
270
- const pullRowStream = dialect.streamIncrementalPullRows
271
- ? dialect.streamIncrementalPullRows(trx, {
272
- partitionId,
273
- table: sub.shape,
274
- scopes: effectiveScopes,
275
- cursor,
276
- limitCommits,
277
- })
278
- : null;
279
- // Collect rows and compute nextCursor in a single pass
280
- const incrementalRows = [];
281
- let nextCursor = cursor;
282
- if (pullRowStream) {
283
- // Streaming path: process rows as they arrive
284
- for await (const row of pullRowStream) {
285
- incrementalRows.push(row);
286
- nextCursor = Math.max(nextCursor, row.commit_seq);
287
- }
288
- }
289
- else {
290
- // Non-streaming fallback: load all rows at once
291
- const rows = await dialect.readIncrementalPullRows(trx, {
292
- partitionId,
293
- table: sub.shape,
294
- scopes: effectiveScopes,
295
- cursor,
296
- limitCommits,
297
- });
298
- incrementalRows.push(...rows);
299
- for (const r of incrementalRows) {
300
- nextCursor = Math.max(nextCursor, r.commit_seq);
301
- }
302
- }
303
- nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
304
- if (incrementalRows.length === 0) {
305
- subResponses.push({
306
- id: sub.id,
307
- status: 'active',
308
- scopes: effectiveScopes,
309
- bootstrap: false,
310
- nextCursor,
311
- commits: [],
312
- });
313
- nextCursors.push(nextCursor);
314
- continue;
315
- }
316
- if (dedupeRows) {
317
- const latestByRowKey = new Map();
318
- for (const r of incrementalRows) {
319
- const rowKey = `${r.table}\u0000${r.row_id}`;
320
- const change = {
321
- table: r.table,
322
- row_id: r.row_id,
323
- op: r.op,
324
- row_json: r.row_json,
325
- row_version: r.row_version,
326
- scopes: dialect.dbToScopes(r.scopes),
327
- };
328
- latestByRowKey.set(rowKey, {
329
- commitSeq: r.commit_seq,
330
- createdAt: r.created_at,
331
- actorId: r.actor_id,
332
- changeId: r.change_id,
333
- change,
334
- });
335
- }
336
- const latest = Array.from(latestByRowKey.values()).sort((a, b) => a.commitSeq - b.commitSeq || a.changeId - b.changeId);
337
- const commitsBySeq = new Map();
338
- for (const item of latest) {
339
- let commit = commitsBySeq.get(item.commitSeq);
340
- if (!commit) {
341
- commit = {
342
- commitSeq: item.commitSeq,
343
- createdAt: item.createdAt,
344
- actorId: item.actorId,
345
- changes: [],
442
+ const commitsBySeq = new Map();
443
+ const commitSeqs = [];
444
+ for (const r of incrementalRows) {
445
+ const seq = r.commit_seq;
446
+ let commit = commitsBySeq.get(seq);
447
+ if (!commit) {
448
+ commit = {
449
+ commitSeq: seq,
450
+ createdAt: r.created_at,
451
+ actorId: r.actor_id,
452
+ changes: [],
453
+ };
454
+ commitsBySeq.set(seq, commit);
455
+ commitSeqs.push(seq);
456
+ }
457
+ const change = {
458
+ table: r.table,
459
+ row_id: r.row_id,
460
+ op: r.op,
461
+ row_json: r.row_json,
462
+ row_version: r.row_version,
463
+ scopes: dialect.dbToScopes(r.scopes),
346
464
  };
347
- commitsBySeq.set(item.commitSeq, commit);
465
+ commit.changes.push(change);
348
466
  }
349
- commit.changes.push(item.change);
350
- }
351
- const commits = Array.from(commitsBySeq.values()).sort((a, b) => a.commitSeq - b.commitSeq);
352
- subResponses.push({
353
- id: sub.id,
354
- status: 'active',
355
- scopes: effectiveScopes,
356
- bootstrap: false,
357
- nextCursor,
358
- commits,
359
- });
360
- nextCursors.push(nextCursor);
361
- continue;
362
- }
363
- const commitsBySeq = new Map();
364
- const commitSeqs = [];
365
- for (const r of incrementalRows) {
366
- const seq = r.commit_seq;
367
- let commit = commitsBySeq.get(seq);
368
- if (!commit) {
369
- commit = {
370
- commitSeq: seq,
371
- createdAt: r.created_at,
372
- actorId: r.actor_id,
373
- changes: [],
374
- };
375
- commitsBySeq.set(seq, commit);
376
- commitSeqs.push(seq);
467
+ const commits = commitSeqs
468
+ .map((seq) => commitsBySeq.get(seq))
469
+ .filter((c) => !!c)
470
+ .filter((c) => c.changes.length > 0);
471
+ subResponses.push({
472
+ id: sub.id,
473
+ status: 'active',
474
+ scopes: effectiveScopes,
475
+ bootstrap: false,
476
+ nextCursor,
477
+ commits,
478
+ });
479
+ nextCursors.push(nextCursor);
377
480
  }
378
- const change = {
379
- table: r.table,
380
- row_id: r.row_id,
381
- op: r.op,
382
- row_json: r.row_json,
383
- row_version: r.row_version,
384
- scopes: dialect.dbToScopes(r.scopes),
481
+ const effectiveScopes = mergeScopes(activeSubscriptions);
482
+ const clientCursor = nextCursors.length > 0 ? Math.min(...nextCursors) : maxCommitSeq;
483
+ return {
484
+ response: {
485
+ ok: true,
486
+ subscriptions: subResponses,
487
+ },
488
+ effectiveScopes,
489
+ clientCursor,
385
490
  };
386
- commit.changes.push(change);
387
- }
388
- const commits = commitSeqs
389
- .map((seq) => commitsBySeq.get(seq))
390
- .filter((c) => !!c)
391
- .filter((c) => c.changes.length > 0);
392
- subResponses.push({
393
- id: sub.id,
394
- status: 'active',
395
- scopes: effectiveScopes,
396
- bootstrap: false,
397
- nextCursor,
398
- commits,
399
491
  });
400
- nextCursors.push(nextCursor);
492
+ const durationMs = Math.max(0, Date.now() - startedAtMs);
493
+ const stats = summarizePullResponse(result.response);
494
+ span.setAttribute('status', 'ok');
495
+ span.setAttribute('duration_ms', durationMs);
496
+ span.setAttribute('subscription_count', stats.subscriptionCount);
497
+ span.setAttribute('commit_count', stats.commitCount);
498
+ span.setAttribute('change_count', stats.changeCount);
499
+ span.setAttribute('snapshot_page_count', stats.snapshotPageCount);
500
+ span.setStatus('ok');
501
+ recordPullMetrics({
502
+ status: 'ok',
503
+ dedupeRows,
504
+ durationMs,
505
+ stats,
506
+ });
507
+ return result;
508
+ }
509
+ catch (error) {
510
+ const durationMs = Math.max(0, Date.now() - startedAtMs);
511
+ span.setAttribute('status', 'error');
512
+ span.setAttribute('duration_ms', durationMs);
513
+ span.setStatus('error');
514
+ recordPullMetrics({
515
+ status: 'error',
516
+ dedupeRows: request.dedupeRows === true,
517
+ durationMs,
518
+ stats: {
519
+ subscriptionCount: 0,
520
+ activeSubscriptionCount: 0,
521
+ revokedSubscriptionCount: 0,
522
+ bootstrapSubscriptionCount: 0,
523
+ commitCount: 0,
524
+ changeCount: 0,
525
+ snapshotPageCount: 0,
526
+ },
527
+ });
528
+ captureSyncException(error, {
529
+ event: 'sync.server.pull',
530
+ requestedSubscriptionCount,
531
+ dedupeRows: request.dedupeRows === true,
532
+ });
533
+ throw error;
401
534
  }
402
- const effectiveScopes = mergeScopes(activeSubscriptions);
403
- const clientCursor = nextCursors.length > 0 ? Math.min(...nextCursors) : maxCommitSeq;
404
- return {
405
- response: {
406
- ok: true,
407
- subscriptions: subResponses,
408
- },
409
- effectiveScopes,
410
- clientCursor,
411
- };
412
535
  });
413
536
  }
414
537
  //# sourceMappingURL=pull.js.map