@syncular/server 0.0.1-73 → 0.0.1-88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobs/adapters/database.d.ts.map +1 -1
- package/dist/blobs/adapters/database.js +25 -3
- package/dist/blobs/adapters/database.js.map +1 -1
- package/dist/blobs/index.js +5 -5
- package/dist/dialect/base.js +1 -1
- package/dist/dialect/index.js +3 -3
- package/dist/helpers/index.js +4 -4
- package/dist/index.js +16 -16
- package/dist/proxy/handler.d.ts.map +1 -1
- package/dist/proxy/handler.js +7 -4
- package/dist/proxy/handler.js.map +1 -1
- package/dist/proxy/index.js +3 -3
- package/dist/proxy/mutation-detector.d.ts +4 -0
- package/dist/proxy/mutation-detector.d.ts.map +1 -1
- package/dist/proxy/mutation-detector.js +209 -24
- package/dist/proxy/mutation-detector.js.map +1 -1
- package/dist/pull.d.ts +1 -1
- package/dist/pull.d.ts.map +1 -1
- package/dist/pull.js +445 -322
- package/dist/pull.js.map +1 -1
- package/dist/push.d.ts +1 -1
- package/dist/push.d.ts.map +1 -1
- package/dist/push.js +341 -260
- package/dist/push.js.map +1 -1
- package/dist/realtime/index.js +1 -1
- package/dist/shapes/index.js +3 -3
- package/dist/snapshot-chunks/adapters/s3.js +1 -1
- package/dist/snapshot-chunks/db-metadata.d.ts.map +1 -1
- package/dist/snapshot-chunks/db-metadata.js +47 -41
- package/dist/snapshot-chunks/db-metadata.js.map +1 -1
- package/dist/snapshot-chunks/index.js +3 -3
- package/dist/subscriptions/index.js +1 -1
- package/package.json +1 -1
- package/src/blobs/adapters/database.test.ts +67 -0
- package/src/blobs/adapters/database.ts +34 -9
- package/src/proxy/handler.test.ts +120 -0
- package/src/proxy/handler.ts +9 -2
- package/src/proxy/mutation-detector.test.ts +71 -0
- package/src/proxy/mutation-detector.ts +227 -29
- package/src/pull.ts +609 -418
- package/src/push.ts +473 -349
- package/src/snapshot-chunks/db-metadata.test.ts +100 -0
- package/src/snapshot-chunks/db-metadata.ts +68 -48
package/dist/pull.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { createHash, randomUUID } from 'node:crypto';
|
|
2
2
|
import { promisify } from 'node:util';
|
|
3
3
|
import { gzip, gzipSync } from 'node:zlib';
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import { captureSyncException, countSyncMetric, distributionSyncMetric, startSyncSpan, } from '@syncular/core';
|
|
5
|
+
import { insertSnapshotChunk, readSnapshotChunkRefByPageKey, } from './snapshot-chunks.js';
|
|
6
|
+
import { resolveEffectiveScopesForSubscriptions } from './subscriptions/resolve.js';
|
|
6
7
|
const gzipAsync = promisify(gzip);
|
|
7
8
|
const ASYNC_GZIP_MIN_BYTES = 64 * 1024;
|
|
8
9
|
async function compressSnapshotNdjson(ndjson) {
|
|
@@ -59,152 +60,187 @@ function mergeScopes(subscriptions) {
|
|
|
59
60
|
}
|
|
60
61
|
return merged;
|
|
61
62
|
}
|
|
63
|
+
function summarizePullResponse(response) {
|
|
64
|
+
const subscriptions = response.subscriptions ?? [];
|
|
65
|
+
let activeSubscriptionCount = 0;
|
|
66
|
+
let revokedSubscriptionCount = 0;
|
|
67
|
+
let bootstrapSubscriptionCount = 0;
|
|
68
|
+
let commitCount = 0;
|
|
69
|
+
let changeCount = 0;
|
|
70
|
+
let snapshotPageCount = 0;
|
|
71
|
+
for (const sub of subscriptions) {
|
|
72
|
+
if (sub.status === 'revoked') {
|
|
73
|
+
revokedSubscriptionCount += 1;
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
activeSubscriptionCount += 1;
|
|
77
|
+
}
|
|
78
|
+
if (sub.bootstrap) {
|
|
79
|
+
bootstrapSubscriptionCount += 1;
|
|
80
|
+
}
|
|
81
|
+
const commits = sub.commits ?? [];
|
|
82
|
+
commitCount += commits.length;
|
|
83
|
+
for (const commit of commits) {
|
|
84
|
+
changeCount += commit.changes?.length ?? 0;
|
|
85
|
+
}
|
|
86
|
+
snapshotPageCount += sub.snapshots?.length ?? 0;
|
|
87
|
+
}
|
|
88
|
+
return {
|
|
89
|
+
subscriptionCount: subscriptions.length,
|
|
90
|
+
activeSubscriptionCount,
|
|
91
|
+
revokedSubscriptionCount,
|
|
92
|
+
bootstrapSubscriptionCount,
|
|
93
|
+
commitCount,
|
|
94
|
+
changeCount,
|
|
95
|
+
snapshotPageCount,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
function recordPullMetrics(args) {
|
|
99
|
+
const { status, dedupeRows, durationMs, stats } = args;
|
|
100
|
+
const attributes = {
|
|
101
|
+
status,
|
|
102
|
+
dedupe_rows: dedupeRows,
|
|
103
|
+
};
|
|
104
|
+
countSyncMetric('sync.server.pull.requests', 1, { attributes });
|
|
105
|
+
distributionSyncMetric('sync.server.pull.duration_ms', durationMs, {
|
|
106
|
+
unit: 'millisecond',
|
|
107
|
+
attributes,
|
|
108
|
+
});
|
|
109
|
+
distributionSyncMetric('sync.server.pull.subscriptions', stats.subscriptionCount, { attributes });
|
|
110
|
+
distributionSyncMetric('sync.server.pull.active_subscriptions', stats.activeSubscriptionCount, { attributes });
|
|
111
|
+
distributionSyncMetric('sync.server.pull.revoked_subscriptions', stats.revokedSubscriptionCount, { attributes });
|
|
112
|
+
distributionSyncMetric('sync.server.pull.bootstrap_subscriptions', stats.bootstrapSubscriptionCount, { attributes });
|
|
113
|
+
distributionSyncMetric('sync.server.pull.commits', stats.commitCount, {
|
|
114
|
+
attributes,
|
|
115
|
+
});
|
|
116
|
+
distributionSyncMetric('sync.server.pull.changes', stats.changeCount, {
|
|
117
|
+
attributes,
|
|
118
|
+
});
|
|
119
|
+
distributionSyncMetric('sync.server.pull.snapshot_pages', stats.snapshotPageCount, { attributes });
|
|
120
|
+
}
|
|
62
121
|
export async function pull(args) {
|
|
63
122
|
const { request, dialect } = args;
|
|
64
123
|
const db = args.db;
|
|
65
124
|
const partitionId = args.partitionId ?? 'default';
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
const
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
})
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
nextCursor: cursor,
|
|
96
|
-
commits: [],
|
|
125
|
+
const requestedSubscriptionCount = Array.isArray(request.subscriptions)
|
|
126
|
+
? request.subscriptions.length
|
|
127
|
+
: 0;
|
|
128
|
+
const startedAtMs = Date.now();
|
|
129
|
+
return startSyncSpan({
|
|
130
|
+
name: 'sync.server.pull',
|
|
131
|
+
op: 'sync.pull',
|
|
132
|
+
attributes: {
|
|
133
|
+
requested_subscription_count: requestedSubscriptionCount,
|
|
134
|
+
dedupe_rows: request.dedupeRows === true,
|
|
135
|
+
},
|
|
136
|
+
}, async (span) => {
|
|
137
|
+
try {
|
|
138
|
+
// Validate and sanitize request limits
|
|
139
|
+
const limitCommits = sanitizeLimit(request.limitCommits, 50, 1, 500);
|
|
140
|
+
const limitSnapshotRows = sanitizeLimit(request.limitSnapshotRows, 1000, 1, 5000);
|
|
141
|
+
const maxSnapshotPages = sanitizeLimit(request.maxSnapshotPages, 1, 1, 50);
|
|
142
|
+
const dedupeRows = request.dedupeRows === true;
|
|
143
|
+
// Resolve effective scopes for each subscription
|
|
144
|
+
const resolved = await resolveEffectiveScopesForSubscriptions({
|
|
145
|
+
db,
|
|
146
|
+
actorId: args.actorId,
|
|
147
|
+
subscriptions: request.subscriptions ?? [],
|
|
148
|
+
shapes: args.shapes,
|
|
149
|
+
});
|
|
150
|
+
const result = await dialect.executeInTransaction(db, async (trx) => {
|
|
151
|
+
await dialect.setRepeatableRead(trx);
|
|
152
|
+
const maxCommitSeq = await dialect.readMaxCommitSeq(trx, {
|
|
153
|
+
partitionId,
|
|
97
154
|
});
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
Array.isArray(requestedState.tables) &&
|
|
120
|
-
typeof requestedState.tableIndex === 'number'
|
|
121
|
-
? requestedState
|
|
122
|
-
: initState;
|
|
123
|
-
// If the bootstrap state's asOfCommitSeq is no longer catch-up-able, restart bootstrap.
|
|
124
|
-
const effectiveState = state.asOfCommitSeq < minCommitSeq - 1 ? initState : state;
|
|
125
|
-
const tableName = effectiveState.tables[effectiveState.tableIndex];
|
|
126
|
-
// No tables (or ran past the end): treat bootstrap as complete.
|
|
127
|
-
if (!tableName) {
|
|
128
|
-
subResponses.push({
|
|
129
|
-
id: sub.id,
|
|
130
|
-
status: 'active',
|
|
131
|
-
scopes: effectiveScopes,
|
|
132
|
-
bootstrap: true,
|
|
133
|
-
bootstrapState: null,
|
|
134
|
-
nextCursor: effectiveState.asOfCommitSeq,
|
|
135
|
-
commits: [],
|
|
136
|
-
snapshots: [],
|
|
137
|
-
});
|
|
138
|
-
nextCursors.push(effectiveState.asOfCommitSeq);
|
|
139
|
-
continue;
|
|
140
|
-
}
|
|
141
|
-
const snapshots = [];
|
|
142
|
-
let nextState = effectiveState;
|
|
143
|
-
for (let pageIndex = 0; pageIndex < maxSnapshotPages; pageIndex++) {
|
|
144
|
-
if (!nextState)
|
|
145
|
-
break;
|
|
146
|
-
const nextTableName = nextState.tables[nextState.tableIndex];
|
|
147
|
-
if (!nextTableName) {
|
|
148
|
-
nextState = null;
|
|
149
|
-
break;
|
|
155
|
+
const minCommitSeq = await dialect.readMinCommitSeq(trx, {
|
|
156
|
+
partitionId,
|
|
157
|
+
});
|
|
158
|
+
const subResponses = [];
|
|
159
|
+
const activeSubscriptions = [];
|
|
160
|
+
const nextCursors = [];
|
|
161
|
+
for (const sub of resolved) {
|
|
162
|
+
const cursor = Math.max(-1, sub.cursor ?? -1);
|
|
163
|
+
// Validate shape exists (throws if not registered)
|
|
164
|
+
args.shapes.getOrThrow(sub.shape);
|
|
165
|
+
if (sub.status === 'revoked' ||
|
|
166
|
+
Object.keys(sub.scopes).length === 0) {
|
|
167
|
+
subResponses.push({
|
|
168
|
+
id: sub.id,
|
|
169
|
+
status: 'revoked',
|
|
170
|
+
scopes: {},
|
|
171
|
+
bootstrap: false,
|
|
172
|
+
nextCursor: cursor,
|
|
173
|
+
commits: [],
|
|
174
|
+
});
|
|
175
|
+
continue;
|
|
150
176
|
}
|
|
151
|
-
const
|
|
152
|
-
|
|
153
|
-
const
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
// Use external chunk storage if available, otherwise fall back to inline
|
|
189
|
-
if (args.chunkStorage) {
|
|
190
|
-
chunkRef = await args.chunkStorage.storeChunk({
|
|
191
|
-
partitionId,
|
|
192
|
-
scopeKey: cacheKey,
|
|
193
|
-
scope: nextTableName,
|
|
194
|
-
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
195
|
-
rowCursor: nextState.rowCursor ?? null,
|
|
196
|
-
rowLimit: limitSnapshotRows,
|
|
197
|
-
encoding: 'ndjson',
|
|
198
|
-
compression: 'gzip',
|
|
199
|
-
sha256,
|
|
200
|
-
body: gz,
|
|
201
|
-
expiresAt,
|
|
177
|
+
const effectiveScopes = sub.scopes;
|
|
178
|
+
activeSubscriptions.push({ scopes: effectiveScopes });
|
|
179
|
+
const needsBootstrap = sub.bootstrapState != null ||
|
|
180
|
+
cursor < 0 ||
|
|
181
|
+
cursor > maxCommitSeq ||
|
|
182
|
+
(minCommitSeq > 0 && cursor < minCommitSeq - 1);
|
|
183
|
+
if (needsBootstrap) {
|
|
184
|
+
const tables = args.shapes
|
|
185
|
+
.getBootstrapOrderFor(sub.shape)
|
|
186
|
+
.map((handler) => handler.table);
|
|
187
|
+
const initState = {
|
|
188
|
+
asOfCommitSeq: maxCommitSeq,
|
|
189
|
+
tables,
|
|
190
|
+
tableIndex: 0,
|
|
191
|
+
rowCursor: null,
|
|
192
|
+
};
|
|
193
|
+
const requestedState = sub.bootstrapState ?? null;
|
|
194
|
+
const state = requestedState &&
|
|
195
|
+
typeof requestedState.asOfCommitSeq === 'number' &&
|
|
196
|
+
Array.isArray(requestedState.tables) &&
|
|
197
|
+
typeof requestedState.tableIndex === 'number'
|
|
198
|
+
? requestedState
|
|
199
|
+
: initState;
|
|
200
|
+
// If the bootstrap state's asOfCommitSeq is no longer catch-up-able, restart bootstrap.
|
|
201
|
+
const effectiveState = state.asOfCommitSeq < minCommitSeq - 1 ? initState : state;
|
|
202
|
+
const tableName = effectiveState.tables[effectiveState.tableIndex];
|
|
203
|
+
// No tables (or ran past the end): treat bootstrap as complete.
|
|
204
|
+
if (!tableName) {
|
|
205
|
+
subResponses.push({
|
|
206
|
+
id: sub.id,
|
|
207
|
+
status: 'active',
|
|
208
|
+
scopes: effectiveScopes,
|
|
209
|
+
bootstrap: true,
|
|
210
|
+
bootstrapState: null,
|
|
211
|
+
nextCursor: effectiveState.asOfCommitSeq,
|
|
212
|
+
commits: [],
|
|
213
|
+
snapshots: [],
|
|
202
214
|
});
|
|
215
|
+
nextCursors.push(effectiveState.asOfCommitSeq);
|
|
216
|
+
continue;
|
|
203
217
|
}
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
218
|
+
const snapshots = [];
|
|
219
|
+
let nextState = effectiveState;
|
|
220
|
+
for (let pageIndex = 0; pageIndex < maxSnapshotPages; pageIndex++) {
|
|
221
|
+
if (!nextState)
|
|
222
|
+
break;
|
|
223
|
+
const nextTableName = nextState.tables[nextState.tableIndex];
|
|
224
|
+
if (!nextTableName) {
|
|
225
|
+
nextState = null;
|
|
226
|
+
break;
|
|
227
|
+
}
|
|
228
|
+
const tableHandler = args.shapes.getOrThrow(nextTableName);
|
|
229
|
+
const isFirstPage = nextState.rowCursor == null;
|
|
230
|
+
const page = await tableHandler.snapshot({
|
|
231
|
+
db: trx,
|
|
232
|
+
actorId: args.actorId,
|
|
233
|
+
scopeValues: effectiveScopes,
|
|
234
|
+
cursor: nextState.rowCursor,
|
|
235
|
+
limit: limitSnapshotRows,
|
|
236
|
+
}, sub.params);
|
|
237
|
+
const isLastPage = page.nextCursor == null;
|
|
238
|
+
// Always use NDJSON+gzip for bootstrap snapshots
|
|
239
|
+
const ttlMs = tableHandler.snapshotChunkTtlMs ?? 24 * 60 * 60 * 1000; // 24h
|
|
240
|
+
const nowIso = new Date().toISOString();
|
|
241
|
+
// Use scope hash for caching
|
|
242
|
+
const cacheKey = `${partitionId}:${scopesToCacheKey(effectiveScopes)}`;
|
|
243
|
+
const cached = await readSnapshotChunkRefByPageKey(trx, {
|
|
208
244
|
partitionId,
|
|
209
245
|
scopeKey: cacheKey,
|
|
210
246
|
scope: nextTableName,
|
|
@@ -213,202 +249,289 @@ export async function pull(args) {
|
|
|
213
249
|
rowLimit: limitSnapshotRows,
|
|
214
250
|
encoding: 'ndjson',
|
|
215
251
|
compression: 'gzip',
|
|
216
|
-
|
|
217
|
-
body: gz,
|
|
218
|
-
expiresAt,
|
|
252
|
+
nowIso,
|
|
219
253
|
});
|
|
254
|
+
let chunkRef = cached;
|
|
255
|
+
if (!chunkRef) {
|
|
256
|
+
const lines = [];
|
|
257
|
+
for (const r of page.rows ?? []) {
|
|
258
|
+
const s = JSON.stringify(r);
|
|
259
|
+
lines.push(s === undefined ? 'null' : s);
|
|
260
|
+
}
|
|
261
|
+
const ndjson = lines.length > 0 ? `${lines.join('\n')}\n` : '';
|
|
262
|
+
const gz = await compressSnapshotNdjson(ndjson);
|
|
263
|
+
const sha256 = createHash('sha256')
|
|
264
|
+
.update(ndjson)
|
|
265
|
+
.digest('hex');
|
|
266
|
+
const expiresAt = new Date(Date.now() + Math.max(1000, ttlMs)).toISOString();
|
|
267
|
+
// Use external chunk storage if available, otherwise fall back to inline
|
|
268
|
+
if (args.chunkStorage) {
|
|
269
|
+
chunkRef = await args.chunkStorage.storeChunk({
|
|
270
|
+
partitionId,
|
|
271
|
+
scopeKey: cacheKey,
|
|
272
|
+
scope: nextTableName,
|
|
273
|
+
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
274
|
+
rowCursor: nextState.rowCursor ?? null,
|
|
275
|
+
rowLimit: limitSnapshotRows,
|
|
276
|
+
encoding: 'ndjson',
|
|
277
|
+
compression: 'gzip',
|
|
278
|
+
sha256,
|
|
279
|
+
body: gz,
|
|
280
|
+
expiresAt,
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
else {
|
|
284
|
+
const chunkId = randomUUID();
|
|
285
|
+
chunkRef = await insertSnapshotChunk(trx, {
|
|
286
|
+
chunkId,
|
|
287
|
+
partitionId,
|
|
288
|
+
scopeKey: cacheKey,
|
|
289
|
+
scope: nextTableName,
|
|
290
|
+
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
291
|
+
rowCursor: nextState.rowCursor,
|
|
292
|
+
rowLimit: limitSnapshotRows,
|
|
293
|
+
encoding: 'ndjson',
|
|
294
|
+
compression: 'gzip',
|
|
295
|
+
sha256,
|
|
296
|
+
body: gz,
|
|
297
|
+
expiresAt,
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
snapshots.push({
|
|
302
|
+
table: nextTableName,
|
|
303
|
+
rows: [],
|
|
304
|
+
chunks: [chunkRef],
|
|
305
|
+
isFirstPage,
|
|
306
|
+
isLastPage,
|
|
307
|
+
});
|
|
308
|
+
if (page.nextCursor != null) {
|
|
309
|
+
nextState = { ...nextState, rowCursor: page.nextCursor };
|
|
310
|
+
continue;
|
|
311
|
+
}
|
|
312
|
+
if (nextState.tableIndex + 1 < nextState.tables.length) {
|
|
313
|
+
nextState = {
|
|
314
|
+
...nextState,
|
|
315
|
+
tableIndex: nextState.tableIndex + 1,
|
|
316
|
+
rowCursor: null,
|
|
317
|
+
};
|
|
318
|
+
continue;
|
|
319
|
+
}
|
|
320
|
+
nextState = null;
|
|
321
|
+
break;
|
|
220
322
|
}
|
|
323
|
+
subResponses.push({
|
|
324
|
+
id: sub.id,
|
|
325
|
+
status: 'active',
|
|
326
|
+
scopes: effectiveScopes,
|
|
327
|
+
bootstrap: true,
|
|
328
|
+
bootstrapState: nextState,
|
|
329
|
+
nextCursor: effectiveState.asOfCommitSeq,
|
|
330
|
+
commits: [],
|
|
331
|
+
snapshots,
|
|
332
|
+
});
|
|
333
|
+
nextCursors.push(effectiveState.asOfCommitSeq);
|
|
334
|
+
continue;
|
|
221
335
|
}
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
336
|
+
// Incremental pull for this subscription
|
|
337
|
+
// Read the commit window for this table up-front so the subscription cursor
|
|
338
|
+
// can advance past commits that don't match the requested scopes.
|
|
339
|
+
const scannedCommitSeqs = await dialect.readCommitSeqsForPull(trx, {
|
|
340
|
+
partitionId,
|
|
341
|
+
cursor,
|
|
342
|
+
limitCommits,
|
|
343
|
+
tables: [sub.shape],
|
|
228
344
|
});
|
|
229
|
-
|
|
230
|
-
|
|
345
|
+
const maxScannedCommitSeq = scannedCommitSeqs.length > 0
|
|
346
|
+
? scannedCommitSeqs[scannedCommitSeqs.length - 1]
|
|
347
|
+
: cursor;
|
|
348
|
+
// Use streaming when available to reduce memory pressure for large pulls
|
|
349
|
+
const pullRowStream = dialect.streamIncrementalPullRows
|
|
350
|
+
? dialect.streamIncrementalPullRows(trx, {
|
|
351
|
+
partitionId,
|
|
352
|
+
table: sub.shape,
|
|
353
|
+
scopes: effectiveScopes,
|
|
354
|
+
cursor,
|
|
355
|
+
limitCommits,
|
|
356
|
+
})
|
|
357
|
+
: null;
|
|
358
|
+
// Collect rows and compute nextCursor in a single pass
|
|
359
|
+
const incrementalRows = [];
|
|
360
|
+
let nextCursor = cursor;
|
|
361
|
+
if (pullRowStream) {
|
|
362
|
+
// Streaming path: process rows as they arrive
|
|
363
|
+
for await (const row of pullRowStream) {
|
|
364
|
+
incrementalRows.push(row);
|
|
365
|
+
nextCursor = Math.max(nextCursor, row.commit_seq);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
else {
|
|
369
|
+
// Non-streaming fallback: load all rows at once
|
|
370
|
+
const rows = await dialect.readIncrementalPullRows(trx, {
|
|
371
|
+
partitionId,
|
|
372
|
+
table: sub.shape,
|
|
373
|
+
scopes: effectiveScopes,
|
|
374
|
+
cursor,
|
|
375
|
+
limitCommits,
|
|
376
|
+
});
|
|
377
|
+
incrementalRows.push(...rows);
|
|
378
|
+
for (const r of incrementalRows) {
|
|
379
|
+
nextCursor = Math.max(nextCursor, r.commit_seq);
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
|
|
383
|
+
if (incrementalRows.length === 0) {
|
|
384
|
+
subResponses.push({
|
|
385
|
+
id: sub.id,
|
|
386
|
+
status: 'active',
|
|
387
|
+
scopes: effectiveScopes,
|
|
388
|
+
bootstrap: false,
|
|
389
|
+
nextCursor,
|
|
390
|
+
commits: [],
|
|
391
|
+
});
|
|
392
|
+
nextCursors.push(nextCursor);
|
|
231
393
|
continue;
|
|
232
394
|
}
|
|
233
|
-
if (
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
395
|
+
if (dedupeRows) {
|
|
396
|
+
const latestByRowKey = new Map();
|
|
397
|
+
for (const r of incrementalRows) {
|
|
398
|
+
const rowKey = `${r.table}\u0000${r.row_id}`;
|
|
399
|
+
const change = {
|
|
400
|
+
table: r.table,
|
|
401
|
+
row_id: r.row_id,
|
|
402
|
+
op: r.op,
|
|
403
|
+
row_json: r.row_json,
|
|
404
|
+
row_version: r.row_version,
|
|
405
|
+
scopes: dialect.dbToScopes(r.scopes),
|
|
406
|
+
};
|
|
407
|
+
latestByRowKey.set(rowKey, {
|
|
408
|
+
commitSeq: r.commit_seq,
|
|
409
|
+
createdAt: r.created_at,
|
|
410
|
+
actorId: r.actor_id,
|
|
411
|
+
changeId: r.change_id,
|
|
412
|
+
change,
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
const latest = Array.from(latestByRowKey.values()).sort((a, b) => a.commitSeq - b.commitSeq || a.changeId - b.changeId);
|
|
416
|
+
const commitsBySeq = new Map();
|
|
417
|
+
for (const item of latest) {
|
|
418
|
+
let commit = commitsBySeq.get(item.commitSeq);
|
|
419
|
+
if (!commit) {
|
|
420
|
+
commit = {
|
|
421
|
+
commitSeq: item.commitSeq,
|
|
422
|
+
createdAt: item.createdAt,
|
|
423
|
+
actorId: item.actorId,
|
|
424
|
+
changes: [],
|
|
425
|
+
};
|
|
426
|
+
commitsBySeq.set(item.commitSeq, commit);
|
|
427
|
+
}
|
|
428
|
+
commit.changes.push(item.change);
|
|
429
|
+
}
|
|
430
|
+
const commits = Array.from(commitsBySeq.values()).sort((a, b) => a.commitSeq - b.commitSeq);
|
|
431
|
+
subResponses.push({
|
|
432
|
+
id: sub.id,
|
|
433
|
+
status: 'active',
|
|
434
|
+
scopes: effectiveScopes,
|
|
435
|
+
bootstrap: false,
|
|
436
|
+
nextCursor,
|
|
437
|
+
commits,
|
|
438
|
+
});
|
|
439
|
+
nextCursors.push(nextCursor);
|
|
239
440
|
continue;
|
|
240
441
|
}
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
limitCommits,
|
|
264
|
-
tables: [sub.shape],
|
|
265
|
-
});
|
|
266
|
-
const maxScannedCommitSeq = scannedCommitSeqs.length > 0
|
|
267
|
-
? scannedCommitSeqs[scannedCommitSeqs.length - 1]
|
|
268
|
-
: cursor;
|
|
269
|
-
// Use streaming when available to reduce memory pressure for large pulls
|
|
270
|
-
const pullRowStream = dialect.streamIncrementalPullRows
|
|
271
|
-
? dialect.streamIncrementalPullRows(trx, {
|
|
272
|
-
partitionId,
|
|
273
|
-
table: sub.shape,
|
|
274
|
-
scopes: effectiveScopes,
|
|
275
|
-
cursor,
|
|
276
|
-
limitCommits,
|
|
277
|
-
})
|
|
278
|
-
: null;
|
|
279
|
-
// Collect rows and compute nextCursor in a single pass
|
|
280
|
-
const incrementalRows = [];
|
|
281
|
-
let nextCursor = cursor;
|
|
282
|
-
if (pullRowStream) {
|
|
283
|
-
// Streaming path: process rows as they arrive
|
|
284
|
-
for await (const row of pullRowStream) {
|
|
285
|
-
incrementalRows.push(row);
|
|
286
|
-
nextCursor = Math.max(nextCursor, row.commit_seq);
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
else {
|
|
290
|
-
// Non-streaming fallback: load all rows at once
|
|
291
|
-
const rows = await dialect.readIncrementalPullRows(trx, {
|
|
292
|
-
partitionId,
|
|
293
|
-
table: sub.shape,
|
|
294
|
-
scopes: effectiveScopes,
|
|
295
|
-
cursor,
|
|
296
|
-
limitCommits,
|
|
297
|
-
});
|
|
298
|
-
incrementalRows.push(...rows);
|
|
299
|
-
for (const r of incrementalRows) {
|
|
300
|
-
nextCursor = Math.max(nextCursor, r.commit_seq);
|
|
301
|
-
}
|
|
302
|
-
}
|
|
303
|
-
nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
|
|
304
|
-
if (incrementalRows.length === 0) {
|
|
305
|
-
subResponses.push({
|
|
306
|
-
id: sub.id,
|
|
307
|
-
status: 'active',
|
|
308
|
-
scopes: effectiveScopes,
|
|
309
|
-
bootstrap: false,
|
|
310
|
-
nextCursor,
|
|
311
|
-
commits: [],
|
|
312
|
-
});
|
|
313
|
-
nextCursors.push(nextCursor);
|
|
314
|
-
continue;
|
|
315
|
-
}
|
|
316
|
-
if (dedupeRows) {
|
|
317
|
-
const latestByRowKey = new Map();
|
|
318
|
-
for (const r of incrementalRows) {
|
|
319
|
-
const rowKey = `${r.table}\u0000${r.row_id}`;
|
|
320
|
-
const change = {
|
|
321
|
-
table: r.table,
|
|
322
|
-
row_id: r.row_id,
|
|
323
|
-
op: r.op,
|
|
324
|
-
row_json: r.row_json,
|
|
325
|
-
row_version: r.row_version,
|
|
326
|
-
scopes: dialect.dbToScopes(r.scopes),
|
|
327
|
-
};
|
|
328
|
-
latestByRowKey.set(rowKey, {
|
|
329
|
-
commitSeq: r.commit_seq,
|
|
330
|
-
createdAt: r.created_at,
|
|
331
|
-
actorId: r.actor_id,
|
|
332
|
-
changeId: r.change_id,
|
|
333
|
-
change,
|
|
334
|
-
});
|
|
335
|
-
}
|
|
336
|
-
const latest = Array.from(latestByRowKey.values()).sort((a, b) => a.commitSeq - b.commitSeq || a.changeId - b.changeId);
|
|
337
|
-
const commitsBySeq = new Map();
|
|
338
|
-
for (const item of latest) {
|
|
339
|
-
let commit = commitsBySeq.get(item.commitSeq);
|
|
340
|
-
if (!commit) {
|
|
341
|
-
commit = {
|
|
342
|
-
commitSeq: item.commitSeq,
|
|
343
|
-
createdAt: item.createdAt,
|
|
344
|
-
actorId: item.actorId,
|
|
345
|
-
changes: [],
|
|
442
|
+
const commitsBySeq = new Map();
|
|
443
|
+
const commitSeqs = [];
|
|
444
|
+
for (const r of incrementalRows) {
|
|
445
|
+
const seq = r.commit_seq;
|
|
446
|
+
let commit = commitsBySeq.get(seq);
|
|
447
|
+
if (!commit) {
|
|
448
|
+
commit = {
|
|
449
|
+
commitSeq: seq,
|
|
450
|
+
createdAt: r.created_at,
|
|
451
|
+
actorId: r.actor_id,
|
|
452
|
+
changes: [],
|
|
453
|
+
};
|
|
454
|
+
commitsBySeq.set(seq, commit);
|
|
455
|
+
commitSeqs.push(seq);
|
|
456
|
+
}
|
|
457
|
+
const change = {
|
|
458
|
+
table: r.table,
|
|
459
|
+
row_id: r.row_id,
|
|
460
|
+
op: r.op,
|
|
461
|
+
row_json: r.row_json,
|
|
462
|
+
row_version: r.row_version,
|
|
463
|
+
scopes: dialect.dbToScopes(r.scopes),
|
|
346
464
|
};
|
|
347
|
-
|
|
465
|
+
commit.changes.push(change);
|
|
348
466
|
}
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
}
|
|
363
|
-
const commitsBySeq = new Map();
|
|
364
|
-
const commitSeqs = [];
|
|
365
|
-
for (const r of incrementalRows) {
|
|
366
|
-
const seq = r.commit_seq;
|
|
367
|
-
let commit = commitsBySeq.get(seq);
|
|
368
|
-
if (!commit) {
|
|
369
|
-
commit = {
|
|
370
|
-
commitSeq: seq,
|
|
371
|
-
createdAt: r.created_at,
|
|
372
|
-
actorId: r.actor_id,
|
|
373
|
-
changes: [],
|
|
374
|
-
};
|
|
375
|
-
commitsBySeq.set(seq, commit);
|
|
376
|
-
commitSeqs.push(seq);
|
|
467
|
+
const commits = commitSeqs
|
|
468
|
+
.map((seq) => commitsBySeq.get(seq))
|
|
469
|
+
.filter((c) => !!c)
|
|
470
|
+
.filter((c) => c.changes.length > 0);
|
|
471
|
+
subResponses.push({
|
|
472
|
+
id: sub.id,
|
|
473
|
+
status: 'active',
|
|
474
|
+
scopes: effectiveScopes,
|
|
475
|
+
bootstrap: false,
|
|
476
|
+
nextCursor,
|
|
477
|
+
commits,
|
|
478
|
+
});
|
|
479
|
+
nextCursors.push(nextCursor);
|
|
377
480
|
}
|
|
378
|
-
const
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
481
|
+
const effectiveScopes = mergeScopes(activeSubscriptions);
|
|
482
|
+
const clientCursor = nextCursors.length > 0 ? Math.min(...nextCursors) : maxCommitSeq;
|
|
483
|
+
return {
|
|
484
|
+
response: {
|
|
485
|
+
ok: true,
|
|
486
|
+
subscriptions: subResponses,
|
|
487
|
+
},
|
|
488
|
+
effectiveScopes,
|
|
489
|
+
clientCursor,
|
|
385
490
|
};
|
|
386
|
-
commit.changes.push(change);
|
|
387
|
-
}
|
|
388
|
-
const commits = commitSeqs
|
|
389
|
-
.map((seq) => commitsBySeq.get(seq))
|
|
390
|
-
.filter((c) => !!c)
|
|
391
|
-
.filter((c) => c.changes.length > 0);
|
|
392
|
-
subResponses.push({
|
|
393
|
-
id: sub.id,
|
|
394
|
-
status: 'active',
|
|
395
|
-
scopes: effectiveScopes,
|
|
396
|
-
bootstrap: false,
|
|
397
|
-
nextCursor,
|
|
398
|
-
commits,
|
|
399
491
|
});
|
|
400
|
-
|
|
492
|
+
const durationMs = Math.max(0, Date.now() - startedAtMs);
|
|
493
|
+
const stats = summarizePullResponse(result.response);
|
|
494
|
+
span.setAttribute('status', 'ok');
|
|
495
|
+
span.setAttribute('duration_ms', durationMs);
|
|
496
|
+
span.setAttribute('subscription_count', stats.subscriptionCount);
|
|
497
|
+
span.setAttribute('commit_count', stats.commitCount);
|
|
498
|
+
span.setAttribute('change_count', stats.changeCount);
|
|
499
|
+
span.setAttribute('snapshot_page_count', stats.snapshotPageCount);
|
|
500
|
+
span.setStatus('ok');
|
|
501
|
+
recordPullMetrics({
|
|
502
|
+
status: 'ok',
|
|
503
|
+
dedupeRows,
|
|
504
|
+
durationMs,
|
|
505
|
+
stats,
|
|
506
|
+
});
|
|
507
|
+
return result;
|
|
508
|
+
}
|
|
509
|
+
catch (error) {
|
|
510
|
+
const durationMs = Math.max(0, Date.now() - startedAtMs);
|
|
511
|
+
span.setAttribute('status', 'error');
|
|
512
|
+
span.setAttribute('duration_ms', durationMs);
|
|
513
|
+
span.setStatus('error');
|
|
514
|
+
recordPullMetrics({
|
|
515
|
+
status: 'error',
|
|
516
|
+
dedupeRows: request.dedupeRows === true,
|
|
517
|
+
durationMs,
|
|
518
|
+
stats: {
|
|
519
|
+
subscriptionCount: 0,
|
|
520
|
+
activeSubscriptionCount: 0,
|
|
521
|
+
revokedSubscriptionCount: 0,
|
|
522
|
+
bootstrapSubscriptionCount: 0,
|
|
523
|
+
commitCount: 0,
|
|
524
|
+
changeCount: 0,
|
|
525
|
+
snapshotPageCount: 0,
|
|
526
|
+
},
|
|
527
|
+
});
|
|
528
|
+
captureSyncException(error, {
|
|
529
|
+
event: 'sync.server.pull',
|
|
530
|
+
requestedSubscriptionCount,
|
|
531
|
+
dedupeRows: request.dedupeRows === true,
|
|
532
|
+
});
|
|
533
|
+
throw error;
|
|
401
534
|
}
|
|
402
|
-
const effectiveScopes = mergeScopes(activeSubscriptions);
|
|
403
|
-
const clientCursor = nextCursors.length > 0 ? Math.min(...nextCursors) : maxCommitSeq;
|
|
404
|
-
return {
|
|
405
|
-
response: {
|
|
406
|
-
ok: true,
|
|
407
|
-
subscriptions: subResponses,
|
|
408
|
-
},
|
|
409
|
-
effectiveScopes,
|
|
410
|
-
clientCursor,
|
|
411
|
-
};
|
|
412
535
|
});
|
|
413
536
|
}
|
|
414
537
|
//# sourceMappingURL=pull.js.map
|