@syncular/server 0.0.6-204 → 0.0.6-206
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dialect/types.d.ts +1 -0
- package/dist/dialect/types.d.ts.map +1 -1
- package/dist/pull.d.ts.map +1 -1
- package/dist/pull.js +546 -605
- package/dist/pull.js.map +1 -1
- package/dist/schema.d.ts +1 -1
- package/dist/schema.d.ts.map +1 -1
- package/dist/snapshot-chunks/db-metadata.d.ts.map +1 -1
- package/dist/snapshot-chunks/db-metadata.js +5 -39
- package/dist/snapshot-chunks/db-metadata.js.map +1 -1
- package/dist/subscriptions/cache.d.ts +3 -0
- package/dist/subscriptions/cache.d.ts.map +1 -1
- package/dist/subscriptions/cache.js +44 -0
- package/dist/subscriptions/cache.js.map +1 -1
- package/dist/subscriptions/resolve.d.ts.map +1 -1
- package/dist/subscriptions/resolve.js +62 -35
- package/dist/subscriptions/resolve.js.map +1 -1
- package/package.json +2 -2
- package/src/dialect/types.ts +1 -0
- package/src/notify.test.ts +2 -2
- package/src/pull.ts +740 -771
- package/src/schema.ts +1 -1
- package/src/snapshot-chunks/db-metadata.test.ts +3 -3
- package/src/snapshot-chunks/db-metadata.ts +6 -41
- package/src/subscriptions/cache.ts +58 -0
- package/src/subscriptions/resolve.test.ts +71 -1
- package/src/subscriptions/resolve.ts +65 -38
package/dist/pull.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { captureSyncException, countSyncMetric, distributionSyncMetric, encodeSnapshotRowFrames, encodeSnapshotRows, gzipBytes, randomId, SYNC_SNAPSHOT_CHUNK_COMPRESSION, SYNC_SNAPSHOT_CHUNK_ENCODING, sha256Hex, startSyncSpan, } from '@syncular/core';
|
|
1
|
+
import { bytesToReadableStream, captureSyncException, concatByteChunks, countSyncMetric, distributionSyncMetric, encodeSnapshotRowFrames, encodeSnapshotRows, gzipBytes, randomId, SYNC_SNAPSHOT_CHUNK_COMPRESSION, SYNC_SNAPSHOT_CHUNK_ENCODING, sha256Hex, startSyncSpan, } from '@syncular/core';
|
|
2
2
|
import { getServerBootstrapOrderFor, } from './handlers/collection.js';
|
|
3
3
|
import { EXTERNAL_CLIENT_ID } from './notify.js';
|
|
4
4
|
import { insertSnapshotChunk, readSnapshotChunkRefByPageKey, scopesToSnapshotChunkScopeKey, } from './snapshot-chunks.js';
|
|
@@ -9,6 +9,8 @@ const DEFAULT_MAX_SNAPSHOT_BUNDLE_ROW_FRAME_BYTES = 512 * 1024;
|
|
|
9
9
|
const MAX_ADAPTIVE_SNAPSHOT_BUNDLE_ROW_FRAME_BYTES = 4 * 1024 * 1024;
|
|
10
10
|
const DEFAULT_INLINE_SNAPSHOT_ROW_FRAME_BYTES = 256 * 1024;
|
|
11
11
|
const EMPTY_SNAPSHOT_ROW_FRAMES = encodeSnapshotRows([]);
|
|
12
|
+
const MAX_PULL_TRANSACTION_RETRIES = 2;
|
|
13
|
+
const PULL_TRANSACTION_RETRY_DELAY_MS = 15;
|
|
12
14
|
function createPullBootstrapTimings() {
|
|
13
15
|
return {
|
|
14
16
|
snapshotQueryMs: 0,
|
|
@@ -19,103 +21,6 @@ function createPullBootstrapTimings() {
|
|
|
19
21
|
chunkPersistMs: 0,
|
|
20
22
|
};
|
|
21
23
|
}
|
|
22
|
-
function concatByteChunks(chunks) {
|
|
23
|
-
if (chunks.length === 1) {
|
|
24
|
-
return chunks[0] ?? new Uint8Array();
|
|
25
|
-
}
|
|
26
|
-
let total = 0;
|
|
27
|
-
for (const chunk of chunks) {
|
|
28
|
-
total += chunk.length;
|
|
29
|
-
}
|
|
30
|
-
const merged = new Uint8Array(total);
|
|
31
|
-
let offset = 0;
|
|
32
|
-
for (const chunk of chunks) {
|
|
33
|
-
merged.set(chunk, offset);
|
|
34
|
-
offset += chunk.length;
|
|
35
|
-
}
|
|
36
|
-
return merged;
|
|
37
|
-
}
|
|
38
|
-
function byteChunksToStream(chunks) {
|
|
39
|
-
return new ReadableStream({
|
|
40
|
-
start(controller) {
|
|
41
|
-
for (const chunk of chunks) {
|
|
42
|
-
if (chunk.length === 0)
|
|
43
|
-
continue;
|
|
44
|
-
controller.enqueue(chunk.slice());
|
|
45
|
-
}
|
|
46
|
-
controller.close();
|
|
47
|
-
},
|
|
48
|
-
});
|
|
49
|
-
}
|
|
50
|
-
function bufferSourceToUint8Array(chunk) {
|
|
51
|
-
if (chunk instanceof Uint8Array) {
|
|
52
|
-
return chunk;
|
|
53
|
-
}
|
|
54
|
-
if (chunk instanceof ArrayBuffer) {
|
|
55
|
-
return new Uint8Array(chunk);
|
|
56
|
-
}
|
|
57
|
-
return new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);
|
|
58
|
-
}
|
|
59
|
-
async function streamToBytes(stream) {
|
|
60
|
-
const reader = stream.getReader();
|
|
61
|
-
const chunks = [];
|
|
62
|
-
let total = 0;
|
|
63
|
-
try {
|
|
64
|
-
while (true) {
|
|
65
|
-
const { done, value } = await reader.read();
|
|
66
|
-
if (done)
|
|
67
|
-
break;
|
|
68
|
-
if (!value)
|
|
69
|
-
continue;
|
|
70
|
-
const bytes = bufferSourceToUint8Array(value);
|
|
71
|
-
if (bytes.length === 0)
|
|
72
|
-
continue;
|
|
73
|
-
chunks.push(bytes);
|
|
74
|
-
total += bytes.length;
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
finally {
|
|
78
|
-
reader.releaseLock();
|
|
79
|
-
}
|
|
80
|
-
if (chunks.length === 0)
|
|
81
|
-
return new Uint8Array();
|
|
82
|
-
if (chunks.length === 1)
|
|
83
|
-
return chunks[0] ?? new Uint8Array();
|
|
84
|
-
const merged = new Uint8Array(total);
|
|
85
|
-
let offset = 0;
|
|
86
|
-
for (const chunk of chunks) {
|
|
87
|
-
merged.set(chunk, offset);
|
|
88
|
-
offset += chunk.length;
|
|
89
|
-
}
|
|
90
|
-
return merged;
|
|
91
|
-
}
|
|
92
|
-
function bufferSourceStreamToUint8ArrayStream(stream) {
|
|
93
|
-
return new ReadableStream({
|
|
94
|
-
async start(controller) {
|
|
95
|
-
const reader = stream.getReader();
|
|
96
|
-
try {
|
|
97
|
-
while (true) {
|
|
98
|
-
const { done, value } = await reader.read();
|
|
99
|
-
if (done)
|
|
100
|
-
break;
|
|
101
|
-
if (!value)
|
|
102
|
-
continue;
|
|
103
|
-
const bytes = bufferSourceToUint8Array(value);
|
|
104
|
-
if (bytes.length === 0)
|
|
105
|
-
continue;
|
|
106
|
-
controller.enqueue(bytes);
|
|
107
|
-
}
|
|
108
|
-
controller.close();
|
|
109
|
-
}
|
|
110
|
-
catch (err) {
|
|
111
|
-
controller.error(err);
|
|
112
|
-
}
|
|
113
|
-
finally {
|
|
114
|
-
reader.releaseLock();
|
|
115
|
-
}
|
|
116
|
-
},
|
|
117
|
-
});
|
|
118
|
-
}
|
|
119
24
|
let nodeCryptoModulePromise = null;
|
|
120
25
|
async function getNodeCryptoModule() {
|
|
121
26
|
if (!nodeCryptoModulePromise) {
|
|
@@ -137,10 +42,6 @@ async function sha256HexFromByteChunks(chunks) {
|
|
|
137
42
|
return sha256Hex(concatByteChunks(chunks));
|
|
138
43
|
}
|
|
139
44
|
async function gzipByteChunks(chunks) {
|
|
140
|
-
if (typeof CompressionStream !== 'undefined') {
|
|
141
|
-
const stream = byteChunksToStream(chunks).pipeThrough(new CompressionStream('gzip'));
|
|
142
|
-
return streamToBytes(stream);
|
|
143
|
-
}
|
|
144
45
|
return gzipBytes(concatByteChunks(chunks));
|
|
145
46
|
}
|
|
146
47
|
async function encodeCompressedSnapshotChunk(chunks) {
|
|
@@ -163,7 +64,7 @@ async function encodeCompressedSnapshotChunk(chunks) {
|
|
|
163
64
|
async function encodeCompressedSnapshotChunkToStream(chunks) {
|
|
164
65
|
const encoded = await encodeCompressedSnapshotChunk(chunks);
|
|
165
66
|
return {
|
|
166
|
-
stream:
|
|
67
|
+
stream: bytesToReadableStream(encoded.body),
|
|
167
68
|
byteLength: encoded.body.length,
|
|
168
69
|
sha256: encoded.sha256,
|
|
169
70
|
gzipMs: encoded.gzipMs,
|
|
@@ -209,6 +110,16 @@ function sanitizeLimit(value, defaultValue, min, max) {
|
|
|
209
110
|
return defaultValue;
|
|
210
111
|
return Math.max(min, Math.min(max, value));
|
|
211
112
|
}
|
|
113
|
+
function isSerializablePullError(error) {
|
|
114
|
+
const withCode = error;
|
|
115
|
+
return (withCode.code === '40001' ||
|
|
116
|
+
error.message.toLowerCase().includes('could not serialize access'));
|
|
117
|
+
}
|
|
118
|
+
async function delay(ms) {
|
|
119
|
+
await new Promise((resolve) => {
|
|
120
|
+
setTimeout(resolve, ms);
|
|
121
|
+
});
|
|
122
|
+
}
|
|
212
123
|
/**
|
|
213
124
|
* Merge all scope values into a flat ScopeValues for cursor tracking.
|
|
214
125
|
*/
|
|
@@ -341,8 +252,6 @@ export async function pull(args) {
|
|
|
341
252
|
const limitSnapshotRows = sanitizeLimit(request.limitSnapshotRows, 1000, 1, 20000);
|
|
342
253
|
const maxSnapshotPages = sanitizeLimit(request.maxSnapshotPages, 4, 1, 50);
|
|
343
254
|
const dedupeRows = request.dedupeRows === true;
|
|
344
|
-
const pendingExternalChunkWrites = [];
|
|
345
|
-
const bootstrapTimings = createPullBootstrapTimings();
|
|
346
255
|
// Resolve effective scopes for each subscription
|
|
347
256
|
const resolved = await resolveEffectiveScopesForSubscriptions({
|
|
348
257
|
db,
|
|
@@ -351,509 +260,491 @@ export async function pull(args) {
|
|
|
351
260
|
handlers: args.handlers,
|
|
352
261
|
scopeCache: args.scopeCache ?? defaultScopeCache,
|
|
353
262
|
});
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
const
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
const subResponses = [];
|
|
363
|
-
const activeSubscriptions = [];
|
|
364
|
-
const nextCursors = [];
|
|
365
|
-
// Detect external data changes (synthetic commits from notifyExternalDataChange)
|
|
366
|
-
// Compute minimum cursor across all active subscriptions to scope the query.
|
|
367
|
-
let minSubCursor = Number.MAX_SAFE_INTEGER;
|
|
368
|
-
const activeTables = new Set();
|
|
369
|
-
for (const sub of resolved) {
|
|
370
|
-
if (sub.status === 'revoked' ||
|
|
371
|
-
Object.keys(sub.scopes).length === 0)
|
|
372
|
-
continue;
|
|
373
|
-
activeTables.add(sub.table);
|
|
374
|
-
const cursor = Math.max(-1, sub.cursor ?? -1);
|
|
375
|
-
if (cursor >= 0 && cursor < minSubCursor) {
|
|
376
|
-
minSubCursor = cursor;
|
|
377
|
-
}
|
|
378
|
-
}
|
|
379
|
-
const maxExternalCommitByTable = minSubCursor < Number.MAX_SAFE_INTEGER && minSubCursor >= 0
|
|
380
|
-
? await readLatestExternalCommitByTable(trx, {
|
|
381
|
-
partitionId,
|
|
382
|
-
afterCursor: minSubCursor,
|
|
383
|
-
tables: Array.from(activeTables),
|
|
384
|
-
})
|
|
385
|
-
: new Map();
|
|
386
|
-
for (const sub of resolved) {
|
|
387
|
-
const cursor = Math.max(-1, sub.cursor ?? -1);
|
|
388
|
-
// Validate table handler exists (throws if not registered)
|
|
389
|
-
if (!args.handlers.byTable.has(sub.table)) {
|
|
390
|
-
throw new Error(`Unknown table: ${sub.table}`);
|
|
391
|
-
}
|
|
392
|
-
if (sub.status === 'revoked' ||
|
|
393
|
-
Object.keys(sub.scopes).length === 0) {
|
|
394
|
-
subResponses.push({
|
|
395
|
-
id: sub.id,
|
|
396
|
-
status: 'revoked',
|
|
397
|
-
scopes: {},
|
|
398
|
-
bootstrap: false,
|
|
399
|
-
nextCursor: cursor,
|
|
400
|
-
commits: [],
|
|
263
|
+
for (let attemptIndex = 0; attemptIndex < MAX_PULL_TRANSACTION_RETRIES; attemptIndex += 1) {
|
|
264
|
+
const pendingExternalChunkWrites = [];
|
|
265
|
+
const bootstrapTimings = createPullBootstrapTimings();
|
|
266
|
+
try {
|
|
267
|
+
const result = await dialect.executeInTransaction(db, async (trx) => {
|
|
268
|
+
await dialect.setRepeatableRead(trx);
|
|
269
|
+
const maxCommitSeq = await dialect.readMaxCommitSeq(trx, {
|
|
270
|
+
partitionId,
|
|
401
271
|
});
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
cursor
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
sub.
|
|
417
|
-
(
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
tables,
|
|
422
|
-
tableIndex: 0,
|
|
423
|
-
rowCursor: null,
|
|
424
|
-
};
|
|
425
|
-
const requestedState = sub.bootstrapState ?? null;
|
|
426
|
-
const state = requestedState &&
|
|
427
|
-
typeof requestedState.asOfCommitSeq === 'number' &&
|
|
428
|
-
Array.isArray(requestedState.tables) &&
|
|
429
|
-
typeof requestedState.tableIndex === 'number'
|
|
430
|
-
? requestedState
|
|
431
|
-
: initState;
|
|
432
|
-
// If the bootstrap state's asOfCommitSeq is no longer catch-up-able, restart bootstrap.
|
|
433
|
-
const effectiveState = state.asOfCommitSeq < minCommitSeq - 1 ? initState : state;
|
|
434
|
-
const tableName = effectiveState.tables[effectiveState.tableIndex];
|
|
435
|
-
// No tables (or ran past the end): treat bootstrap as complete.
|
|
436
|
-
if (!tableName) {
|
|
437
|
-
subResponses.push({
|
|
438
|
-
id: sub.id,
|
|
439
|
-
status: 'active',
|
|
440
|
-
scopes: effectiveScopes,
|
|
441
|
-
bootstrap: true,
|
|
442
|
-
bootstrapState: null,
|
|
443
|
-
nextCursor: effectiveState.asOfCommitSeq,
|
|
444
|
-
commits: [],
|
|
445
|
-
snapshots: [],
|
|
446
|
-
});
|
|
447
|
-
nextCursors.push(effectiveState.asOfCommitSeq);
|
|
448
|
-
continue;
|
|
272
|
+
const minCommitSeq = await dialect.readMinCommitSeq(trx, {
|
|
273
|
+
partitionId,
|
|
274
|
+
});
|
|
275
|
+
const subResponses = [];
|
|
276
|
+
const activeSubscriptions = [];
|
|
277
|
+
const nextCursors = [];
|
|
278
|
+
// Detect external data changes (synthetic commits from notifyExternalDataChange)
|
|
279
|
+
// Compute minimum cursor across all active subscriptions to scope the query.
|
|
280
|
+
let minSubCursor = Number.MAX_SAFE_INTEGER;
|
|
281
|
+
const activeTables = new Set();
|
|
282
|
+
for (const sub of resolved) {
|
|
283
|
+
if (sub.status === 'revoked' ||
|
|
284
|
+
Object.keys(sub.scopes).length === 0)
|
|
285
|
+
continue;
|
|
286
|
+
activeTables.add(sub.table);
|
|
287
|
+
const cursor = Math.max(-1, sub.cursor ?? -1);
|
|
288
|
+
if (cursor >= 0 && cursor < minSubCursor) {
|
|
289
|
+
minSubCursor = cursor;
|
|
290
|
+
}
|
|
449
291
|
}
|
|
450
|
-
const
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
292
|
+
const maxExternalCommitByTable = minSubCursor < Number.MAX_SAFE_INTEGER && minSubCursor >= 0
|
|
293
|
+
? await readLatestExternalCommitByTable(trx, {
|
|
294
|
+
partitionId,
|
|
295
|
+
afterCursor: minSubCursor,
|
|
296
|
+
tables: Array.from(activeTables),
|
|
297
|
+
})
|
|
298
|
+
: new Map();
|
|
299
|
+
for (const sub of resolved) {
|
|
300
|
+
const cursor = Math.max(-1, sub.cursor ?? -1);
|
|
301
|
+
// Validate table handler exists (throws if not registered)
|
|
302
|
+
if (!args.handlers.byTable.has(sub.table)) {
|
|
303
|
+
throw new Error(`Unknown table: ${sub.table}`);
|
|
304
|
+
}
|
|
305
|
+
if (sub.status === 'revoked' ||
|
|
306
|
+
Object.keys(sub.scopes).length === 0) {
|
|
307
|
+
subResponses.push({
|
|
308
|
+
id: sub.id,
|
|
309
|
+
status: 'revoked',
|
|
310
|
+
scopes: {},
|
|
311
|
+
bootstrap: false,
|
|
312
|
+
nextCursor: cursor,
|
|
313
|
+
commits: [],
|
|
460
314
|
});
|
|
461
|
-
|
|
315
|
+
continue;
|
|
462
316
|
}
|
|
463
|
-
const
|
|
464
|
-
|
|
465
|
-
const
|
|
466
|
-
const
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
317
|
+
const effectiveScopes = sub.scopes;
|
|
318
|
+
activeSubscriptions.push({ scopes: effectiveScopes });
|
|
319
|
+
const latestExternalCommitForTable = maxExternalCommitByTable.get(sub.table);
|
|
320
|
+
const needsBootstrap = sub.bootstrapState != null ||
|
|
321
|
+
cursor < 0 ||
|
|
322
|
+
cursor > maxCommitSeq ||
|
|
323
|
+
(minCommitSeq > 0 && cursor < minCommitSeq - 1) ||
|
|
324
|
+
(latestExternalCommitForTable !== undefined &&
|
|
325
|
+
latestExternalCommitForTable > cursor);
|
|
326
|
+
if (needsBootstrap) {
|
|
327
|
+
const tables = getServerBootstrapOrderFor(args.handlers, sub.table).map((handler) => handler.table);
|
|
328
|
+
const preferInlineBootstrapSnapshot = cursor >= 0 ||
|
|
329
|
+
sub.bootstrapState != null ||
|
|
330
|
+
(latestExternalCommitForTable !== undefined &&
|
|
331
|
+
latestExternalCommitForTable > cursor);
|
|
332
|
+
const initState = {
|
|
333
|
+
asOfCommitSeq: maxCommitSeq,
|
|
334
|
+
tables,
|
|
335
|
+
tableIndex: 0,
|
|
336
|
+
rowCursor: null,
|
|
337
|
+
};
|
|
338
|
+
const requestedState = sub.bootstrapState ?? null;
|
|
339
|
+
const state = requestedState &&
|
|
340
|
+
typeof requestedState.asOfCommitSeq === 'number' &&
|
|
341
|
+
Array.isArray(requestedState.tables) &&
|
|
342
|
+
typeof requestedState.tableIndex === 'number'
|
|
343
|
+
? requestedState
|
|
344
|
+
: initState;
|
|
345
|
+
// If the bootstrap state's asOfCommitSeq is no longer catch-up-able, restart bootstrap.
|
|
346
|
+
const effectiveState = state.asOfCommitSeq < minCommitSeq - 1
|
|
347
|
+
? initState
|
|
348
|
+
: state;
|
|
349
|
+
const tableName = effectiveState.tables[effectiveState.tableIndex];
|
|
350
|
+
// No tables (or ran past the end): treat bootstrap as complete.
|
|
351
|
+
if (!tableName) {
|
|
352
|
+
subResponses.push({
|
|
353
|
+
id: sub.id,
|
|
354
|
+
status: 'active',
|
|
355
|
+
scopes: effectiveScopes,
|
|
356
|
+
bootstrap: true,
|
|
357
|
+
bootstrapState: null,
|
|
358
|
+
nextCursor: effectiveState.asOfCommitSeq,
|
|
359
|
+
commits: [],
|
|
360
|
+
snapshots: [],
|
|
361
|
+
});
|
|
362
|
+
nextCursors.push(effectiveState.asOfCommitSeq);
|
|
363
|
+
continue;
|
|
364
|
+
}
|
|
365
|
+
const snapshots = [];
|
|
366
|
+
let nextState = effectiveState;
|
|
367
|
+
const cacheKey = `${partitionId}:${await scopesToSnapshotChunkScopeKey(effectiveScopes)}`;
|
|
368
|
+
const flushSnapshotBundle = async (bundle) => {
|
|
369
|
+
if (bundle.inlineRows) {
|
|
370
|
+
snapshots.push({
|
|
371
|
+
table: bundle.table,
|
|
372
|
+
rows: bundle.inlineRows,
|
|
373
|
+
isFirstPage: bundle.isFirstPage,
|
|
374
|
+
isLastPage: bundle.isLastPage,
|
|
375
|
+
});
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
const nowIso = new Date().toISOString();
|
|
379
|
+
const bundleRowLimit = Math.max(1, limitSnapshotRows * bundle.pageCount);
|
|
380
|
+
const cacheLookupStartedAt = Date.now();
|
|
381
|
+
const cached = await readSnapshotChunkRefByPageKey(trx, {
|
|
382
|
+
partitionId,
|
|
383
|
+
scopeKey: cacheKey,
|
|
384
|
+
scope: bundle.table,
|
|
385
|
+
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
386
|
+
rowCursor: bundle.startCursor,
|
|
387
|
+
rowLimit: bundleRowLimit,
|
|
388
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
389
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
390
|
+
nowIso,
|
|
391
|
+
});
|
|
392
|
+
bootstrapTimings.chunkCacheLookupMs += Math.max(0, Date.now() - cacheLookupStartedAt);
|
|
393
|
+
let chunkRef = cached;
|
|
394
|
+
if (!chunkRef) {
|
|
395
|
+
const expiresAt = new Date(Date.now() + Math.max(1000, bundle.ttlMs)).toISOString();
|
|
396
|
+
if (args.chunkStorage) {
|
|
397
|
+
const snapshot = {
|
|
398
|
+
table: bundle.table,
|
|
399
|
+
rows: [],
|
|
400
|
+
chunks: [],
|
|
401
|
+
isFirstPage: bundle.isFirstPage,
|
|
402
|
+
isLastPage: bundle.isLastPage,
|
|
403
|
+
};
|
|
404
|
+
snapshots.push(snapshot);
|
|
405
|
+
pendingExternalChunkWrites.push({
|
|
406
|
+
snapshot,
|
|
407
|
+
cacheLookup: {
|
|
408
|
+
partitionId,
|
|
409
|
+
scopeKey: cacheKey,
|
|
410
|
+
scope: bundle.table,
|
|
411
|
+
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
412
|
+
rowCursor: bundle.startCursor,
|
|
413
|
+
rowLimit: bundleRowLimit,
|
|
414
|
+
},
|
|
415
|
+
rowFrameParts: [...bundle.rowFrameParts],
|
|
416
|
+
expiresAt,
|
|
417
|
+
});
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
const encodedChunk = await encodeCompressedSnapshotChunk(bundle.rowFrameParts);
|
|
421
|
+
bootstrapTimings.chunkGzipMs += encodedChunk.gzipMs;
|
|
422
|
+
bootstrapTimings.chunkHashMs += encodedChunk.hashMs;
|
|
423
|
+
const chunkId = randomId();
|
|
424
|
+
const chunkPersistStartedAt = Date.now();
|
|
425
|
+
chunkRef = await insertSnapshotChunk(trx, {
|
|
426
|
+
chunkId,
|
|
493
427
|
partitionId,
|
|
494
428
|
scopeKey: cacheKey,
|
|
495
429
|
scope: bundle.table,
|
|
496
430
|
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
497
431
|
rowCursor: bundle.startCursor,
|
|
498
432
|
rowLimit: bundleRowLimit,
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
433
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
434
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
435
|
+
sha256: encodedChunk.sha256,
|
|
436
|
+
body: encodedChunk.body,
|
|
437
|
+
expiresAt,
|
|
438
|
+
});
|
|
439
|
+
bootstrapTimings.chunkPersistMs += Math.max(0, Date.now() - chunkPersistStartedAt);
|
|
440
|
+
}
|
|
441
|
+
snapshots.push({
|
|
442
|
+
table: bundle.table,
|
|
443
|
+
rows: [],
|
|
444
|
+
chunks: [chunkRef],
|
|
445
|
+
isFirstPage: bundle.isFirstPage,
|
|
446
|
+
isLastPage: bundle.isLastPage,
|
|
502
447
|
});
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
448
|
+
};
|
|
449
|
+
let activeBundle = null;
|
|
450
|
+
for (let pageIndex = 0; pageIndex < maxSnapshotPages; pageIndex++) {
|
|
451
|
+
if (!nextState)
|
|
452
|
+
break;
|
|
453
|
+
const nextTableName = nextState.tables[nextState.tableIndex];
|
|
454
|
+
if (!nextTableName) {
|
|
455
|
+
if (activeBundle) {
|
|
456
|
+
activeBundle.isLastPage = true;
|
|
457
|
+
await flushSnapshotBundle(activeBundle);
|
|
458
|
+
activeBundle = null;
|
|
459
|
+
}
|
|
460
|
+
nextState = null;
|
|
461
|
+
break;
|
|
462
|
+
}
|
|
463
|
+
const tableHandler = args.handlers.byTable.get(nextTableName);
|
|
464
|
+
if (!tableHandler) {
|
|
465
|
+
throw new Error(`Unknown table: ${nextTableName}`);
|
|
466
|
+
}
|
|
467
|
+
if (!activeBundle ||
|
|
468
|
+
activeBundle.table !== nextTableName) {
|
|
469
|
+
if (activeBundle) {
|
|
470
|
+
await flushSnapshotBundle(activeBundle);
|
|
471
|
+
}
|
|
472
|
+
const bundleHeader = EMPTY_SNAPSHOT_ROW_FRAMES;
|
|
473
|
+
activeBundle = {
|
|
474
|
+
table: nextTableName,
|
|
475
|
+
startCursor: nextState.rowCursor,
|
|
476
|
+
isFirstPage: nextState.rowCursor == null,
|
|
477
|
+
isLastPage: false,
|
|
478
|
+
pageCount: 0,
|
|
479
|
+
ttlMs: tableHandler.snapshotChunkTtlMs ??
|
|
480
|
+
24 * 60 * 60 * 1000,
|
|
481
|
+
rowFrameByteLength: bundleHeader.length,
|
|
482
|
+
rowFrameParts: [bundleHeader],
|
|
483
|
+
inlineRows: null,
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
const snapshotQueryStartedAt = Date.now();
|
|
487
|
+
const page = await tableHandler.snapshot({
|
|
488
|
+
db: trx,
|
|
489
|
+
actorId: args.auth.actorId,
|
|
490
|
+
auth: args.auth,
|
|
491
|
+
scopeValues: effectiveScopes,
|
|
492
|
+
cursor: nextState.rowCursor,
|
|
493
|
+
limit: limitSnapshotRows,
|
|
494
|
+
}, sub.params);
|
|
495
|
+
bootstrapTimings.snapshotQueryMs += Math.max(0, Date.now() - snapshotQueryStartedAt);
|
|
496
|
+
const rowFrameEncodeStartedAt = Date.now();
|
|
497
|
+
const rowFrames = encodeSnapshotRowFrames(page.rows ?? []);
|
|
498
|
+
bootstrapTimings.rowFrameEncodeMs += Math.max(0, Date.now() - rowFrameEncodeStartedAt);
|
|
499
|
+
const bundleMaxBytes = resolveSnapshotBundleMaxBytes({
|
|
500
|
+
configuredMaxBytes: tableHandler.snapshotBundleMaxBytes,
|
|
501
|
+
pageRowCount: page.rows?.length ?? 0,
|
|
502
|
+
pageRowFrameBytes: rowFrames.length,
|
|
503
|
+
});
|
|
504
|
+
if (activeBundle.pageCount > 0 &&
|
|
505
|
+
activeBundle.rowFrameByteLength + rowFrames.length >
|
|
506
|
+
bundleMaxBytes) {
|
|
507
|
+
await flushSnapshotBundle(activeBundle);
|
|
508
|
+
const bundleHeader = EMPTY_SNAPSHOT_ROW_FRAMES;
|
|
509
|
+
activeBundle = {
|
|
510
|
+
table: nextTableName,
|
|
511
|
+
startCursor: nextState.rowCursor,
|
|
512
|
+
isFirstPage: nextState.rowCursor == null,
|
|
513
|
+
isLastPage: false,
|
|
514
|
+
pageCount: 0,
|
|
515
|
+
ttlMs: tableHandler.snapshotChunkTtlMs ??
|
|
516
|
+
24 * 60 * 60 * 1000,
|
|
517
|
+
rowFrameByteLength: bundleHeader.length,
|
|
518
|
+
rowFrameParts: [bundleHeader],
|
|
519
|
+
inlineRows: null,
|
|
520
|
+
};
|
|
521
|
+
}
|
|
522
|
+
if (preferInlineBootstrapSnapshot &&
|
|
523
|
+
activeBundle.pageCount === 0 &&
|
|
524
|
+
page.nextCursor == null &&
|
|
525
|
+
rowFrames.length <=
|
|
526
|
+
DEFAULT_INLINE_SNAPSHOT_ROW_FRAME_BYTES) {
|
|
527
|
+
activeBundle.inlineRows = page.rows ?? [];
|
|
528
|
+
}
|
|
529
|
+
else {
|
|
530
|
+
activeBundle.inlineRows = null;
|
|
531
|
+
}
|
|
532
|
+
activeBundle.rowFrameParts.push(rowFrames);
|
|
533
|
+
activeBundle.rowFrameByteLength += rowFrames.length;
|
|
534
|
+
activeBundle.pageCount += 1;
|
|
535
|
+
if (page.nextCursor != null) {
|
|
536
|
+
nextState = {
|
|
537
|
+
...nextState,
|
|
538
|
+
rowCursor: page.nextCursor,
|
|
539
|
+
};
|
|
540
|
+
continue;
|
|
541
|
+
}
|
|
541
542
|
activeBundle.isLastPage = true;
|
|
542
543
|
await flushSnapshotBundle(activeBundle);
|
|
543
544
|
activeBundle = null;
|
|
545
|
+
if (nextState.tableIndex + 1 < nextState.tables.length) {
|
|
546
|
+
nextState = {
|
|
547
|
+
...nextState,
|
|
548
|
+
tableIndex: nextState.tableIndex + 1,
|
|
549
|
+
rowCursor: null,
|
|
550
|
+
};
|
|
551
|
+
continue;
|
|
552
|
+
}
|
|
553
|
+
nextState = null;
|
|
554
|
+
break;
|
|
544
555
|
}
|
|
545
|
-
nextState = null;
|
|
546
|
-
break;
|
|
547
|
-
}
|
|
548
|
-
const tableHandler = args.handlers.byTable.get(nextTableName);
|
|
549
|
-
if (!tableHandler) {
|
|
550
|
-
throw new Error(`Unknown table: ${nextTableName}`);
|
|
551
|
-
}
|
|
552
|
-
if (!activeBundle || activeBundle.table !== nextTableName) {
|
|
553
556
|
if (activeBundle) {
|
|
554
557
|
await flushSnapshotBundle(activeBundle);
|
|
555
558
|
}
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
}
|
|
569
|
-
const snapshotQueryStartedAt = Date.now();
|
|
570
|
-
const page = await tableHandler.snapshot({
|
|
571
|
-
db: trx,
|
|
572
|
-
actorId: args.auth.actorId,
|
|
573
|
-
auth: args.auth,
|
|
574
|
-
scopeValues: effectiveScopes,
|
|
575
|
-
cursor: nextState.rowCursor,
|
|
576
|
-
limit: limitSnapshotRows,
|
|
577
|
-
}, sub.params);
|
|
578
|
-
bootstrapTimings.snapshotQueryMs += Math.max(0, Date.now() - snapshotQueryStartedAt);
|
|
579
|
-
const rowFrameEncodeStartedAt = Date.now();
|
|
580
|
-
const rowFrames = encodeSnapshotRowFrames(page.rows ?? []);
|
|
581
|
-
bootstrapTimings.rowFrameEncodeMs += Math.max(0, Date.now() - rowFrameEncodeStartedAt);
|
|
582
|
-
const bundleMaxBytes = resolveSnapshotBundleMaxBytes({
|
|
583
|
-
configuredMaxBytes: tableHandler.snapshotBundleMaxBytes,
|
|
584
|
-
pageRowCount: page.rows?.length ?? 0,
|
|
585
|
-
pageRowFrameBytes: rowFrames.length,
|
|
586
|
-
});
|
|
587
|
-
if (activeBundle.pageCount > 0 &&
|
|
588
|
-
activeBundle.rowFrameByteLength + rowFrames.length >
|
|
589
|
-
bundleMaxBytes) {
|
|
590
|
-
await flushSnapshotBundle(activeBundle);
|
|
591
|
-
const bundleHeader = EMPTY_SNAPSHOT_ROW_FRAMES;
|
|
592
|
-
activeBundle = {
|
|
593
|
-
table: nextTableName,
|
|
594
|
-
startCursor: nextState.rowCursor,
|
|
595
|
-
isFirstPage: nextState.rowCursor == null,
|
|
596
|
-
isLastPage: false,
|
|
597
|
-
pageCount: 0,
|
|
598
|
-
ttlMs: tableHandler.snapshotChunkTtlMs ?? 24 * 60 * 60 * 1000,
|
|
599
|
-
rowFrameByteLength: bundleHeader.length,
|
|
600
|
-
rowFrameParts: [bundleHeader],
|
|
601
|
-
inlineRows: null,
|
|
602
|
-
};
|
|
559
|
+
subResponses.push({
|
|
560
|
+
id: sub.id,
|
|
561
|
+
status: 'active',
|
|
562
|
+
scopes: effectiveScopes,
|
|
563
|
+
bootstrap: true,
|
|
564
|
+
bootstrapState: nextState,
|
|
565
|
+
nextCursor: effectiveState.asOfCommitSeq,
|
|
566
|
+
commits: [],
|
|
567
|
+
snapshots,
|
|
568
|
+
});
|
|
569
|
+
nextCursors.push(effectiveState.asOfCommitSeq);
|
|
570
|
+
continue;
|
|
603
571
|
}
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
572
|
+
// Incremental pull for this subscription. The dialect row query
|
|
573
|
+
// carries the scanned commit-window max when matching rows exist,
|
|
574
|
+
// so we only need a separate commit-window scan when the row query
|
|
575
|
+
// returns no matches at all.
|
|
576
|
+
const incrementalRows = [];
|
|
577
|
+
let maxScannedCommitSeq = cursor;
|
|
578
|
+
for await (const row of dialect.iterateIncrementalPullRows(trx, {
|
|
579
|
+
partitionId,
|
|
580
|
+
table: sub.table,
|
|
581
|
+
scopes: effectiveScopes,
|
|
582
|
+
cursor,
|
|
583
|
+
limitCommits,
|
|
584
|
+
})) {
|
|
585
|
+
incrementalRows.push(row);
|
|
586
|
+
maxScannedCommitSeq = Math.max(maxScannedCommitSeq, row.scanned_max_commit_seq ?? row.commit_seq);
|
|
609
587
|
}
|
|
610
|
-
|
|
611
|
-
|
|
588
|
+
if (incrementalRows.length === 0) {
|
|
589
|
+
const scannedCommitSeqs = await dialect.readCommitSeqsForPull(trx, {
|
|
590
|
+
partitionId,
|
|
591
|
+
cursor,
|
|
592
|
+
limitCommits,
|
|
593
|
+
tables: [sub.table],
|
|
594
|
+
});
|
|
595
|
+
maxScannedCommitSeq =
|
|
596
|
+
scannedCommitSeqs.length > 0
|
|
597
|
+
? scannedCommitSeqs[scannedCommitSeqs.length - 1]
|
|
598
|
+
: cursor;
|
|
599
|
+
if (scannedCommitSeqs.length === 0) {
|
|
600
|
+
subResponses.push({
|
|
601
|
+
id: sub.id,
|
|
602
|
+
status: 'active',
|
|
603
|
+
scopes: effectiveScopes,
|
|
604
|
+
bootstrap: false,
|
|
605
|
+
nextCursor: cursor,
|
|
606
|
+
commits: [],
|
|
607
|
+
});
|
|
608
|
+
nextCursors.push(cursor);
|
|
609
|
+
continue;
|
|
610
|
+
}
|
|
612
611
|
}
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
612
|
+
let nextCursor = cursor;
|
|
613
|
+
if (dedupeRows) {
|
|
614
|
+
const latestByRowKey = new Map();
|
|
615
|
+
for (const r of incrementalRows) {
|
|
616
|
+
nextCursor = Math.max(nextCursor, r.commit_seq);
|
|
617
|
+
const rowKey = `${r.table}\u0000${r.row_id}`;
|
|
618
|
+
const change = {
|
|
619
|
+
table: r.table,
|
|
620
|
+
row_id: r.row_id,
|
|
621
|
+
op: r.op,
|
|
622
|
+
row_json: r.row_json,
|
|
623
|
+
row_version: r.row_version,
|
|
624
|
+
scopes: r.scopes,
|
|
625
|
+
};
|
|
626
|
+
// Move row keys to insertion tail so Map iteration yields
|
|
627
|
+
// "latest change wins" order without a full array sort.
|
|
628
|
+
if (latestByRowKey.has(rowKey)) {
|
|
629
|
+
latestByRowKey.delete(rowKey);
|
|
630
|
+
}
|
|
631
|
+
latestByRowKey.set(rowKey, {
|
|
632
|
+
commitSeq: r.commit_seq,
|
|
633
|
+
createdAt: r.created_at,
|
|
634
|
+
actorId: r.actor_id,
|
|
635
|
+
change,
|
|
636
|
+
});
|
|
637
|
+
}
|
|
638
|
+
nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
|
|
639
|
+
if (latestByRowKey.size === 0) {
|
|
640
|
+
subResponses.push({
|
|
641
|
+
id: sub.id,
|
|
642
|
+
status: 'active',
|
|
643
|
+
scopes: effectiveScopes,
|
|
644
|
+
bootstrap: false,
|
|
645
|
+
nextCursor,
|
|
646
|
+
commits: [],
|
|
647
|
+
});
|
|
648
|
+
nextCursors.push(nextCursor);
|
|
649
|
+
continue;
|
|
650
|
+
}
|
|
651
|
+
const commits = [];
|
|
652
|
+
for (const item of latestByRowKey.values()) {
|
|
653
|
+
const lastCommit = commits[commits.length - 1];
|
|
654
|
+
if (!lastCommit ||
|
|
655
|
+
lastCommit.commitSeq !== item.commitSeq) {
|
|
656
|
+
commits.push({
|
|
657
|
+
commitSeq: item.commitSeq,
|
|
658
|
+
createdAt: item.createdAt,
|
|
659
|
+
actorId: item.actorId,
|
|
660
|
+
changes: [item.change],
|
|
661
|
+
});
|
|
662
|
+
continue;
|
|
663
|
+
}
|
|
664
|
+
lastCommit.changes.push(item.change);
|
|
665
|
+
}
|
|
666
|
+
subResponses.push({
|
|
667
|
+
id: sub.id,
|
|
668
|
+
status: 'active',
|
|
669
|
+
scopes: effectiveScopes,
|
|
670
|
+
bootstrap: false,
|
|
671
|
+
nextCursor,
|
|
672
|
+
commits,
|
|
673
|
+
});
|
|
674
|
+
nextCursors.push(nextCursor);
|
|
618
675
|
continue;
|
|
619
676
|
}
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
677
|
+
const commitsBySeq = new Map();
|
|
678
|
+
const commitSeqs = [];
|
|
679
|
+
for (const r of incrementalRows) {
|
|
680
|
+
nextCursor = Math.max(nextCursor, r.commit_seq);
|
|
681
|
+
const seq = r.commit_seq;
|
|
682
|
+
let commit = commitsBySeq.get(seq);
|
|
683
|
+
if (!commit) {
|
|
684
|
+
commit = {
|
|
685
|
+
commitSeq: seq,
|
|
686
|
+
createdAt: r.created_at,
|
|
687
|
+
actorId: r.actor_id,
|
|
688
|
+
changes: [],
|
|
689
|
+
};
|
|
690
|
+
commitsBySeq.set(seq, commit);
|
|
691
|
+
commitSeqs.push(seq);
|
|
692
|
+
}
|
|
693
|
+
const change = {
|
|
694
|
+
table: r.table,
|
|
695
|
+
row_id: r.row_id,
|
|
696
|
+
op: r.op,
|
|
697
|
+
row_json: r.row_json,
|
|
698
|
+
row_version: r.row_version,
|
|
699
|
+
scopes: r.scopes,
|
|
628
700
|
};
|
|
629
|
-
|
|
701
|
+
commit.changes.push(change);
|
|
630
702
|
}
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
nextCursor: effectiveState.asOfCommitSeq,
|
|
644
|
-
commits: [],
|
|
645
|
-
snapshots,
|
|
646
|
-
});
|
|
647
|
-
nextCursors.push(effectiveState.asOfCommitSeq);
|
|
648
|
-
continue;
|
|
649
|
-
}
|
|
650
|
-
// Incremental pull for this subscription
|
|
651
|
-
// Read the commit window for this table up-front so the subscription cursor
|
|
652
|
-
// can advance past commits that don't match the requested scopes.
|
|
653
|
-
const scannedCommitSeqs = await dialect.readCommitSeqsForPull(trx, {
|
|
654
|
-
partitionId,
|
|
655
|
-
cursor,
|
|
656
|
-
limitCommits,
|
|
657
|
-
tables: [sub.table],
|
|
658
|
-
});
|
|
659
|
-
const maxScannedCommitSeq = scannedCommitSeqs.length > 0
|
|
660
|
-
? scannedCommitSeqs[scannedCommitSeqs.length - 1]
|
|
661
|
-
: cursor;
|
|
662
|
-
if (scannedCommitSeqs.length === 0) {
|
|
663
|
-
subResponses.push({
|
|
664
|
-
id: sub.id,
|
|
665
|
-
status: 'active',
|
|
666
|
-
scopes: effectiveScopes,
|
|
667
|
-
bootstrap: false,
|
|
668
|
-
nextCursor: cursor,
|
|
669
|
-
commits: [],
|
|
670
|
-
});
|
|
671
|
-
nextCursors.push(cursor);
|
|
672
|
-
continue;
|
|
673
|
-
}
|
|
674
|
-
let nextCursor = cursor;
|
|
675
|
-
if (dedupeRows) {
|
|
676
|
-
const latestByRowKey = new Map();
|
|
677
|
-
for await (const r of dialect.iterateIncrementalPullRows(trx, {
|
|
678
|
-
partitionId,
|
|
679
|
-
table: sub.table,
|
|
680
|
-
scopes: effectiveScopes,
|
|
681
|
-
cursor,
|
|
682
|
-
limitCommits,
|
|
683
|
-
})) {
|
|
684
|
-
nextCursor = Math.max(nextCursor, r.commit_seq);
|
|
685
|
-
const rowKey = `${r.table}\u0000${r.row_id}`;
|
|
686
|
-
const change = {
|
|
687
|
-
table: r.table,
|
|
688
|
-
row_id: r.row_id,
|
|
689
|
-
op: r.op,
|
|
690
|
-
row_json: r.row_json,
|
|
691
|
-
row_version: r.row_version,
|
|
692
|
-
scopes: r.scopes,
|
|
693
|
-
};
|
|
694
|
-
// Move row keys to insertion tail so Map iteration yields
|
|
695
|
-
// "latest change wins" order without a full array sort.
|
|
696
|
-
if (latestByRowKey.has(rowKey)) {
|
|
697
|
-
latestByRowKey.delete(rowKey);
|
|
703
|
+
nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
|
|
704
|
+
if (commitSeqs.length === 0) {
|
|
705
|
+
subResponses.push({
|
|
706
|
+
id: sub.id,
|
|
707
|
+
status: 'active',
|
|
708
|
+
scopes: effectiveScopes,
|
|
709
|
+
bootstrap: false,
|
|
710
|
+
nextCursor,
|
|
711
|
+
commits: [],
|
|
712
|
+
});
|
|
713
|
+
nextCursors.push(nextCursor);
|
|
714
|
+
continue;
|
|
698
715
|
}
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
change,
|
|
704
|
-
});
|
|
705
|
-
}
|
|
706
|
-
nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
|
|
707
|
-
if (latestByRowKey.size === 0) {
|
|
716
|
+
const commits = commitSeqs
|
|
717
|
+
.map((seq) => commitsBySeq.get(seq))
|
|
718
|
+
.filter((c) => !!c)
|
|
719
|
+
.filter((c) => c.changes.length > 0);
|
|
708
720
|
subResponses.push({
|
|
709
721
|
id: sub.id,
|
|
710
722
|
status: 'active',
|
|
711
723
|
scopes: effectiveScopes,
|
|
712
724
|
bootstrap: false,
|
|
713
725
|
nextCursor,
|
|
714
|
-
commits
|
|
726
|
+
commits,
|
|
715
727
|
});
|
|
716
728
|
nextCursors.push(nextCursor);
|
|
717
|
-
continue;
|
|
718
|
-
}
|
|
719
|
-
const commits = [];
|
|
720
|
-
for (const item of latestByRowKey.values()) {
|
|
721
|
-
const lastCommit = commits[commits.length - 1];
|
|
722
|
-
if (!lastCommit || lastCommit.commitSeq !== item.commitSeq) {
|
|
723
|
-
commits.push({
|
|
724
|
-
commitSeq: item.commitSeq,
|
|
725
|
-
createdAt: item.createdAt,
|
|
726
|
-
actorId: item.actorId,
|
|
727
|
-
changes: [item.change],
|
|
728
|
-
});
|
|
729
|
-
continue;
|
|
730
|
-
}
|
|
731
|
-
lastCommit.changes.push(item.change);
|
|
732
|
-
}
|
|
733
|
-
subResponses.push({
|
|
734
|
-
id: sub.id,
|
|
735
|
-
status: 'active',
|
|
736
|
-
scopes: effectiveScopes,
|
|
737
|
-
bootstrap: false,
|
|
738
|
-
nextCursor,
|
|
739
|
-
commits,
|
|
740
|
-
});
|
|
741
|
-
nextCursors.push(nextCursor);
|
|
742
|
-
continue;
|
|
743
|
-
}
|
|
744
|
-
const commitsBySeq = new Map();
|
|
745
|
-
const commitSeqs = [];
|
|
746
|
-
for await (const r of dialect.iterateIncrementalPullRows(trx, {
|
|
747
|
-
partitionId,
|
|
748
|
-
table: sub.table,
|
|
749
|
-
scopes: effectiveScopes,
|
|
750
|
-
cursor,
|
|
751
|
-
limitCommits,
|
|
752
|
-
})) {
|
|
753
|
-
nextCursor = Math.max(nextCursor, r.commit_seq);
|
|
754
|
-
const seq = r.commit_seq;
|
|
755
|
-
let commit = commitsBySeq.get(seq);
|
|
756
|
-
if (!commit) {
|
|
757
|
-
commit = {
|
|
758
|
-
commitSeq: seq,
|
|
759
|
-
createdAt: r.created_at,
|
|
760
|
-
actorId: r.actor_id,
|
|
761
|
-
changes: [],
|
|
762
|
-
};
|
|
763
|
-
commitsBySeq.set(seq, commit);
|
|
764
|
-
commitSeqs.push(seq);
|
|
765
729
|
}
|
|
766
|
-
const
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
730
|
+
const effectiveScopes = mergeScopes(activeSubscriptions);
|
|
731
|
+
const clientCursor = nextCursors.length > 0
|
|
732
|
+
? Math.min(...nextCursors)
|
|
733
|
+
: maxCommitSeq;
|
|
734
|
+
return {
|
|
735
|
+
response: {
|
|
736
|
+
ok: true,
|
|
737
|
+
subscriptions: subResponses,
|
|
738
|
+
},
|
|
739
|
+
effectiveScopes,
|
|
740
|
+
clientCursor,
|
|
773
741
|
};
|
|
774
|
-
commit.changes.push(change);
|
|
775
|
-
}
|
|
776
|
-
nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
|
|
777
|
-
if (commitSeqs.length === 0) {
|
|
778
|
-
subResponses.push({
|
|
779
|
-
id: sub.id,
|
|
780
|
-
status: 'active',
|
|
781
|
-
scopes: effectiveScopes,
|
|
782
|
-
bootstrap: false,
|
|
783
|
-
nextCursor,
|
|
784
|
-
commits: [],
|
|
785
|
-
});
|
|
786
|
-
nextCursors.push(nextCursor);
|
|
787
|
-
continue;
|
|
788
|
-
}
|
|
789
|
-
const commits = commitSeqs
|
|
790
|
-
.map((seq) => commitsBySeq.get(seq))
|
|
791
|
-
.filter((c) => !!c)
|
|
792
|
-
.filter((c) => c.changes.length > 0);
|
|
793
|
-
subResponses.push({
|
|
794
|
-
id: sub.id,
|
|
795
|
-
status: 'active',
|
|
796
|
-
scopes: effectiveScopes,
|
|
797
|
-
bootstrap: false,
|
|
798
|
-
nextCursor,
|
|
799
|
-
commits,
|
|
800
742
|
});
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
response: {
|
|
807
|
-
ok: true,
|
|
808
|
-
subscriptions: subResponses,
|
|
809
|
-
},
|
|
810
|
-
effectiveScopes,
|
|
811
|
-
clientCursor,
|
|
812
|
-
};
|
|
813
|
-
});
|
|
814
|
-
const chunkStorage = args.chunkStorage;
|
|
815
|
-
if (chunkStorage && pendingExternalChunkWrites.length > 0) {
|
|
816
|
-
await runWithConcurrency(pendingExternalChunkWrites, 4, async (pending) => {
|
|
817
|
-
const cacheLookupStartedAt = Date.now();
|
|
818
|
-
let chunkRef = await readSnapshotChunkRefByPageKey(db, {
|
|
819
|
-
partitionId: pending.cacheLookup.partitionId,
|
|
820
|
-
scopeKey: pending.cacheLookup.scopeKey,
|
|
821
|
-
scope: pending.cacheLookup.scope,
|
|
822
|
-
asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
|
|
823
|
-
rowCursor: pending.cacheLookup.rowCursor,
|
|
824
|
-
rowLimit: pending.cacheLookup.rowLimit,
|
|
825
|
-
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
826
|
-
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
827
|
-
});
|
|
828
|
-
bootstrapTimings.chunkCacheLookupMs += Math.max(0, Date.now() - cacheLookupStartedAt);
|
|
829
|
-
if (!chunkRef) {
|
|
830
|
-
if (chunkStorage.storeChunkStream) {
|
|
831
|
-
const { stream: bodyStream, byteLength, sha256, gzipMs, hashMs, } = await encodeCompressedSnapshotChunkToStream(pending.rowFrameParts);
|
|
832
|
-
bootstrapTimings.chunkGzipMs += gzipMs;
|
|
833
|
-
bootstrapTimings.chunkHashMs += hashMs;
|
|
834
|
-
const chunkPersistStartedAt = Date.now();
|
|
835
|
-
chunkRef = await chunkStorage.storeChunkStream({
|
|
836
|
-
partitionId: pending.cacheLookup.partitionId,
|
|
837
|
-
scopeKey: pending.cacheLookup.scopeKey,
|
|
838
|
-
scope: pending.cacheLookup.scope,
|
|
839
|
-
asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
|
|
840
|
-
rowCursor: pending.cacheLookup.rowCursor,
|
|
841
|
-
rowLimit: pending.cacheLookup.rowLimit,
|
|
842
|
-
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
843
|
-
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
844
|
-
sha256,
|
|
845
|
-
byteLength,
|
|
846
|
-
bodyStream,
|
|
847
|
-
expiresAt: pending.expiresAt,
|
|
848
|
-
});
|
|
849
|
-
bootstrapTimings.chunkPersistMs += Math.max(0, Date.now() - chunkPersistStartedAt);
|
|
850
|
-
}
|
|
851
|
-
else {
|
|
852
|
-
const encodedChunk = await encodeCompressedSnapshotChunk(pending.rowFrameParts);
|
|
853
|
-
bootstrapTimings.chunkGzipMs += encodedChunk.gzipMs;
|
|
854
|
-
bootstrapTimings.chunkHashMs += encodedChunk.hashMs;
|
|
855
|
-
const chunkPersistStartedAt = Date.now();
|
|
856
|
-
chunkRef = await chunkStorage.storeChunk({
|
|
743
|
+
const chunkStorage = args.chunkStorage;
|
|
744
|
+
if (chunkStorage && pendingExternalChunkWrites.length > 0) {
|
|
745
|
+
await runWithConcurrency(pendingExternalChunkWrites, 4, async (pending) => {
|
|
746
|
+
const cacheLookupStartedAt = Date.now();
|
|
747
|
+
let chunkRef = await readSnapshotChunkRefByPageKey(db, {
|
|
857
748
|
partitionId: pending.cacheLookup.partitionId,
|
|
858
749
|
scopeKey: pending.cacheLookup.scopeKey,
|
|
859
750
|
scope: pending.cacheLookup.scope,
|
|
@@ -862,43 +753,93 @@ export async function pull(args) {
|
|
|
862
753
|
rowLimit: pending.cacheLookup.rowLimit,
|
|
863
754
|
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
864
755
|
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
865
|
-
sha256: encodedChunk.sha256,
|
|
866
|
-
body: encodedChunk.body,
|
|
867
|
-
expiresAt: pending.expiresAt,
|
|
868
756
|
});
|
|
869
|
-
bootstrapTimings.
|
|
870
|
-
|
|
757
|
+
bootstrapTimings.chunkCacheLookupMs += Math.max(0, Date.now() - cacheLookupStartedAt);
|
|
758
|
+
if (!chunkRef) {
|
|
759
|
+
if (chunkStorage.storeChunkStream) {
|
|
760
|
+
const { stream: bodyStream, byteLength, sha256, gzipMs, hashMs, } = await encodeCompressedSnapshotChunkToStream(pending.rowFrameParts);
|
|
761
|
+
bootstrapTimings.chunkGzipMs += gzipMs;
|
|
762
|
+
bootstrapTimings.chunkHashMs += hashMs;
|
|
763
|
+
const chunkPersistStartedAt = Date.now();
|
|
764
|
+
chunkRef = await chunkStorage.storeChunkStream({
|
|
765
|
+
partitionId: pending.cacheLookup.partitionId,
|
|
766
|
+
scopeKey: pending.cacheLookup.scopeKey,
|
|
767
|
+
scope: pending.cacheLookup.scope,
|
|
768
|
+
asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
|
|
769
|
+
rowCursor: pending.cacheLookup.rowCursor,
|
|
770
|
+
rowLimit: pending.cacheLookup.rowLimit,
|
|
771
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
772
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
773
|
+
sha256,
|
|
774
|
+
byteLength,
|
|
775
|
+
bodyStream,
|
|
776
|
+
expiresAt: pending.expiresAt,
|
|
777
|
+
});
|
|
778
|
+
bootstrapTimings.chunkPersistMs += Math.max(0, Date.now() - chunkPersistStartedAt);
|
|
779
|
+
}
|
|
780
|
+
else {
|
|
781
|
+
const encodedChunk = await encodeCompressedSnapshotChunk(pending.rowFrameParts);
|
|
782
|
+
bootstrapTimings.chunkGzipMs += encodedChunk.gzipMs;
|
|
783
|
+
bootstrapTimings.chunkHashMs += encodedChunk.hashMs;
|
|
784
|
+
const chunkPersistStartedAt = Date.now();
|
|
785
|
+
chunkRef = await chunkStorage.storeChunk({
|
|
786
|
+
partitionId: pending.cacheLookup.partitionId,
|
|
787
|
+
scopeKey: pending.cacheLookup.scopeKey,
|
|
788
|
+
scope: pending.cacheLookup.scope,
|
|
789
|
+
asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
|
|
790
|
+
rowCursor: pending.cacheLookup.rowCursor,
|
|
791
|
+
rowLimit: pending.cacheLookup.rowLimit,
|
|
792
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
793
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
794
|
+
sha256: encodedChunk.sha256,
|
|
795
|
+
body: encodedChunk.body,
|
|
796
|
+
expiresAt: pending.expiresAt,
|
|
797
|
+
});
|
|
798
|
+
bootstrapTimings.chunkPersistMs += Math.max(0, Date.now() - chunkPersistStartedAt);
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
pending.snapshot.chunks = [chunkRef];
|
|
802
|
+
});
|
|
871
803
|
}
|
|
872
|
-
|
|
873
|
-
|
|
804
|
+
const durationMs = Math.max(0, Date.now() - startedAtMs);
|
|
805
|
+
const stats = summarizePullResponse(result.response);
|
|
806
|
+
span.setAttribute('status', 'ok');
|
|
807
|
+
span.setAttribute('duration_ms', durationMs);
|
|
808
|
+
span.setAttribute('subscription_count', stats.subscriptionCount);
|
|
809
|
+
span.setAttribute('commit_count', stats.commitCount);
|
|
810
|
+
span.setAttribute('change_count', stats.changeCount);
|
|
811
|
+
span.setAttribute('snapshot_page_count', stats.snapshotPageCount);
|
|
812
|
+
span.setAttributes({
|
|
813
|
+
bootstrap_snapshot_query_ms: bootstrapTimings.snapshotQueryMs,
|
|
814
|
+
bootstrap_row_frame_encode_ms: bootstrapTimings.rowFrameEncodeMs,
|
|
815
|
+
bootstrap_chunk_cache_lookup_ms: bootstrapTimings.chunkCacheLookupMs,
|
|
816
|
+
bootstrap_chunk_gzip_ms: bootstrapTimings.chunkGzipMs,
|
|
817
|
+
bootstrap_chunk_hash_ms: bootstrapTimings.chunkHashMs,
|
|
818
|
+
bootstrap_chunk_persist_ms: bootstrapTimings.chunkPersistMs,
|
|
819
|
+
});
|
|
820
|
+
span.setStatus('ok');
|
|
821
|
+
recordPullMetrics({
|
|
822
|
+
status: 'ok',
|
|
823
|
+
dedupeRows,
|
|
824
|
+
durationMs,
|
|
825
|
+
stats,
|
|
826
|
+
});
|
|
827
|
+
return {
|
|
828
|
+
...result,
|
|
829
|
+
bootstrapTimings,
|
|
830
|
+
};
|
|
831
|
+
}
|
|
832
|
+
catch (error) {
|
|
833
|
+
if (error instanceof Error &&
|
|
834
|
+
attemptIndex < MAX_PULL_TRANSACTION_RETRIES - 1 &&
|
|
835
|
+
isSerializablePullError(error)) {
|
|
836
|
+
await delay(PULL_TRANSACTION_RETRY_DELAY_MS * (attemptIndex + 1));
|
|
837
|
+
continue;
|
|
838
|
+
}
|
|
839
|
+
throw error;
|
|
840
|
+
}
|
|
874
841
|
}
|
|
875
|
-
|
|
876
|
-
const stats = summarizePullResponse(result.response);
|
|
877
|
-
span.setAttribute('status', 'ok');
|
|
878
|
-
span.setAttribute('duration_ms', durationMs);
|
|
879
|
-
span.setAttribute('subscription_count', stats.subscriptionCount);
|
|
880
|
-
span.setAttribute('commit_count', stats.commitCount);
|
|
881
|
-
span.setAttribute('change_count', stats.changeCount);
|
|
882
|
-
span.setAttribute('snapshot_page_count', stats.snapshotPageCount);
|
|
883
|
-
span.setAttributes({
|
|
884
|
-
bootstrap_snapshot_query_ms: bootstrapTimings.snapshotQueryMs,
|
|
885
|
-
bootstrap_row_frame_encode_ms: bootstrapTimings.rowFrameEncodeMs,
|
|
886
|
-
bootstrap_chunk_cache_lookup_ms: bootstrapTimings.chunkCacheLookupMs,
|
|
887
|
-
bootstrap_chunk_gzip_ms: bootstrapTimings.chunkGzipMs,
|
|
888
|
-
bootstrap_chunk_hash_ms: bootstrapTimings.chunkHashMs,
|
|
889
|
-
bootstrap_chunk_persist_ms: bootstrapTimings.chunkPersistMs,
|
|
890
|
-
});
|
|
891
|
-
span.setStatus('ok');
|
|
892
|
-
recordPullMetrics({
|
|
893
|
-
status: 'ok',
|
|
894
|
-
dedupeRows,
|
|
895
|
-
durationMs,
|
|
896
|
-
stats,
|
|
897
|
-
});
|
|
898
|
-
return {
|
|
899
|
-
...result,
|
|
900
|
-
bootstrapTimings,
|
|
901
|
-
};
|
|
842
|
+
throw new Error('Pull transaction retry loop exhausted unexpectedly');
|
|
902
843
|
}
|
|
903
844
|
catch (error) {
|
|
904
845
|
const durationMs = Math.max(0, Date.now() - startedAtMs);
|