@syncular/server 0.0.1-100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobs/adapters/database.d.ts +83 -0
- package/dist/blobs/adapters/database.d.ts.map +1 -0
- package/dist/blobs/adapters/database.js +202 -0
- package/dist/blobs/adapters/database.js.map +1 -0
- package/dist/blobs/adapters/s3.d.ts +82 -0
- package/dist/blobs/adapters/s3.d.ts.map +1 -0
- package/dist/blobs/adapters/s3.js +170 -0
- package/dist/blobs/adapters/s3.js.map +1 -0
- package/dist/blobs/index.d.ts +9 -0
- package/dist/blobs/index.d.ts.map +1 -0
- package/dist/blobs/index.js +9 -0
- package/dist/blobs/index.js.map +1 -0
- package/dist/blobs/manager.d.ts +195 -0
- package/dist/blobs/manager.d.ts.map +1 -0
- package/dist/blobs/manager.js +440 -0
- package/dist/blobs/manager.js.map +1 -0
- package/dist/blobs/migrate.d.ts +27 -0
- package/dist/blobs/migrate.d.ts.map +1 -0
- package/dist/blobs/migrate.js +119 -0
- package/dist/blobs/migrate.js.map +1 -0
- package/dist/blobs/types.d.ts +54 -0
- package/dist/blobs/types.d.ts.map +1 -0
- package/dist/blobs/types.js +5 -0
- package/dist/blobs/types.js.map +1 -0
- package/dist/clients.d.ts +14 -0
- package/dist/clients.d.ts.map +1 -0
- package/dist/clients.js +7 -0
- package/dist/clients.js.map +1 -0
- package/dist/compaction.d.ts +27 -0
- package/dist/compaction.d.ts.map +1 -0
- package/dist/compaction.js +49 -0
- package/dist/compaction.js.map +1 -0
- package/dist/dialect/base.d.ts +83 -0
- package/dist/dialect/base.d.ts.map +1 -0
- package/dist/dialect/base.js +144 -0
- package/dist/dialect/base.js.map +1 -0
- package/dist/dialect/helpers.d.ts +10 -0
- package/dist/dialect/helpers.d.ts.map +1 -0
- package/dist/dialect/helpers.js +59 -0
- package/dist/dialect/helpers.js.map +1 -0
- package/dist/dialect/index.d.ts +7 -0
- package/dist/dialect/index.d.ts.map +1 -0
- package/dist/dialect/index.js +7 -0
- package/dist/dialect/index.js.map +1 -0
- package/dist/dialect/types.d.ts +149 -0
- package/dist/dialect/types.d.ts.map +1 -0
- package/dist/dialect/types.js +8 -0
- package/dist/dialect/types.js.map +1 -0
- package/dist/helpers/conflict.d.ts +52 -0
- package/dist/helpers/conflict.d.ts.map +1 -0
- package/dist/helpers/conflict.js +49 -0
- package/dist/helpers/conflict.js.map +1 -0
- package/dist/helpers/emitted-change.d.ts +56 -0
- package/dist/helpers/emitted-change.d.ts.map +1 -0
- package/dist/helpers/emitted-change.js +46 -0
- package/dist/helpers/emitted-change.js.map +1 -0
- package/dist/helpers/index.d.ts +10 -0
- package/dist/helpers/index.d.ts.map +1 -0
- package/dist/helpers/index.js +10 -0
- package/dist/helpers/index.js.map +1 -0
- package/dist/helpers/paginate.d.ts +49 -0
- package/dist/helpers/paginate.d.ts.map +1 -0
- package/dist/helpers/paginate.js +54 -0
- package/dist/helpers/paginate.js.map +1 -0
- package/dist/helpers/scope-strings.d.ts +74 -0
- package/dist/helpers/scope-strings.d.ts.map +1 -0
- package/dist/helpers/scope-strings.js +82 -0
- package/dist/helpers/scope-strings.js.map +1 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +27 -0
- package/dist/index.js.map +1 -0
- package/dist/migrate.d.ts +14 -0
- package/dist/migrate.d.ts.map +1 -0
- package/dist/migrate.js +13 -0
- package/dist/migrate.js.map +1 -0
- package/dist/proxy/handler.d.ts +42 -0
- package/dist/proxy/handler.d.ts.map +1 -0
- package/dist/proxy/handler.js +102 -0
- package/dist/proxy/handler.js.map +1 -0
- package/dist/proxy/index.d.ts +9 -0
- package/dist/proxy/index.d.ts.map +1 -0
- package/dist/proxy/index.js +14 -0
- package/dist/proxy/index.js.map +1 -0
- package/dist/proxy/mutation-detector.d.ts +35 -0
- package/dist/proxy/mutation-detector.d.ts.map +1 -0
- package/dist/proxy/mutation-detector.js +246 -0
- package/dist/proxy/mutation-detector.js.map +1 -0
- package/dist/proxy/oplog.d.ts +30 -0
- package/dist/proxy/oplog.d.ts.map +1 -0
- package/dist/proxy/oplog.js +110 -0
- package/dist/proxy/oplog.js.map +1 -0
- package/dist/proxy/registry.d.ts +35 -0
- package/dist/proxy/registry.d.ts.map +1 -0
- package/dist/proxy/registry.js +49 -0
- package/dist/proxy/registry.js.map +1 -0
- package/dist/proxy/types.d.ts +44 -0
- package/dist/proxy/types.d.ts.map +1 -0
- package/dist/proxy/types.js +7 -0
- package/dist/proxy/types.js.map +1 -0
- package/dist/prune.d.ts +37 -0
- package/dist/prune.d.ts.map +1 -0
- package/dist/prune.js +112 -0
- package/dist/prune.js.map +1 -0
- package/dist/pull.d.ts +31 -0
- package/dist/pull.d.ts.map +1 -0
- package/dist/pull.js +608 -0
- package/dist/pull.js.map +1 -0
- package/dist/push.d.ts +33 -0
- package/dist/push.d.ts.map +1 -0
- package/dist/push.js +412 -0
- package/dist/push.js.map +1 -0
- package/dist/realtime/in-memory.d.ts +13 -0
- package/dist/realtime/in-memory.d.ts.map +1 -0
- package/dist/realtime/in-memory.js +28 -0
- package/dist/realtime/in-memory.js.map +1 -0
- package/dist/realtime/index.d.ts +3 -0
- package/dist/realtime/index.d.ts.map +1 -0
- package/dist/realtime/index.js +2 -0
- package/dist/realtime/index.js.map +1 -0
- package/dist/realtime/types.d.ts +50 -0
- package/dist/realtime/types.d.ts.map +1 -0
- package/dist/realtime/types.js +7 -0
- package/dist/realtime/types.js.map +1 -0
- package/dist/schema.d.ts +164 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +10 -0
- package/dist/schema.js.map +1 -0
- package/dist/shapes/create-handler.d.ts +119 -0
- package/dist/shapes/create-handler.d.ts.map +1 -0
- package/dist/shapes/create-handler.js +327 -0
- package/dist/shapes/create-handler.js.map +1 -0
- package/dist/shapes/index.d.ts +4 -0
- package/dist/shapes/index.d.ts.map +1 -0
- package/dist/shapes/index.js +4 -0
- package/dist/shapes/index.js.map +1 -0
- package/dist/shapes/registry.d.ts +20 -0
- package/dist/shapes/registry.d.ts.map +1 -0
- package/dist/shapes/registry.js +88 -0
- package/dist/shapes/registry.js.map +1 -0
- package/dist/shapes/types.d.ts +204 -0
- package/dist/shapes/types.d.ts.map +1 -0
- package/dist/shapes/types.js +2 -0
- package/dist/shapes/types.js.map +1 -0
- package/dist/snapshot-chunks/adapters/s3.d.ts +74 -0
- package/dist/snapshot-chunks/adapters/s3.d.ts.map +1 -0
- package/dist/snapshot-chunks/adapters/s3.js +50 -0
- package/dist/snapshot-chunks/adapters/s3.js.map +1 -0
- package/dist/snapshot-chunks/db-metadata.d.ts +38 -0
- package/dist/snapshot-chunks/db-metadata.d.ts.map +1 -0
- package/dist/snapshot-chunks/db-metadata.js +324 -0
- package/dist/snapshot-chunks/db-metadata.js.map +1 -0
- package/dist/snapshot-chunks/index.d.ts +9 -0
- package/dist/snapshot-chunks/index.d.ts.map +1 -0
- package/dist/snapshot-chunks/index.js +9 -0
- package/dist/snapshot-chunks/index.js.map +1 -0
- package/dist/snapshot-chunks/types.d.ts +78 -0
- package/dist/snapshot-chunks/types.d.ts.map +1 -0
- package/dist/snapshot-chunks/types.js +8 -0
- package/dist/snapshot-chunks/types.js.map +1 -0
- package/dist/snapshot-chunks.d.ts +60 -0
- package/dist/snapshot-chunks.d.ts.map +1 -0
- package/dist/snapshot-chunks.js +223 -0
- package/dist/snapshot-chunks.js.map +1 -0
- package/dist/stats.d.ts +19 -0
- package/dist/stats.d.ts.map +1 -0
- package/dist/stats.js +57 -0
- package/dist/stats.js.map +1 -0
- package/dist/subscriptions/index.d.ts +2 -0
- package/dist/subscriptions/index.d.ts.map +1 -0
- package/dist/subscriptions/index.js +2 -0
- package/dist/subscriptions/index.js.map +1 -0
- package/dist/subscriptions/resolve.d.ts +35 -0
- package/dist/subscriptions/resolve.d.ts.map +1 -0
- package/dist/subscriptions/resolve.js +134 -0
- package/dist/subscriptions/resolve.js.map +1 -0
- package/package.json +80 -0
- package/src/blobs/adapters/database.test.ts +67 -0
- package/src/blobs/adapters/database.ts +315 -0
- package/src/blobs/adapters/s3.ts +271 -0
- package/src/blobs/index.ts +9 -0
- package/src/blobs/manager.ts +600 -0
- package/src/blobs/migrate.ts +150 -0
- package/src/blobs/types.ts +70 -0
- package/src/clients.ts +21 -0
- package/src/compaction.ts +77 -0
- package/src/dialect/base.ts +292 -0
- package/src/dialect/helpers.ts +61 -0
- package/src/dialect/index.ts +7 -0
- package/src/dialect/types.ts +197 -0
- package/src/helpers/conflict.ts +64 -0
- package/src/helpers/emitted-change.ts +69 -0
- package/src/helpers/index.ts +10 -0
- package/src/helpers/paginate.ts +82 -0
- package/src/helpers/scope-strings.ts +101 -0
- package/src/index.ts +28 -0
- package/src/migrate.ts +20 -0
- package/src/proxy/handler.test.ts +120 -0
- package/src/proxy/handler.ts +159 -0
- package/src/proxy/index.ts +18 -0
- package/src/proxy/mutation-detector.test.ts +71 -0
- package/src/proxy/mutation-detector.ts +281 -0
- package/src/proxy/oplog.ts +146 -0
- package/src/proxy/registry.ts +56 -0
- package/src/proxy/types.ts +46 -0
- package/src/prune.ts +200 -0
- package/src/pull.ts +858 -0
- package/src/push.ts +583 -0
- package/src/realtime/in-memory.ts +33 -0
- package/src/realtime/index.ts +5 -0
- package/src/realtime/types.ts +55 -0
- package/src/schema.ts +172 -0
- package/src/shapes/create-handler.ts +590 -0
- package/src/shapes/index.ts +3 -0
- package/src/shapes/registry.ts +109 -0
- package/src/shapes/types.ts +267 -0
- package/src/snapshot-chunks/adapters/s3.ts +68 -0
- package/src/snapshot-chunks/db-metadata.test.ts +100 -0
- package/src/snapshot-chunks/db-metadata.ts +466 -0
- package/src/snapshot-chunks/index.ts +9 -0
- package/src/snapshot-chunks/types.ts +103 -0
- package/src/snapshot-chunks.ts +329 -0
- package/src/stats.ts +104 -0
- package/src/subscriptions/index.ts +1 -0
- package/src/subscriptions/resolve.ts +185 -0
package/src/pull.ts
ADDED
|
@@ -0,0 +1,858 @@
|
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
+
import { promisify } from 'node:util';
|
|
3
|
+
import { gzip, gzipSync } from 'node:zlib';
|
|
4
|
+
import {
|
|
5
|
+
captureSyncException,
|
|
6
|
+
countSyncMetric,
|
|
7
|
+
distributionSyncMetric,
|
|
8
|
+
encodeSnapshotRowFrames,
|
|
9
|
+
encodeSnapshotRows,
|
|
10
|
+
type ScopeValues,
|
|
11
|
+
SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
12
|
+
SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
13
|
+
type SyncBootstrapState,
|
|
14
|
+
type SyncChange,
|
|
15
|
+
type SyncCommit,
|
|
16
|
+
type SyncPullRequest,
|
|
17
|
+
type SyncPullResponse,
|
|
18
|
+
type SyncPullSubscriptionResponse,
|
|
19
|
+
type SyncSnapshot,
|
|
20
|
+
startSyncSpan,
|
|
21
|
+
} from '@syncular/core';
|
|
22
|
+
import type { Kysely } from 'kysely';
|
|
23
|
+
import type { ServerSyncDialect } from './dialect/types';
|
|
24
|
+
import type { SyncCoreDb } from './schema';
|
|
25
|
+
import type { TableRegistry } from './shapes/registry';
|
|
26
|
+
import {
|
|
27
|
+
insertSnapshotChunk,
|
|
28
|
+
readSnapshotChunkRefByPageKey,
|
|
29
|
+
} from './snapshot-chunks';
|
|
30
|
+
import type { SnapshotChunkStorage } from './snapshot-chunks/types';
|
|
31
|
+
import { resolveEffectiveScopesForSubscriptions } from './subscriptions/resolve';
|
|
32
|
+
|
|
33
|
+
const gzipAsync = promisify(gzip);
|
|
34
|
+
const ASYNC_GZIP_MIN_BYTES = 64 * 1024;
|
|
35
|
+
|
|
36
|
+
function concatByteChunks(chunks: readonly Uint8Array[]): Uint8Array {
|
|
37
|
+
if (chunks.length === 1) {
|
|
38
|
+
return chunks[0] ?? new Uint8Array();
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
let total = 0;
|
|
42
|
+
for (const chunk of chunks) {
|
|
43
|
+
total += chunk.length;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const merged = new Uint8Array(total);
|
|
47
|
+
let offset = 0;
|
|
48
|
+
for (const chunk of chunks) {
|
|
49
|
+
merged.set(chunk, offset);
|
|
50
|
+
offset += chunk.length;
|
|
51
|
+
}
|
|
52
|
+
return merged;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function bytesToReadableStream(bytes: Uint8Array): ReadableStream<Uint8Array> {
|
|
56
|
+
return new ReadableStream<Uint8Array>({
|
|
57
|
+
start(controller) {
|
|
58
|
+
controller.enqueue(bytes);
|
|
59
|
+
controller.close();
|
|
60
|
+
},
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function chunksToReadableStream(
|
|
65
|
+
chunks: readonly Uint8Array[]
|
|
66
|
+
): ReadableStream<Uint8Array> {
|
|
67
|
+
return new ReadableStream<Uint8Array>({
|
|
68
|
+
start(controller) {
|
|
69
|
+
for (const chunk of chunks) {
|
|
70
|
+
controller.enqueue(chunk);
|
|
71
|
+
}
|
|
72
|
+
controller.close();
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function compressSnapshotPayload(
|
|
78
|
+
payload: Uint8Array
|
|
79
|
+
): Promise<Uint8Array> {
|
|
80
|
+
if (payload.byteLength < ASYNC_GZIP_MIN_BYTES) {
|
|
81
|
+
return new Uint8Array(gzipSync(payload));
|
|
82
|
+
}
|
|
83
|
+
const compressed = await gzipAsync(payload);
|
|
84
|
+
return new Uint8Array(compressed);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async function compressSnapshotPayloadStream(
|
|
88
|
+
chunks: readonly Uint8Array[]
|
|
89
|
+
): Promise<{
|
|
90
|
+
stream: ReadableStream<Uint8Array>;
|
|
91
|
+
byteLength?: number;
|
|
92
|
+
}> {
|
|
93
|
+
if (typeof CompressionStream !== 'undefined') {
|
|
94
|
+
const source = chunksToReadableStream(chunks);
|
|
95
|
+
const gzipStream = new CompressionStream(
|
|
96
|
+
'gzip'
|
|
97
|
+
) as unknown as TransformStream<Uint8Array, Uint8Array>;
|
|
98
|
+
return {
|
|
99
|
+
stream: source.pipeThrough(gzipStream),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const payload = concatByteChunks(chunks);
|
|
104
|
+
const compressed = await compressSnapshotPayload(payload);
|
|
105
|
+
return {
|
|
106
|
+
stream: bytesToReadableStream(compressed),
|
|
107
|
+
byteLength: compressed.length,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export interface PullResult {
|
|
112
|
+
response: SyncPullResponse;
|
|
113
|
+
/**
|
|
114
|
+
* Effective scopes for all active subscriptions (for cursor tracking).
|
|
115
|
+
* Maps subscription ID to effective scopes.
|
|
116
|
+
*/
|
|
117
|
+
effectiveScopes: ScopeValues;
|
|
118
|
+
/** Minimum nextCursor across active subscriptions (for pruning cursor tracking). */
|
|
119
|
+
clientCursor: number;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Generate a stable cache key for snapshot chunks.
|
|
124
|
+
*/
|
|
125
|
+
function scopesToCacheKey(scopes: ScopeValues): string {
|
|
126
|
+
const sorted = Object.entries(scopes)
|
|
127
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
128
|
+
.map(([k, v]) => {
|
|
129
|
+
const arr = Array.isArray(v) ? [...v].sort() : [v];
|
|
130
|
+
return `${k}:${arr.join(',')}`;
|
|
131
|
+
})
|
|
132
|
+
.join('|');
|
|
133
|
+
return createHash('sha256').update(sorted).digest('hex');
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Sanitize a numeric limit parameter with bounds checking.
|
|
138
|
+
* Handles NaN, negative values, and undefined.
|
|
139
|
+
*/
|
|
140
|
+
function sanitizeLimit(
|
|
141
|
+
value: number | undefined,
|
|
142
|
+
defaultValue: number,
|
|
143
|
+
min: number,
|
|
144
|
+
max: number
|
|
145
|
+
): number {
|
|
146
|
+
if (value === undefined || value === null) return defaultValue;
|
|
147
|
+
if (Number.isNaN(value)) return defaultValue;
|
|
148
|
+
return Math.max(min, Math.min(max, value));
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Merge all scope values into a flat ScopeValues for cursor tracking.
|
|
153
|
+
*/
|
|
154
|
+
function mergeScopes(subscriptions: { scopes: ScopeValues }[]): ScopeValues {
|
|
155
|
+
const result: Record<string, Set<string>> = {};
|
|
156
|
+
|
|
157
|
+
for (const sub of subscriptions) {
|
|
158
|
+
for (const [key, value] of Object.entries(sub.scopes)) {
|
|
159
|
+
if (!result[key]) result[key] = new Set();
|
|
160
|
+
const arr = Array.isArray(value) ? value : [value];
|
|
161
|
+
for (const v of arr) result[key].add(v);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const merged: ScopeValues = {};
|
|
166
|
+
for (const [key, set] of Object.entries(result)) {
|
|
167
|
+
const arr = Array.from(set);
|
|
168
|
+
if (arr.length === 0) continue;
|
|
169
|
+
merged[key] = arr.length === 1 ? arr[0]! : arr;
|
|
170
|
+
}
|
|
171
|
+
return merged;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
interface PullResponseStats {
|
|
175
|
+
subscriptionCount: number;
|
|
176
|
+
activeSubscriptionCount: number;
|
|
177
|
+
revokedSubscriptionCount: number;
|
|
178
|
+
bootstrapSubscriptionCount: number;
|
|
179
|
+
commitCount: number;
|
|
180
|
+
changeCount: number;
|
|
181
|
+
snapshotPageCount: number;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function summarizePullResponse(response: SyncPullResponse): PullResponseStats {
|
|
185
|
+
const subscriptions = response.subscriptions ?? [];
|
|
186
|
+
let activeSubscriptionCount = 0;
|
|
187
|
+
let revokedSubscriptionCount = 0;
|
|
188
|
+
let bootstrapSubscriptionCount = 0;
|
|
189
|
+
let commitCount = 0;
|
|
190
|
+
let changeCount = 0;
|
|
191
|
+
let snapshotPageCount = 0;
|
|
192
|
+
|
|
193
|
+
for (const sub of subscriptions) {
|
|
194
|
+
if (sub.status === 'revoked') {
|
|
195
|
+
revokedSubscriptionCount += 1;
|
|
196
|
+
} else {
|
|
197
|
+
activeSubscriptionCount += 1;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if (sub.bootstrap) {
|
|
201
|
+
bootstrapSubscriptionCount += 1;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
const commits = sub.commits ?? [];
|
|
205
|
+
commitCount += commits.length;
|
|
206
|
+
for (const commit of commits) {
|
|
207
|
+
changeCount += commit.changes?.length ?? 0;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
snapshotPageCount += sub.snapshots?.length ?? 0;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
return {
|
|
214
|
+
subscriptionCount: subscriptions.length,
|
|
215
|
+
activeSubscriptionCount,
|
|
216
|
+
revokedSubscriptionCount,
|
|
217
|
+
bootstrapSubscriptionCount,
|
|
218
|
+
commitCount,
|
|
219
|
+
changeCount,
|
|
220
|
+
snapshotPageCount,
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function recordPullMetrics(args: {
|
|
225
|
+
status: string;
|
|
226
|
+
dedupeRows: boolean;
|
|
227
|
+
durationMs: number;
|
|
228
|
+
stats: PullResponseStats;
|
|
229
|
+
}): void {
|
|
230
|
+
const { status, dedupeRows, durationMs, stats } = args;
|
|
231
|
+
const attributes = {
|
|
232
|
+
status,
|
|
233
|
+
dedupe_rows: dedupeRows,
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
countSyncMetric('sync.server.pull.requests', 1, { attributes });
|
|
237
|
+
distributionSyncMetric('sync.server.pull.duration_ms', durationMs, {
|
|
238
|
+
unit: 'millisecond',
|
|
239
|
+
attributes,
|
|
240
|
+
});
|
|
241
|
+
distributionSyncMetric(
|
|
242
|
+
'sync.server.pull.subscriptions',
|
|
243
|
+
stats.subscriptionCount,
|
|
244
|
+
{ attributes }
|
|
245
|
+
);
|
|
246
|
+
distributionSyncMetric(
|
|
247
|
+
'sync.server.pull.active_subscriptions',
|
|
248
|
+
stats.activeSubscriptionCount,
|
|
249
|
+
{ attributes }
|
|
250
|
+
);
|
|
251
|
+
distributionSyncMetric(
|
|
252
|
+
'sync.server.pull.revoked_subscriptions',
|
|
253
|
+
stats.revokedSubscriptionCount,
|
|
254
|
+
{ attributes }
|
|
255
|
+
);
|
|
256
|
+
distributionSyncMetric(
|
|
257
|
+
'sync.server.pull.bootstrap_subscriptions',
|
|
258
|
+
stats.bootstrapSubscriptionCount,
|
|
259
|
+
{ attributes }
|
|
260
|
+
);
|
|
261
|
+
distributionSyncMetric('sync.server.pull.commits', stats.commitCount, {
|
|
262
|
+
attributes,
|
|
263
|
+
});
|
|
264
|
+
distributionSyncMetric('sync.server.pull.changes', stats.changeCount, {
|
|
265
|
+
attributes,
|
|
266
|
+
});
|
|
267
|
+
distributionSyncMetric(
|
|
268
|
+
'sync.server.pull.snapshot_pages',
|
|
269
|
+
stats.snapshotPageCount,
|
|
270
|
+
{ attributes }
|
|
271
|
+
);
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
export async function pull<DB extends SyncCoreDb>(args: {
|
|
275
|
+
db: Kysely<DB>;
|
|
276
|
+
dialect: ServerSyncDialect;
|
|
277
|
+
shapes: TableRegistry<DB>;
|
|
278
|
+
actorId: string;
|
|
279
|
+
partitionId?: string;
|
|
280
|
+
request: SyncPullRequest;
|
|
281
|
+
/**
|
|
282
|
+
* Optional snapshot chunk storage adapter.
|
|
283
|
+
* When provided, stores chunk bodies in external storage (S3, etc.)
|
|
284
|
+
* instead of inline in the database.
|
|
285
|
+
*/
|
|
286
|
+
chunkStorage?: SnapshotChunkStorage;
|
|
287
|
+
}): Promise<PullResult> {
|
|
288
|
+
const { request, dialect } = args;
|
|
289
|
+
const db = args.db;
|
|
290
|
+
const partitionId = args.partitionId ?? 'default';
|
|
291
|
+
const requestedSubscriptionCount = Array.isArray(request.subscriptions)
|
|
292
|
+
? request.subscriptions.length
|
|
293
|
+
: 0;
|
|
294
|
+
const startedAtMs = Date.now();
|
|
295
|
+
|
|
296
|
+
return startSyncSpan(
|
|
297
|
+
{
|
|
298
|
+
name: 'sync.server.pull',
|
|
299
|
+
op: 'sync.pull',
|
|
300
|
+
attributes: {
|
|
301
|
+
requested_subscription_count: requestedSubscriptionCount,
|
|
302
|
+
dedupe_rows: request.dedupeRows === true,
|
|
303
|
+
},
|
|
304
|
+
},
|
|
305
|
+
async (span) => {
|
|
306
|
+
try {
|
|
307
|
+
// Validate and sanitize request limits
|
|
308
|
+
const limitCommits = sanitizeLimit(request.limitCommits, 50, 1, 500);
|
|
309
|
+
const limitSnapshotRows = sanitizeLimit(
|
|
310
|
+
request.limitSnapshotRows,
|
|
311
|
+
1000,
|
|
312
|
+
1,
|
|
313
|
+
5000
|
|
314
|
+
);
|
|
315
|
+
const maxSnapshotPages = sanitizeLimit(
|
|
316
|
+
request.maxSnapshotPages,
|
|
317
|
+
1,
|
|
318
|
+
1,
|
|
319
|
+
50
|
|
320
|
+
);
|
|
321
|
+
const dedupeRows = request.dedupeRows === true;
|
|
322
|
+
|
|
323
|
+
// Resolve effective scopes for each subscription
|
|
324
|
+
const resolved = await resolveEffectiveScopesForSubscriptions({
|
|
325
|
+
db,
|
|
326
|
+
actorId: args.actorId,
|
|
327
|
+
subscriptions: request.subscriptions ?? [],
|
|
328
|
+
shapes: args.shapes,
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
const result = await dialect.executeInTransaction(db, async (trx) => {
|
|
332
|
+
await dialect.setRepeatableRead(trx);
|
|
333
|
+
|
|
334
|
+
const maxCommitSeq = await dialect.readMaxCommitSeq(trx, {
|
|
335
|
+
partitionId,
|
|
336
|
+
});
|
|
337
|
+
const minCommitSeq = await dialect.readMinCommitSeq(trx, {
|
|
338
|
+
partitionId,
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
const subResponses: SyncPullSubscriptionResponse[] = [];
|
|
342
|
+
const activeSubscriptions: { scopes: ScopeValues }[] = [];
|
|
343
|
+
const nextCursors: number[] = [];
|
|
344
|
+
|
|
345
|
+
for (const sub of resolved) {
|
|
346
|
+
const cursor = Math.max(-1, sub.cursor ?? -1);
|
|
347
|
+
// Validate shape exists (throws if not registered)
|
|
348
|
+
args.shapes.getOrThrow(sub.shape);
|
|
349
|
+
|
|
350
|
+
if (
|
|
351
|
+
sub.status === 'revoked' ||
|
|
352
|
+
Object.keys(sub.scopes).length === 0
|
|
353
|
+
) {
|
|
354
|
+
subResponses.push({
|
|
355
|
+
id: sub.id,
|
|
356
|
+
status: 'revoked',
|
|
357
|
+
scopes: {},
|
|
358
|
+
bootstrap: false,
|
|
359
|
+
nextCursor: cursor,
|
|
360
|
+
commits: [],
|
|
361
|
+
});
|
|
362
|
+
continue;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
const effectiveScopes = sub.scopes;
|
|
366
|
+
activeSubscriptions.push({ scopes: effectiveScopes });
|
|
367
|
+
|
|
368
|
+
const needsBootstrap =
|
|
369
|
+
sub.bootstrapState != null ||
|
|
370
|
+
cursor < 0 ||
|
|
371
|
+
cursor > maxCommitSeq ||
|
|
372
|
+
(minCommitSeq > 0 && cursor < minCommitSeq - 1);
|
|
373
|
+
|
|
374
|
+
if (needsBootstrap) {
|
|
375
|
+
const tables = args.shapes
|
|
376
|
+
.getBootstrapOrderFor(sub.shape)
|
|
377
|
+
.map((handler) => handler.table);
|
|
378
|
+
|
|
379
|
+
const initState: SyncBootstrapState = {
|
|
380
|
+
asOfCommitSeq: maxCommitSeq,
|
|
381
|
+
tables,
|
|
382
|
+
tableIndex: 0,
|
|
383
|
+
rowCursor: null,
|
|
384
|
+
};
|
|
385
|
+
|
|
386
|
+
const requestedState = sub.bootstrapState ?? null;
|
|
387
|
+
const state =
|
|
388
|
+
requestedState &&
|
|
389
|
+
typeof requestedState.asOfCommitSeq === 'number' &&
|
|
390
|
+
Array.isArray(requestedState.tables) &&
|
|
391
|
+
typeof requestedState.tableIndex === 'number'
|
|
392
|
+
? (requestedState as SyncBootstrapState)
|
|
393
|
+
: initState;
|
|
394
|
+
|
|
395
|
+
// If the bootstrap state's asOfCommitSeq is no longer catch-up-able, restart bootstrap.
|
|
396
|
+
const effectiveState =
|
|
397
|
+
state.asOfCommitSeq < minCommitSeq - 1 ? initState : state;
|
|
398
|
+
|
|
399
|
+
const tableName =
|
|
400
|
+
effectiveState.tables[effectiveState.tableIndex];
|
|
401
|
+
|
|
402
|
+
// No tables (or ran past the end): treat bootstrap as complete.
|
|
403
|
+
if (!tableName) {
|
|
404
|
+
subResponses.push({
|
|
405
|
+
id: sub.id,
|
|
406
|
+
status: 'active',
|
|
407
|
+
scopes: effectiveScopes,
|
|
408
|
+
bootstrap: true,
|
|
409
|
+
bootstrapState: null,
|
|
410
|
+
nextCursor: effectiveState.asOfCommitSeq,
|
|
411
|
+
commits: [],
|
|
412
|
+
snapshots: [],
|
|
413
|
+
});
|
|
414
|
+
nextCursors.push(effectiveState.asOfCommitSeq);
|
|
415
|
+
continue;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
const snapshots: SyncSnapshot[] = [];
|
|
419
|
+
let nextState: SyncBootstrapState | null = effectiveState;
|
|
420
|
+
const cacheKey = `${partitionId}:${scopesToCacheKey(effectiveScopes)}`;
|
|
421
|
+
|
|
422
|
+
interface SnapshotBundle {
|
|
423
|
+
table: string;
|
|
424
|
+
startCursor: string | null;
|
|
425
|
+
isFirstPage: boolean;
|
|
426
|
+
isLastPage: boolean;
|
|
427
|
+
pageCount: number;
|
|
428
|
+
ttlMs: number;
|
|
429
|
+
hash: ReturnType<typeof createHash>;
|
|
430
|
+
rowFrameParts: Uint8Array[];
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
const flushSnapshotBundle = async (
|
|
434
|
+
bundle: SnapshotBundle
|
|
435
|
+
): Promise<void> => {
|
|
436
|
+
const nowIso = new Date().toISOString();
|
|
437
|
+
const bundleRowLimit = Math.max(
|
|
438
|
+
1,
|
|
439
|
+
limitSnapshotRows * bundle.pageCount
|
|
440
|
+
);
|
|
441
|
+
|
|
442
|
+
const cached = await readSnapshotChunkRefByPageKey(trx, {
|
|
443
|
+
partitionId,
|
|
444
|
+
scopeKey: cacheKey,
|
|
445
|
+
scope: bundle.table,
|
|
446
|
+
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
447
|
+
rowCursor: bundle.startCursor,
|
|
448
|
+
rowLimit: bundleRowLimit,
|
|
449
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
450
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
451
|
+
nowIso,
|
|
452
|
+
});
|
|
453
|
+
|
|
454
|
+
let chunkRef = cached;
|
|
455
|
+
if (!chunkRef) {
|
|
456
|
+
const sha256 = bundle.hash.digest('hex');
|
|
457
|
+
const expiresAt = new Date(
|
|
458
|
+
Date.now() + Math.max(1000, bundle.ttlMs)
|
|
459
|
+
).toISOString();
|
|
460
|
+
|
|
461
|
+
if (args.chunkStorage) {
|
|
462
|
+
if (args.chunkStorage.storeChunkStream) {
|
|
463
|
+
const { stream: bodyStream, byteLength } =
|
|
464
|
+
await compressSnapshotPayloadStream(
|
|
465
|
+
bundle.rowFrameParts
|
|
466
|
+
);
|
|
467
|
+
chunkRef = await args.chunkStorage.storeChunkStream({
|
|
468
|
+
partitionId,
|
|
469
|
+
scopeKey: cacheKey,
|
|
470
|
+
scope: bundle.table,
|
|
471
|
+
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
472
|
+
rowCursor: bundle.startCursor,
|
|
473
|
+
rowLimit: bundleRowLimit,
|
|
474
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
475
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
476
|
+
sha256,
|
|
477
|
+
byteLength,
|
|
478
|
+
bodyStream,
|
|
479
|
+
expiresAt,
|
|
480
|
+
});
|
|
481
|
+
} else {
|
|
482
|
+
const compressedBody = await compressSnapshotPayload(
|
|
483
|
+
concatByteChunks(bundle.rowFrameParts)
|
|
484
|
+
);
|
|
485
|
+
chunkRef = await args.chunkStorage.storeChunk({
|
|
486
|
+
partitionId,
|
|
487
|
+
scopeKey: cacheKey,
|
|
488
|
+
scope: bundle.table,
|
|
489
|
+
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
490
|
+
rowCursor: bundle.startCursor,
|
|
491
|
+
rowLimit: bundleRowLimit,
|
|
492
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
493
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
494
|
+
sha256,
|
|
495
|
+
body: compressedBody,
|
|
496
|
+
expiresAt,
|
|
497
|
+
});
|
|
498
|
+
}
|
|
499
|
+
} else {
|
|
500
|
+
const compressedBody = await compressSnapshotPayload(
|
|
501
|
+
concatByteChunks(bundle.rowFrameParts)
|
|
502
|
+
);
|
|
503
|
+
const chunkId = randomUUID();
|
|
504
|
+
chunkRef = await insertSnapshotChunk(trx, {
|
|
505
|
+
chunkId,
|
|
506
|
+
partitionId,
|
|
507
|
+
scopeKey: cacheKey,
|
|
508
|
+
scope: bundle.table,
|
|
509
|
+
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
510
|
+
rowCursor: bundle.startCursor,
|
|
511
|
+
rowLimit: bundleRowLimit,
|
|
512
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
513
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
514
|
+
sha256,
|
|
515
|
+
body: compressedBody,
|
|
516
|
+
expiresAt,
|
|
517
|
+
});
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
snapshots.push({
|
|
522
|
+
table: bundle.table,
|
|
523
|
+
rows: [],
|
|
524
|
+
chunks: [chunkRef],
|
|
525
|
+
isFirstPage: bundle.isFirstPage,
|
|
526
|
+
isLastPage: bundle.isLastPage,
|
|
527
|
+
});
|
|
528
|
+
};
|
|
529
|
+
|
|
530
|
+
let activeBundle: SnapshotBundle | null = null;
|
|
531
|
+
|
|
532
|
+
for (
|
|
533
|
+
let pageIndex = 0;
|
|
534
|
+
pageIndex < maxSnapshotPages;
|
|
535
|
+
pageIndex++
|
|
536
|
+
) {
|
|
537
|
+
if (!nextState) break;
|
|
538
|
+
|
|
539
|
+
const nextTableName = nextState.tables[nextState.tableIndex];
|
|
540
|
+
if (!nextTableName) {
|
|
541
|
+
if (activeBundle) {
|
|
542
|
+
activeBundle.isLastPage = true;
|
|
543
|
+
await flushSnapshotBundle(activeBundle);
|
|
544
|
+
activeBundle = null;
|
|
545
|
+
}
|
|
546
|
+
nextState = null;
|
|
547
|
+
break;
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
const tableHandler = args.shapes.getOrThrow(nextTableName);
|
|
551
|
+
if (!activeBundle || activeBundle.table !== nextTableName) {
|
|
552
|
+
if (activeBundle) {
|
|
553
|
+
await flushSnapshotBundle(activeBundle);
|
|
554
|
+
}
|
|
555
|
+
const bundleHash = createHash('sha256');
|
|
556
|
+
const bundleHeader = encodeSnapshotRows([]);
|
|
557
|
+
bundleHash.update(bundleHeader);
|
|
558
|
+
activeBundle = {
|
|
559
|
+
table: nextTableName,
|
|
560
|
+
startCursor: nextState.rowCursor,
|
|
561
|
+
isFirstPage: nextState.rowCursor == null,
|
|
562
|
+
isLastPage: false,
|
|
563
|
+
pageCount: 0,
|
|
564
|
+
ttlMs:
|
|
565
|
+
tableHandler.snapshotChunkTtlMs ?? 24 * 60 * 60 * 1000,
|
|
566
|
+
hash: bundleHash,
|
|
567
|
+
rowFrameParts: [bundleHeader],
|
|
568
|
+
};
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
const page = await tableHandler.snapshot(
|
|
572
|
+
{
|
|
573
|
+
db: trx,
|
|
574
|
+
actorId: args.actorId,
|
|
575
|
+
scopeValues: effectiveScopes,
|
|
576
|
+
cursor: nextState.rowCursor,
|
|
577
|
+
limit: limitSnapshotRows,
|
|
578
|
+
},
|
|
579
|
+
sub.params
|
|
580
|
+
);
|
|
581
|
+
|
|
582
|
+
const rowFrames = encodeSnapshotRowFrames(page.rows ?? []);
|
|
583
|
+
activeBundle.hash.update(rowFrames);
|
|
584
|
+
activeBundle.rowFrameParts.push(rowFrames);
|
|
585
|
+
activeBundle.pageCount += 1;
|
|
586
|
+
|
|
587
|
+
if (page.nextCursor != null) {
|
|
588
|
+
nextState = { ...nextState, rowCursor: page.nextCursor };
|
|
589
|
+
continue;
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
activeBundle.isLastPage = true;
|
|
593
|
+
await flushSnapshotBundle(activeBundle);
|
|
594
|
+
activeBundle = null;
|
|
595
|
+
|
|
596
|
+
if (nextState.tableIndex + 1 < nextState.tables.length) {
|
|
597
|
+
nextState = {
|
|
598
|
+
...nextState,
|
|
599
|
+
tableIndex: nextState.tableIndex + 1,
|
|
600
|
+
rowCursor: null,
|
|
601
|
+
};
|
|
602
|
+
continue;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
nextState = null;
|
|
606
|
+
break;
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
if (activeBundle) {
|
|
610
|
+
await flushSnapshotBundle(activeBundle);
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
subResponses.push({
|
|
614
|
+
id: sub.id,
|
|
615
|
+
status: 'active',
|
|
616
|
+
scopes: effectiveScopes,
|
|
617
|
+
bootstrap: true,
|
|
618
|
+
bootstrapState: nextState,
|
|
619
|
+
nextCursor: effectiveState.asOfCommitSeq,
|
|
620
|
+
commits: [],
|
|
621
|
+
snapshots,
|
|
622
|
+
});
|
|
623
|
+
nextCursors.push(effectiveState.asOfCommitSeq);
|
|
624
|
+
continue;
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
// Incremental pull for this subscription
|
|
628
|
+
// Read the commit window for this table up-front so the subscription cursor
|
|
629
|
+
// can advance past commits that don't match the requested scopes.
|
|
630
|
+
const scannedCommitSeqs = await dialect.readCommitSeqsForPull(trx, {
|
|
631
|
+
partitionId,
|
|
632
|
+
cursor,
|
|
633
|
+
limitCommits,
|
|
634
|
+
tables: [sub.shape],
|
|
635
|
+
});
|
|
636
|
+
const maxScannedCommitSeq =
|
|
637
|
+
scannedCommitSeqs.length > 0
|
|
638
|
+
? scannedCommitSeqs[scannedCommitSeqs.length - 1]!
|
|
639
|
+
: cursor;
|
|
640
|
+
|
|
641
|
+
// Collect rows and compute nextCursor in a single pass
|
|
642
|
+
const incrementalRows: Array<{
|
|
643
|
+
commit_seq: number;
|
|
644
|
+
actor_id: string;
|
|
645
|
+
created_at: string;
|
|
646
|
+
change_id: number;
|
|
647
|
+
table: string;
|
|
648
|
+
row_id: string;
|
|
649
|
+
op: 'upsert' | 'delete';
|
|
650
|
+
row_json: unknown | null;
|
|
651
|
+
row_version: number | null;
|
|
652
|
+
scopes: Record<string, string | string[]>;
|
|
653
|
+
}> = [];
|
|
654
|
+
|
|
655
|
+
let nextCursor = cursor;
|
|
656
|
+
|
|
657
|
+
for await (const row of dialect.iterateIncrementalPullRows(trx, {
|
|
658
|
+
partitionId,
|
|
659
|
+
table: sub.shape,
|
|
660
|
+
scopes: effectiveScopes,
|
|
661
|
+
cursor,
|
|
662
|
+
limitCommits,
|
|
663
|
+
})) {
|
|
664
|
+
incrementalRows.push(row);
|
|
665
|
+
nextCursor = Math.max(nextCursor, row.commit_seq);
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
|
|
669
|
+
|
|
670
|
+
if (incrementalRows.length === 0) {
|
|
671
|
+
subResponses.push({
|
|
672
|
+
id: sub.id,
|
|
673
|
+
status: 'active',
|
|
674
|
+
scopes: effectiveScopes,
|
|
675
|
+
bootstrap: false,
|
|
676
|
+
nextCursor,
|
|
677
|
+
commits: [],
|
|
678
|
+
});
|
|
679
|
+
nextCursors.push(nextCursor);
|
|
680
|
+
continue;
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
if (dedupeRows) {
|
|
684
|
+
const latestByRowKey = new Map<
|
|
685
|
+
string,
|
|
686
|
+
{
|
|
687
|
+
commitSeq: number;
|
|
688
|
+
createdAt: string;
|
|
689
|
+
actorId: string;
|
|
690
|
+
changeId: number;
|
|
691
|
+
change: SyncChange;
|
|
692
|
+
}
|
|
693
|
+
>();
|
|
694
|
+
|
|
695
|
+
for (const r of incrementalRows) {
|
|
696
|
+
const rowKey = `${r.table}\u0000${r.row_id}`;
|
|
697
|
+
const change: SyncChange = {
|
|
698
|
+
table: r.table,
|
|
699
|
+
row_id: r.row_id,
|
|
700
|
+
op: r.op,
|
|
701
|
+
row_json: r.row_json,
|
|
702
|
+
row_version: r.row_version,
|
|
703
|
+
scopes: dialect.dbToScopes(r.scopes),
|
|
704
|
+
};
|
|
705
|
+
|
|
706
|
+
latestByRowKey.set(rowKey, {
|
|
707
|
+
commitSeq: r.commit_seq,
|
|
708
|
+
createdAt: r.created_at,
|
|
709
|
+
actorId: r.actor_id,
|
|
710
|
+
changeId: r.change_id,
|
|
711
|
+
change,
|
|
712
|
+
});
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
const latest = Array.from(latestByRowKey.values()).sort(
|
|
716
|
+
(a, b) => a.commitSeq - b.commitSeq || a.changeId - b.changeId
|
|
717
|
+
);
|
|
718
|
+
|
|
719
|
+
const commitsBySeq = new Map<number, SyncCommit>();
|
|
720
|
+
for (const item of latest) {
|
|
721
|
+
let commit = commitsBySeq.get(item.commitSeq);
|
|
722
|
+
if (!commit) {
|
|
723
|
+
commit = {
|
|
724
|
+
commitSeq: item.commitSeq,
|
|
725
|
+
createdAt: item.createdAt,
|
|
726
|
+
actorId: item.actorId,
|
|
727
|
+
changes: [],
|
|
728
|
+
};
|
|
729
|
+
commitsBySeq.set(item.commitSeq, commit);
|
|
730
|
+
}
|
|
731
|
+
commit.changes.push(item.change);
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
const commits = Array.from(commitsBySeq.values()).sort(
|
|
735
|
+
(a, b) => a.commitSeq - b.commitSeq
|
|
736
|
+
);
|
|
737
|
+
|
|
738
|
+
subResponses.push({
|
|
739
|
+
id: sub.id,
|
|
740
|
+
status: 'active',
|
|
741
|
+
scopes: effectiveScopes,
|
|
742
|
+
bootstrap: false,
|
|
743
|
+
nextCursor,
|
|
744
|
+
commits,
|
|
745
|
+
});
|
|
746
|
+
nextCursors.push(nextCursor);
|
|
747
|
+
continue;
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
const commitsBySeq = new Map<number, SyncCommit>();
|
|
751
|
+
const commitSeqs: number[] = [];
|
|
752
|
+
|
|
753
|
+
for (const r of incrementalRows) {
|
|
754
|
+
const seq = r.commit_seq;
|
|
755
|
+
let commit = commitsBySeq.get(seq);
|
|
756
|
+
if (!commit) {
|
|
757
|
+
commit = {
|
|
758
|
+
commitSeq: seq,
|
|
759
|
+
createdAt: r.created_at,
|
|
760
|
+
actorId: r.actor_id,
|
|
761
|
+
changes: [],
|
|
762
|
+
};
|
|
763
|
+
commitsBySeq.set(seq, commit);
|
|
764
|
+
commitSeqs.push(seq);
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
const change: SyncChange = {
|
|
768
|
+
table: r.table,
|
|
769
|
+
row_id: r.row_id,
|
|
770
|
+
op: r.op,
|
|
771
|
+
row_json: r.row_json,
|
|
772
|
+
row_version: r.row_version,
|
|
773
|
+
scopes: dialect.dbToScopes(r.scopes),
|
|
774
|
+
};
|
|
775
|
+
commit.changes.push(change);
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
const commits: SyncCommit[] = commitSeqs
|
|
779
|
+
.map((seq) => commitsBySeq.get(seq))
|
|
780
|
+
.filter((c): c is SyncCommit => !!c)
|
|
781
|
+
.filter((c) => c.changes.length > 0);
|
|
782
|
+
|
|
783
|
+
subResponses.push({
|
|
784
|
+
id: sub.id,
|
|
785
|
+
status: 'active',
|
|
786
|
+
scopes: effectiveScopes,
|
|
787
|
+
bootstrap: false,
|
|
788
|
+
nextCursor,
|
|
789
|
+
commits,
|
|
790
|
+
});
|
|
791
|
+
nextCursors.push(nextCursor);
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
const effectiveScopes = mergeScopes(activeSubscriptions);
|
|
795
|
+
const clientCursor =
|
|
796
|
+
nextCursors.length > 0 ? Math.min(...nextCursors) : maxCommitSeq;
|
|
797
|
+
|
|
798
|
+
return {
|
|
799
|
+
response: {
|
|
800
|
+
ok: true as const,
|
|
801
|
+
subscriptions: subResponses,
|
|
802
|
+
},
|
|
803
|
+
effectiveScopes,
|
|
804
|
+
clientCursor,
|
|
805
|
+
};
|
|
806
|
+
});
|
|
807
|
+
|
|
808
|
+
const durationMs = Math.max(0, Date.now() - startedAtMs);
|
|
809
|
+
const stats = summarizePullResponse(result.response);
|
|
810
|
+
|
|
811
|
+
span.setAttribute('status', 'ok');
|
|
812
|
+
span.setAttribute('duration_ms', durationMs);
|
|
813
|
+
span.setAttribute('subscription_count', stats.subscriptionCount);
|
|
814
|
+
span.setAttribute('commit_count', stats.commitCount);
|
|
815
|
+
span.setAttribute('change_count', stats.changeCount);
|
|
816
|
+
span.setAttribute('snapshot_page_count', stats.snapshotPageCount);
|
|
817
|
+
span.setStatus('ok');
|
|
818
|
+
|
|
819
|
+
recordPullMetrics({
|
|
820
|
+
status: 'ok',
|
|
821
|
+
dedupeRows,
|
|
822
|
+
durationMs,
|
|
823
|
+
stats,
|
|
824
|
+
});
|
|
825
|
+
|
|
826
|
+
return result;
|
|
827
|
+
} catch (error) {
|
|
828
|
+
const durationMs = Math.max(0, Date.now() - startedAtMs);
|
|
829
|
+
|
|
830
|
+
span.setAttribute('status', 'error');
|
|
831
|
+
span.setAttribute('duration_ms', durationMs);
|
|
832
|
+
span.setStatus('error');
|
|
833
|
+
|
|
834
|
+
recordPullMetrics({
|
|
835
|
+
status: 'error',
|
|
836
|
+
dedupeRows: request.dedupeRows === true,
|
|
837
|
+
durationMs,
|
|
838
|
+
stats: {
|
|
839
|
+
subscriptionCount: 0,
|
|
840
|
+
activeSubscriptionCount: 0,
|
|
841
|
+
revokedSubscriptionCount: 0,
|
|
842
|
+
bootstrapSubscriptionCount: 0,
|
|
843
|
+
commitCount: 0,
|
|
844
|
+
changeCount: 0,
|
|
845
|
+
snapshotPageCount: 0,
|
|
846
|
+
},
|
|
847
|
+
});
|
|
848
|
+
|
|
849
|
+
captureSyncException(error, {
|
|
850
|
+
event: 'sync.server.pull',
|
|
851
|
+
requestedSubscriptionCount,
|
|
852
|
+
dedupeRows: request.dedupeRows === true,
|
|
853
|
+
});
|
|
854
|
+
throw error;
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
);
|
|
858
|
+
}
|