@syncular/server-hono 0.0.1-60

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/dist/api-key-auth.d.ts +49 -0
  2. package/dist/api-key-auth.d.ts.map +1 -0
  3. package/dist/api-key-auth.js +110 -0
  4. package/dist/api-key-auth.js.map +1 -0
  5. package/dist/blobs.d.ts +69 -0
  6. package/dist/blobs.d.ts.map +1 -0
  7. package/dist/blobs.js +383 -0
  8. package/dist/blobs.js.map +1 -0
  9. package/dist/console/index.d.ts +8 -0
  10. package/dist/console/index.d.ts.map +1 -0
  11. package/dist/console/index.js +7 -0
  12. package/dist/console/index.js.map +1 -0
  13. package/dist/console/routes.d.ts +106 -0
  14. package/dist/console/routes.d.ts.map +1 -0
  15. package/dist/console/routes.js +1612 -0
  16. package/dist/console/routes.js.map +1 -0
  17. package/dist/console/schemas.d.ts +308 -0
  18. package/dist/console/schemas.d.ts.map +1 -0
  19. package/dist/console/schemas.js +201 -0
  20. package/dist/console/schemas.js.map +1 -0
  21. package/dist/create-server.d.ts +78 -0
  22. package/dist/create-server.d.ts.map +1 -0
  23. package/dist/create-server.js +99 -0
  24. package/dist/create-server.js.map +1 -0
  25. package/dist/index.d.ts +16 -0
  26. package/dist/index.d.ts.map +1 -0
  27. package/dist/index.js +25 -0
  28. package/dist/index.js.map +1 -0
  29. package/dist/openapi.d.ts +45 -0
  30. package/dist/openapi.d.ts.map +1 -0
  31. package/dist/openapi.js +59 -0
  32. package/dist/openapi.js.map +1 -0
  33. package/dist/proxy/connection-manager.d.ts +78 -0
  34. package/dist/proxy/connection-manager.d.ts.map +1 -0
  35. package/dist/proxy/connection-manager.js +251 -0
  36. package/dist/proxy/connection-manager.js.map +1 -0
  37. package/dist/proxy/index.d.ts +8 -0
  38. package/dist/proxy/index.d.ts.map +1 -0
  39. package/dist/proxy/index.js +8 -0
  40. package/dist/proxy/index.js.map +1 -0
  41. package/dist/proxy/routes.d.ts +74 -0
  42. package/dist/proxy/routes.d.ts.map +1 -0
  43. package/dist/proxy/routes.js +147 -0
  44. package/dist/proxy/routes.js.map +1 -0
  45. package/dist/rate-limit.d.ts +101 -0
  46. package/dist/rate-limit.d.ts.map +1 -0
  47. package/dist/rate-limit.js +186 -0
  48. package/dist/rate-limit.js.map +1 -0
  49. package/dist/routes.d.ts +126 -0
  50. package/dist/routes.d.ts.map +1 -0
  51. package/dist/routes.js +788 -0
  52. package/dist/routes.js.map +1 -0
  53. package/dist/ws.d.ts +230 -0
  54. package/dist/ws.d.ts.map +1 -0
  55. package/dist/ws.js +601 -0
  56. package/dist/ws.js.map +1 -0
  57. package/package.json +73 -0
  58. package/src/__tests__/create-server.test.ts +187 -0
  59. package/src/__tests__/pull-chunk-storage.test.ts +189 -0
  60. package/src/__tests__/rate-limit.test.ts +78 -0
  61. package/src/__tests__/realtime-bridge.test.ts +131 -0
  62. package/src/__tests__/ws-connection-manager.test.ts +176 -0
  63. package/src/api-key-auth.ts +179 -0
  64. package/src/blobs.ts +534 -0
  65. package/src/console/index.ts +17 -0
  66. package/src/console/routes.ts +2155 -0
  67. package/src/console/schemas.ts +299 -0
  68. package/src/create-server.ts +180 -0
  69. package/src/index.ts +42 -0
  70. package/src/openapi.ts +74 -0
  71. package/src/proxy/connection-manager.ts +340 -0
  72. package/src/proxy/index.ts +8 -0
  73. package/src/proxy/routes.ts +223 -0
  74. package/src/rate-limit.ts +321 -0
  75. package/src/routes.ts +1186 -0
  76. package/src/ws.ts +789 -0
package/src/routes.ts ADDED
@@ -0,0 +1,1186 @@
1
+ /**
2
+ * @syncular/server-hono - Sync routes for Hono
3
+ *
4
+ * Provides:
5
+ * - POST / (combined push + pull in one round-trip)
6
+ * - GET /snapshot-chunks/:chunkId (download encoded snapshot chunks)
7
+ * - GET /realtime (optional WebSocket "wake up" notifications)
8
+ */
9
+
10
+ import {
11
+ createSyncTimer,
12
+ ErrorResponseSchema,
13
+ logSyncEvent,
14
+ SyncCombinedRequestSchema,
15
+ SyncCombinedResponseSchema,
16
+ SyncPushRequestSchema,
17
+ } from '@syncular/core';
18
+ import type {
19
+ ServerSyncDialect,
20
+ ServerTableHandler,
21
+ SnapshotChunkStorage,
22
+ SyncCoreDb,
23
+ SyncRealtimeBroadcaster,
24
+ SyncRealtimeEvent,
25
+ } from '@syncular/server';
26
+ import {
27
+ type CompactOptions,
28
+ InvalidSubscriptionScopeError,
29
+ type PruneOptions,
30
+ type PullResult,
31
+ pull,
32
+ pushCommit,
33
+ readSnapshotChunk,
34
+ recordClientCursor,
35
+ TableRegistry,
36
+ } from '@syncular/server';
37
+ import type { Context } from 'hono';
38
+ import { Hono } from 'hono';
39
+
40
+ import type { UpgradeWebSocket } from 'hono/ws';
41
+ import { describeRoute, resolver, validator as zValidator } from 'hono-openapi';
42
+ import {
43
+ type Kysely,
44
+ type SelectQueryBuilder,
45
+ type SqlBool,
46
+ sql,
47
+ } from 'kysely';
48
+ import { z } from 'zod';
49
+ import {
50
+ createRateLimiter,
51
+ DEFAULT_SYNC_RATE_LIMITS,
52
+ type SyncRateLimitConfig,
53
+ } from './rate-limit';
54
+ import {
55
+ createWebSocketConnection,
56
+ type WebSocketConnection,
57
+ WebSocketConnectionManager,
58
+ } from './ws';
59
+
60
+ /**
61
+ * WeakMaps for storing Hono-instance-specific data without augmenting the type.
62
+ */
63
+ const wsConnectionManagerMap = new WeakMap<Hono, WebSocketConnectionManager>();
64
+ const realtimeUnsubscribeMap = new WeakMap<Hono, () => void>();
65
+
66
+ export interface SyncAuthResult {
67
+ actorId: string;
68
+ partitionId?: string;
69
+ }
70
+
71
+ /**
72
+ * WebSocket configuration for realtime sync.
73
+ *
74
+ * Note: this endpoint is only a "wake up" mechanism; clients must still pull.
75
+ */
76
+ export interface SyncWebSocketConfig {
77
+ enabled?: boolean;
78
+ /**
79
+ * Runtime-provided WebSocket upgrader (e.g. from `hono/bun`'s `createBunWebSocket()`).
80
+ */
81
+ upgradeWebSocket?: UpgradeWebSocket;
82
+ heartbeatIntervalMs?: number;
83
+ /**
84
+ * Maximum number of concurrent WebSocket connections across the entire process.
85
+ * Default: 5000
86
+ */
87
+ maxConnectionsTotal?: number;
88
+ /**
89
+ * Maximum number of concurrent WebSocket connections per clientId.
90
+ * Default: 3
91
+ */
92
+ maxConnectionsPerClient?: number;
93
+ }
94
+
95
+ export interface SyncRoutesConfigWithRateLimit {
96
+ /**
97
+ * Max commits per pull request.
98
+ * Default: 100
99
+ */
100
+ maxPullLimitCommits?: number;
101
+ /**
102
+ * Max subscriptions per pull request.
103
+ * Default: 200
104
+ */
105
+ maxSubscriptionsPerPull?: number;
106
+ /**
107
+ * Max snapshot rows per snapshot page.
108
+ * Default: 5000
109
+ */
110
+ maxPullLimitSnapshotRows?: number;
111
+ /**
112
+ * Max snapshot pages per subscription per pull response.
113
+ * Default: 10
114
+ */
115
+ maxPullMaxSnapshotPages?: number;
116
+ /**
117
+ * Max operations per pushed commit.
118
+ * Default: 200
119
+ */
120
+ maxOperationsPerPush?: number;
121
+ /**
122
+ * Rate limiting configuration.
123
+ * Set to false to disable all rate limiting.
124
+ */
125
+ rateLimit?: SyncRateLimitConfig | false;
126
+ /**
127
+ * WebSocket realtime configuration.
128
+ */
129
+ websocket?: SyncWebSocketConfig;
130
+
131
+ /**
132
+ * Optional pruning configuration. When enabled, the server periodically prunes
133
+ * old commit history based on active client cursors.
134
+ */
135
+ prune?: {
136
+ /** Minimum time between prune runs. Default: 5 minutes. */
137
+ minIntervalMs?: number;
138
+ /** Pruning watermark options. */
139
+ options?: PruneOptions;
140
+ };
141
+
142
+ /**
143
+ * Optional compaction configuration. When enabled, the server periodically
144
+ * compacts older change history to reduce storage.
145
+ */
146
+ compact?: {
147
+ /** Minimum time between compaction runs. Default: 30 minutes. */
148
+ minIntervalMs?: number;
149
+ /** Compaction options. */
150
+ options?: CompactOptions;
151
+ };
152
+
153
+ /**
154
+ * Optional multi-instance realtime broadcaster.
155
+ * When provided, instances publish/subscribe commit wakeups via the broadcaster.
156
+ */
157
+ realtime?: {
158
+ broadcaster: SyncRealtimeBroadcaster;
159
+ /** Optional stable instance id (useful in tests). */
160
+ instanceId?: string;
161
+ };
162
+ }
163
+
164
+ export interface CreateSyncRoutesOptions<DB extends SyncCoreDb = SyncCoreDb> {
165
+ db: Kysely<DB>;
166
+ dialect: ServerSyncDialect;
167
+ handlers: ServerTableHandler<DB>[];
168
+ authenticate: (c: Context) => Promise<SyncAuthResult | null>;
169
+ sync?: SyncRoutesConfigWithRateLimit;
170
+ wsConnectionManager?: WebSocketConnectionManager;
171
+ /**
172
+ * Optional snapshot chunk storage adapter.
173
+ * When provided, stores snapshot chunk bodies in external storage
174
+ * (S3, R2, etc.) instead of inline in the database.
175
+ */
176
+ chunkStorage?: SnapshotChunkStorage;
177
+ }
178
+
179
+ // ============================================================================
180
+ // Route Schemas
181
+ // ============================================================================
182
+
183
+ const snapshotChunkParamsSchema = z.object({
184
+ chunkId: z.string().min(1),
185
+ });
186
+
187
+ export function createSyncRoutes<DB extends SyncCoreDb = SyncCoreDb>(
188
+ options: CreateSyncRoutesOptions<DB>
189
+ ): Hono {
190
+ const routes = new Hono();
191
+ const handlerRegistry = new TableRegistry<DB>();
192
+ for (const handler of options.handlers) {
193
+ handlerRegistry.register(handler);
194
+ }
195
+ const config = options.sync ?? {};
196
+ const maxPullLimitCommits = config.maxPullLimitCommits ?? 100;
197
+ const maxSubscriptionsPerPull = config.maxSubscriptionsPerPull ?? 200;
198
+ const maxPullLimitSnapshotRows = config.maxPullLimitSnapshotRows ?? 5000;
199
+ const maxPullMaxSnapshotPages = config.maxPullMaxSnapshotPages ?? 10;
200
+ const maxOperationsPerPush = config.maxOperationsPerPush ?? 200;
201
+
202
+ // -------------------------------------------------------------------------
203
+ // Optional WebSocket manager (scope-key based wake-ups)
204
+ // -------------------------------------------------------------------------
205
+
206
+ const websocketConfig = config.websocket;
207
+ if (websocketConfig?.enabled && !websocketConfig.upgradeWebSocket) {
208
+ throw new Error(
209
+ 'sync.websocket.enabled requires sync.websocket.upgradeWebSocket'
210
+ );
211
+ }
212
+
213
+ const wsConnectionManager = websocketConfig?.enabled
214
+ ? (options.wsConnectionManager ??
215
+ new WebSocketConnectionManager({
216
+ heartbeatIntervalMs: websocketConfig.heartbeatIntervalMs ?? 30_000,
217
+ }))
218
+ : null;
219
+
220
+ if (wsConnectionManager) {
221
+ wsConnectionManagerMap.set(routes, wsConnectionManager);
222
+ }
223
+
224
+ // -------------------------------------------------------------------------
225
+ // Multi-instance realtime broadcaster (optional)
226
+ // -------------------------------------------------------------------------
227
+
228
+ const realtimeBroadcaster = config.realtime?.broadcaster ?? null;
229
+ const instanceId =
230
+ config.realtime?.instanceId ??
231
+ (typeof crypto !== 'undefined' && 'randomUUID' in crypto
232
+ ? crypto.randomUUID()
233
+ : `${Date.now()}-${Math.random().toString(16).slice(2)}`);
234
+
235
+ if (wsConnectionManager && realtimeBroadcaster) {
236
+ const unsubscribe = realtimeBroadcaster.subscribe(
237
+ (event: SyncRealtimeEvent) => {
238
+ void handleRealtimeEvent(event).catch(() => {});
239
+ }
240
+ );
241
+
242
+ realtimeUnsubscribeMap.set(routes, unsubscribe);
243
+ }
244
+
245
+ // -------------------------------------------------------------------------
246
+ // Request event recording (for console inspector)
247
+ // -------------------------------------------------------------------------
248
+
249
+ const recordRequestEvent = async (event: {
250
+ eventType: 'push' | 'pull';
251
+ actorId: string;
252
+ clientId: string;
253
+ transportPath: 'direct' | 'relay';
254
+ statusCode: number;
255
+ outcome: string;
256
+ durationMs: number;
257
+ commitSeq?: number | null;
258
+ operationCount?: number | null;
259
+ rowCount?: number | null;
260
+ tables?: string[];
261
+ errorMessage?: string | null;
262
+ }) => {
263
+ try {
264
+ const tablesValue = options.dialect.arrayToDb(event.tables ?? []);
265
+ await sql`
266
+ INSERT INTO sync_request_events (
267
+ event_type, actor_id, client_id, status_code, outcome,
268
+ duration_ms, commit_seq, operation_count, row_count,
269
+ tables, error_message, transport_path
270
+ ) VALUES (
271
+ ${event.eventType}, ${event.actorId}, ${event.clientId},
272
+ ${event.statusCode}, ${event.outcome}, ${event.durationMs},
273
+ ${event.commitSeq ?? null}, ${event.operationCount ?? null},
274
+ ${event.rowCount ?? null}, ${tablesValue}, ${event.errorMessage ?? null},
275
+ ${event.transportPath}
276
+ )
277
+ `.execute(options.db);
278
+ } catch {
279
+ // Silently ignore - event recording should not block sync
280
+ }
281
+ };
282
+
283
+ // -------------------------------------------------------------------------
284
+ // Rate limiting (optional)
285
+ // -------------------------------------------------------------------------
286
+
287
+ const rateLimitConfig = config.rateLimit;
288
+ if (rateLimitConfig !== false) {
289
+ const pullRateLimit =
290
+ rateLimitConfig?.pull ?? DEFAULT_SYNC_RATE_LIMITS.pull;
291
+ const pushRateLimit =
292
+ rateLimitConfig?.push ?? DEFAULT_SYNC_RATE_LIMITS.push;
293
+
294
+ const createAuthBasedRateLimiter = (
295
+ limitConfig: Omit<SyncRateLimitConfig['pull'], never> | false | undefined
296
+ ) => {
297
+ if (limitConfig === false || !limitConfig) return null;
298
+ return createRateLimiter({
299
+ ...limitConfig,
300
+ keyGenerator: async (c) => {
301
+ const auth = await options.authenticate(c);
302
+ return auth?.actorId ?? null;
303
+ },
304
+ });
305
+ };
306
+
307
+ const pullLimiter = createAuthBasedRateLimiter(pullRateLimit);
308
+ if (pullLimiter) routes.use('/', pullLimiter);
309
+
310
+ const pushLimiter = createAuthBasedRateLimiter(pushRateLimit);
311
+ if (pushLimiter) routes.use('/', pushLimiter);
312
+ }
313
+
314
+ // -------------------------------------------------------------------------
315
+ // GET /health
316
+ // -------------------------------------------------------------------------
317
+
318
+ routes.get('/health', (c) => {
319
+ return c.json({
320
+ status: 'healthy',
321
+ timestamp: new Date().toISOString(),
322
+ });
323
+ });
324
+
325
+ // -------------------------------------------------------------------------
326
+ // POST / (combined push + pull in one round-trip)
327
+ // -------------------------------------------------------------------------
328
+
329
+ routes.post(
330
+ '/',
331
+ describeRoute({
332
+ tags: ['sync'],
333
+ summary: 'Combined push and pull',
334
+ description:
335
+ 'Perform push and/or pull in a single request to reduce round-trips',
336
+ responses: {
337
+ 200: {
338
+ description: 'Combined sync response',
339
+ content: {
340
+ 'application/json': {
341
+ schema: resolver(SyncCombinedResponseSchema),
342
+ },
343
+ },
344
+ },
345
+ 400: {
346
+ description: 'Invalid request',
347
+ content: {
348
+ 'application/json': { schema: resolver(ErrorResponseSchema) },
349
+ },
350
+ },
351
+ 401: {
352
+ description: 'Unauthenticated',
353
+ content: {
354
+ 'application/json': { schema: resolver(ErrorResponseSchema) },
355
+ },
356
+ },
357
+ },
358
+ }),
359
+ zValidator('json', SyncCombinedRequestSchema),
360
+ async (c) => {
361
+ const auth = await options.authenticate(c);
362
+ if (!auth) return c.json({ error: 'UNAUTHENTICATED' }, 401);
363
+ const partitionId = auth.partitionId ?? 'default';
364
+
365
+ const body = c.req.valid('json');
366
+ const clientId = body.clientId;
367
+
368
+ let pushResponse:
369
+ | undefined
370
+ | Awaited<ReturnType<typeof pushCommit>>['response'];
371
+ let pullResponse: undefined | PullResult['response'];
372
+
373
+ // --- Push phase ---
374
+ if (body.push) {
375
+ const pushOps = body.push.operations ?? [];
376
+ if (pushOps.length > maxOperationsPerPush) {
377
+ return c.json(
378
+ {
379
+ error: 'TOO_MANY_OPERATIONS',
380
+ message: `Maximum ${maxOperationsPerPush} operations per push`,
381
+ },
382
+ 400
383
+ );
384
+ }
385
+
386
+ const timer = createSyncTimer();
387
+
388
+ const pushed = await pushCommit({
389
+ db: options.db,
390
+ dialect: options.dialect,
391
+ shapes: handlerRegistry,
392
+ actorId: auth.actorId,
393
+ partitionId,
394
+ request: {
395
+ clientId,
396
+ clientCommitId: body.push.clientCommitId,
397
+ operations: body.push.operations,
398
+ schemaVersion: body.push.schemaVersion,
399
+ },
400
+ });
401
+
402
+ const pushDurationMs = timer();
403
+
404
+ logSyncEvent({
405
+ event: 'sync.push',
406
+ userId: auth.actorId,
407
+ durationMs: pushDurationMs,
408
+ operationCount: pushOps.length,
409
+ status: pushed.response.status,
410
+ commitSeq: pushed.response.commitSeq,
411
+ });
412
+
413
+ recordRequestEvent({
414
+ eventType: 'push',
415
+ actorId: auth.actorId,
416
+ clientId,
417
+ transportPath: readTransportPath(c),
418
+ statusCode: 200,
419
+ outcome: pushed.response.status,
420
+ durationMs: pushDurationMs,
421
+ commitSeq: pushed.response.commitSeq,
422
+ operationCount: pushOps.length,
423
+ tables: pushed.affectedTables,
424
+ });
425
+
426
+ // WS notifications
427
+ if (
428
+ wsConnectionManager &&
429
+ pushed.response.ok === true &&
430
+ pushed.response.status === 'applied' &&
431
+ typeof pushed.response.commitSeq === 'number'
432
+ ) {
433
+ const scopeKeys = applyPartitionToScopeKeys(
434
+ partitionId,
435
+ pushed.scopeKeys
436
+ );
437
+ if (scopeKeys.length > 0) {
438
+ wsConnectionManager.notifyScopeKeys(
439
+ scopeKeys,
440
+ pushed.response.commitSeq,
441
+ {
442
+ excludeClientIds: [clientId],
443
+ changes: pushed.emittedChanges,
444
+ }
445
+ );
446
+
447
+ if (realtimeBroadcaster) {
448
+ realtimeBroadcaster
449
+ .publish({
450
+ type: 'commit',
451
+ commitSeq: pushed.response.commitSeq,
452
+ partitionId,
453
+ scopeKeys,
454
+ sourceInstanceId: instanceId,
455
+ })
456
+ .catch(() => {});
457
+ }
458
+ }
459
+ }
460
+
461
+ pushResponse = pushed.response;
462
+ }
463
+
464
+ // --- Pull phase ---
465
+ if (body.pull) {
466
+ if (body.pull.subscriptions.length > maxSubscriptionsPerPull) {
467
+ return c.json(
468
+ {
469
+ error: 'INVALID_REQUEST',
470
+ message: `Too many subscriptions (max ${maxSubscriptionsPerPull})`,
471
+ },
472
+ 400
473
+ );
474
+ }
475
+
476
+ const seenSubscriptionIds = new Set<string>();
477
+ for (const sub of body.pull.subscriptions) {
478
+ const id = sub.id;
479
+ if (seenSubscriptionIds.has(id)) {
480
+ return c.json(
481
+ {
482
+ error: 'INVALID_REQUEST',
483
+ message: `Duplicate subscription id: ${id}`,
484
+ },
485
+ 400
486
+ );
487
+ }
488
+ seenSubscriptionIds.add(id);
489
+ }
490
+
491
+ const request = {
492
+ clientId,
493
+ limitCommits: clampInt(
494
+ body.pull.limitCommits ?? 50,
495
+ 1,
496
+ maxPullLimitCommits
497
+ ),
498
+ limitSnapshotRows: clampInt(
499
+ body.pull.limitSnapshotRows ?? 1000,
500
+ 1,
501
+ maxPullLimitSnapshotRows
502
+ ),
503
+ maxSnapshotPages: clampInt(
504
+ body.pull.maxSnapshotPages ?? 1,
505
+ 1,
506
+ maxPullMaxSnapshotPages
507
+ ),
508
+ dedupeRows: body.pull.dedupeRows === true,
509
+ subscriptions: body.pull.subscriptions.map((sub) => ({
510
+ id: sub.id,
511
+ shape: sub.shape,
512
+ scopes: (sub.scopes ?? {}) as Record<string, string | string[]>,
513
+ params: sub.params as Record<string, unknown>,
514
+ cursor: Math.max(-1, sub.cursor),
515
+ bootstrapState: sub.bootstrapState ?? null,
516
+ })),
517
+ };
518
+
519
+ const timer = createSyncTimer();
520
+
521
+ let pullResult: PullResult;
522
+ try {
523
+ pullResult = await pull({
524
+ db: options.db,
525
+ dialect: options.dialect,
526
+ shapes: handlerRegistry,
527
+ actorId: auth.actorId,
528
+ partitionId,
529
+ request,
530
+ chunkStorage: options.chunkStorage,
531
+ });
532
+ } catch (err) {
533
+ if (err instanceof InvalidSubscriptionScopeError) {
534
+ return c.json(
535
+ { error: 'INVALID_SUBSCRIPTION', message: err.message },
536
+ 400
537
+ );
538
+ }
539
+ throw err;
540
+ }
541
+
542
+ // Fire-and-forget bookkeeping
543
+ recordClientCursor(options.db, options.dialect, {
544
+ partitionId,
545
+ clientId,
546
+ actorId: auth.actorId,
547
+ cursor: pullResult.clientCursor,
548
+ effectiveScopes: pullResult.effectiveScopes,
549
+ }).catch(() => {});
550
+
551
+ wsConnectionManager?.updateClientScopeKeys(
552
+ clientId,
553
+ applyPartitionToScopeKeys(
554
+ partitionId,
555
+ scopeValuesToScopeKeys(pullResult.effectiveScopes)
556
+ )
557
+ );
558
+
559
+ const pullDurationMs = timer();
560
+
561
+ logSyncEvent({
562
+ event: 'sync.pull',
563
+ userId: auth.actorId,
564
+ durationMs: pullDurationMs,
565
+ subscriptionCount: pullResult.response.subscriptions.length,
566
+ clientCursor: pullResult.clientCursor,
567
+ });
568
+
569
+ recordRequestEvent({
570
+ eventType: 'pull',
571
+ actorId: auth.actorId,
572
+ clientId,
573
+ transportPath: readTransportPath(c),
574
+ statusCode: 200,
575
+ outcome: 'applied',
576
+ durationMs: pullDurationMs,
577
+ });
578
+
579
+ pullResponse = pullResult.response;
580
+ }
581
+
582
+ return c.json(
583
+ {
584
+ ok: true as const,
585
+ ...(pushResponse ? { push: pushResponse } : {}),
586
+ ...(pullResponse ? { pull: pullResponse } : {}),
587
+ },
588
+ 200
589
+ );
590
+ }
591
+ );
592
+
593
+ // -------------------------------------------------------------------------
594
+ // GET /snapshot-chunks/:chunkId
595
+ // -------------------------------------------------------------------------
596
+
597
+ routes.get(
598
+ '/snapshot-chunks/:chunkId',
599
+ describeRoute({
600
+ tags: ['sync'],
601
+ summary: 'Download snapshot chunk',
602
+ description: 'Download an encoded bootstrap snapshot chunk',
603
+ responses: {
604
+ 200: {
605
+ description: 'Snapshot chunk data (gzip-compressed NDJSON)',
606
+ },
607
+ 304: {
608
+ description: 'Not modified (cached)',
609
+ },
610
+ 401: {
611
+ description: 'Unauthenticated',
612
+ content: {
613
+ 'application/json': { schema: resolver(ErrorResponseSchema) },
614
+ },
615
+ },
616
+ 403: {
617
+ description: 'Forbidden',
618
+ content: {
619
+ 'application/json': { schema: resolver(ErrorResponseSchema) },
620
+ },
621
+ },
622
+ 404: {
623
+ description: 'Not found',
624
+ content: {
625
+ 'application/json': { schema: resolver(ErrorResponseSchema) },
626
+ },
627
+ },
628
+ },
629
+ }),
630
+ zValidator('param', snapshotChunkParamsSchema),
631
+ async (c) => {
632
+ const auth = await options.authenticate(c);
633
+ if (!auth) return c.json({ error: 'UNAUTHENTICATED' }, 401);
634
+ const partitionId = auth.partitionId ?? 'default';
635
+
636
+ const { chunkId } = c.req.valid('param');
637
+
638
+ const chunk = await readSnapshotChunk(options.db, chunkId, {
639
+ chunkStorage: options.chunkStorage,
640
+ });
641
+ if (!chunk) return c.json({ error: 'NOT_FOUND' }, 404);
642
+ if (chunk.partitionId !== partitionId) {
643
+ return c.json({ error: 'FORBIDDEN' }, 403);
644
+ }
645
+
646
+ const nowIso = new Date().toISOString();
647
+ if (chunk.expiresAt <= nowIso) {
648
+ return c.json({ error: 'NOT_FOUND' }, 404);
649
+ }
650
+
651
+ // Note: Snapshot chunks are created during authorized pull requests
652
+ // and have opaque IDs that expire. Additional authorization is handled
653
+ // at the pull layer via shape-level resolveScopes.
654
+
655
+ const etag = `"sha256:${chunk.sha256}"`;
656
+ const ifNoneMatch = c.req.header('if-none-match');
657
+ if (ifNoneMatch && ifNoneMatch === etag) {
658
+ return new Response(null, {
659
+ status: 304,
660
+ headers: {
661
+ ETag: etag,
662
+ 'Cache-Control': 'private, max-age=0',
663
+ Vary: 'Authorization',
664
+ },
665
+ });
666
+ }
667
+
668
+ return new Response(chunk.body as BodyInit, {
669
+ status: 200,
670
+ headers: {
671
+ 'Content-Type': 'application/x-ndjson; charset=utf-8',
672
+ 'Content-Encoding': 'gzip',
673
+ 'Content-Length': String(chunk.body.length),
674
+ ETag: etag,
675
+ 'Cache-Control': 'private, max-age=0',
676
+ Vary: 'Authorization',
677
+ 'X-Sync-Chunk-Id': chunk.chunkId,
678
+ 'X-Sync-Chunk-Sha256': chunk.sha256,
679
+ 'X-Sync-Chunk-Encoding': chunk.encoding,
680
+ 'X-Sync-Chunk-Compression': chunk.compression,
681
+ },
682
+ });
683
+ }
684
+ );
685
+
686
+ // -------------------------------------------------------------------------
687
+ // GET /realtime (optional WebSocket wake-ups)
688
+ // -------------------------------------------------------------------------
689
+
690
+ if (wsConnectionManager && websocketConfig?.enabled) {
691
+ routes.get('/realtime', async (c) => {
692
+ const auth = await options.authenticate(c);
693
+ if (!auth) return c.json({ error: 'UNAUTHENTICATED' }, 401);
694
+ const partitionId = auth.partitionId ?? 'default';
695
+
696
+ const clientId = c.req.query('clientId');
697
+ if (!clientId || typeof clientId !== 'string') {
698
+ return c.json(
699
+ {
700
+ error: 'INVALID_REQUEST',
701
+ message: 'clientId query param is required',
702
+ },
703
+ 400
704
+ );
705
+ }
706
+ const realtimeTransportPath = readTransportPath(
707
+ c,
708
+ c.req.query('transportPath')
709
+ );
710
+
711
+ // Load last-known effective scopes for this client (best-effort).
712
+ // Keeps /realtime lightweight and avoids sending large subscription payloads over the URL.
713
+ let initialScopeKeys: string[] = [];
714
+ try {
715
+ const cursorsQ = options.db.selectFrom(
716
+ 'sync_client_cursors'
717
+ ) as SelectQueryBuilder<
718
+ DB,
719
+ 'sync_client_cursors',
720
+ // biome-ignore lint/complexity/noBannedTypes: Kysely uses `{}` as the initial "no selected columns yet" marker.
721
+ {}
722
+ >;
723
+
724
+ const row = await cursorsQ
725
+ .selectAll()
726
+ .where(sql<SqlBool>`partition_id = ${partitionId}`)
727
+ .where(sql<SqlBool>`client_id = ${clientId}`)
728
+ .executeTakeFirst();
729
+
730
+ if (row && row.actor_id !== auth.actorId) {
731
+ return c.json({ error: 'FORBIDDEN' }, 403);
732
+ }
733
+
734
+ const raw = row?.effective_scopes;
735
+ let parsed: unknown = raw;
736
+ if (typeof raw === 'string') {
737
+ try {
738
+ parsed = JSON.parse(raw);
739
+ } catch {
740
+ parsed = null;
741
+ }
742
+ }
743
+
744
+ initialScopeKeys = applyPartitionToScopeKeys(
745
+ partitionId,
746
+ scopeValuesToScopeKeys(parsed)
747
+ );
748
+ } catch {
749
+ // ignore; realtime is best-effort
750
+ }
751
+
752
+ const maxConnectionsTotal = websocketConfig.maxConnectionsTotal ?? 5000;
753
+ const maxConnectionsPerClient =
754
+ websocketConfig.maxConnectionsPerClient ?? 3;
755
+
756
+ if (
757
+ maxConnectionsTotal > 0 &&
758
+ wsConnectionManager.getTotalConnections() >= maxConnectionsTotal
759
+ ) {
760
+ logSyncEvent({
761
+ event: 'sync.realtime.rejected',
762
+ userId: auth.actorId,
763
+ reason: 'max_total',
764
+ });
765
+ return c.json({ error: 'WEBSOCKET_CONNECTION_LIMIT_TOTAL' }, 429);
766
+ }
767
+
768
+ if (
769
+ maxConnectionsPerClient > 0 &&
770
+ wsConnectionManager.getConnectionCount(clientId) >=
771
+ maxConnectionsPerClient
772
+ ) {
773
+ logSyncEvent({
774
+ event: 'sync.realtime.rejected',
775
+ userId: auth.actorId,
776
+ reason: 'max_per_client',
777
+ });
778
+ return c.json({ error: 'WEBSOCKET_CONNECTION_LIMIT_CLIENT' }, 429);
779
+ }
780
+
781
+ logSyncEvent({ event: 'sync.realtime.connect', userId: auth.actorId });
782
+
783
+ let unregister: (() => void) | null = null;
784
+ let connRef: ReturnType<typeof createWebSocketConnection> | null = null;
785
+
786
+ const upgradeWebSocket = websocketConfig.upgradeWebSocket;
787
+ if (!upgradeWebSocket) {
788
+ return c.json({ error: 'WEBSOCKET_NOT_CONFIGURED' }, 500);
789
+ }
790
+
791
+ return upgradeWebSocket(c, {
792
+ onOpen(_evt, ws) {
793
+ const conn = createWebSocketConnection(ws, {
794
+ actorId: auth.actorId,
795
+ clientId,
796
+ transportPath: realtimeTransportPath,
797
+ });
798
+ connRef = conn;
799
+
800
+ unregister = wsConnectionManager.register(conn, initialScopeKeys);
801
+ conn.sendHeartbeat();
802
+ },
803
+ onClose(_evt, _ws) {
804
+ unregister?.();
805
+ unregister = null;
806
+ connRef = null;
807
+ logSyncEvent({
808
+ event: 'sync.realtime.disconnect',
809
+ userId: auth.actorId,
810
+ });
811
+ },
812
+ onError(_evt, _ws) {
813
+ unregister?.();
814
+ unregister = null;
815
+ connRef = null;
816
+ logSyncEvent({
817
+ event: 'sync.realtime.disconnect',
818
+ userId: auth.actorId,
819
+ });
820
+ },
821
+ onMessage(evt, _ws) {
822
+ if (!connRef) return;
823
+ try {
824
+ const raw =
825
+ typeof evt.data === 'string' ? evt.data : String(evt.data);
826
+ const msg = JSON.parse(raw);
827
+ if (!msg || typeof msg !== 'object') return;
828
+
829
+ if (msg.type === 'push') {
830
+ void handleWsPush(
831
+ msg,
832
+ connRef,
833
+ auth.actorId,
834
+ partitionId,
835
+ clientId
836
+ );
837
+ return;
838
+ }
839
+
840
+ if (msg.type !== 'presence' || !msg.scopeKey) return;
841
+
842
+ const scopeKey = normalizeScopeKeyForPartition(
843
+ partitionId,
844
+ String(msg.scopeKey)
845
+ );
846
+ if (!scopeKey) return;
847
+
848
+ switch (msg.action) {
849
+ case 'join':
850
+ if (
851
+ !wsConnectionManager.joinPresence(
852
+ clientId,
853
+ scopeKey,
854
+ msg.metadata
855
+ )
856
+ ) {
857
+ logSyncEvent({
858
+ event: 'sync.realtime.presence.rejected',
859
+ userId: auth.actorId,
860
+ reason: 'scope_not_authorized',
861
+ scopeKey,
862
+ });
863
+ return;
864
+ }
865
+ // Send presence snapshot back to the joining client
866
+ {
867
+ const entries = wsConnectionManager.getPresence(scopeKey);
868
+ connRef.sendPresence({
869
+ action: 'snapshot',
870
+ scopeKey,
871
+ entries,
872
+ });
873
+ }
874
+ break;
875
+ case 'leave':
876
+ wsConnectionManager.leavePresence(clientId, scopeKey);
877
+ break;
878
+ case 'update':
879
+ if (
880
+ !wsConnectionManager.updatePresenceMetadata(
881
+ clientId,
882
+ scopeKey,
883
+ msg.metadata ?? {}
884
+ ) &&
885
+ !wsConnectionManager.isClientSubscribedToScopeKey(
886
+ clientId,
887
+ scopeKey
888
+ )
889
+ ) {
890
+ logSyncEvent({
891
+ event: 'sync.realtime.presence.rejected',
892
+ userId: auth.actorId,
893
+ reason: 'scope_not_authorized',
894
+ scopeKey,
895
+ });
896
+ }
897
+ break;
898
+ }
899
+ } catch {
900
+ // Ignore malformed messages
901
+ }
902
+ },
903
+ });
904
+ });
905
+ }
906
+
907
+ return routes;
908
+
909
+ async function handleRealtimeEvent(event: SyncRealtimeEvent): Promise<void> {
910
+ if (!wsConnectionManager) return;
911
+ if (event.type !== 'commit') return;
912
+ if (event.sourceInstanceId && event.sourceInstanceId === instanceId) return;
913
+
914
+ const commitSeq = event.commitSeq;
915
+ const partitionId = event.partitionId ?? 'default';
916
+ const scopeKeys =
917
+ event.scopeKeys && event.scopeKeys.length > 0
918
+ ? event.scopeKeys
919
+ : await readCommitScopeKeys(options.db, commitSeq, partitionId);
920
+
921
+ if (scopeKeys.length === 0) return;
922
+ wsConnectionManager.notifyScopeKeys(scopeKeys, commitSeq);
923
+ }
924
+
925
+ async function handleWsPush(
926
+ msg: Record<string, unknown>,
927
+ conn: WebSocketConnection,
928
+ actorId: string,
929
+ partitionId: string,
930
+ clientId: string
931
+ ): Promise<void> {
932
+ const requestId = typeof msg.requestId === 'string' ? msg.requestId : '';
933
+ if (!requestId) return;
934
+
935
+ try {
936
+ // Validate the push payload
937
+ const parsed = SyncPushRequestSchema.omit({ clientId: true }).safeParse(
938
+ msg
939
+ );
940
+ if (!parsed.success) {
941
+ conn.sendPushResponse({
942
+ requestId,
943
+ ok: false,
944
+ status: 'rejected',
945
+ results: [
946
+ { opIndex: 0, status: 'error', error: 'Invalid push payload' },
947
+ ],
948
+ });
949
+ return;
950
+ }
951
+
952
+ const pushOps = parsed.data.operations ?? [];
953
+ if (pushOps.length > maxOperationsPerPush) {
954
+ conn.sendPushResponse({
955
+ requestId,
956
+ ok: false,
957
+ status: 'rejected',
958
+ results: [
959
+ {
960
+ opIndex: 0,
961
+ status: 'error',
962
+ error: `Maximum ${maxOperationsPerPush} operations per push`,
963
+ },
964
+ ],
965
+ });
966
+ return;
967
+ }
968
+
969
+ const timer = createSyncTimer();
970
+
971
+ const pushed = await pushCommit({
972
+ db: options.db,
973
+ dialect: options.dialect,
974
+ shapes: handlerRegistry,
975
+ actorId,
976
+ partitionId,
977
+ request: {
978
+ clientId,
979
+ clientCommitId: parsed.data.clientCommitId,
980
+ operations: parsed.data.operations,
981
+ schemaVersion: parsed.data.schemaVersion,
982
+ },
983
+ });
984
+
985
+ const pushDurationMs = timer();
986
+
987
+ logSyncEvent({
988
+ event: 'sync.push',
989
+ userId: actorId,
990
+ durationMs: pushDurationMs,
991
+ operationCount: pushOps.length,
992
+ status: pushed.response.status,
993
+ commitSeq: pushed.response.commitSeq,
994
+ });
995
+
996
+ recordRequestEvent({
997
+ eventType: 'push',
998
+ actorId,
999
+ clientId,
1000
+ transportPath: conn.transportPath,
1001
+ statusCode: 200,
1002
+ outcome: pushed.response.status,
1003
+ durationMs: pushDurationMs,
1004
+ commitSeq: pushed.response.commitSeq,
1005
+ operationCount: pushOps.length,
1006
+ tables: pushed.affectedTables,
1007
+ });
1008
+
1009
+ // WS notifications to other clients
1010
+ if (
1011
+ wsConnectionManager &&
1012
+ pushed.response.ok === true &&
1013
+ pushed.response.status === 'applied' &&
1014
+ typeof pushed.response.commitSeq === 'number'
1015
+ ) {
1016
+ const scopeKeys = applyPartitionToScopeKeys(
1017
+ partitionId,
1018
+ pushed.scopeKeys
1019
+ );
1020
+ if (scopeKeys.length > 0) {
1021
+ wsConnectionManager.notifyScopeKeys(
1022
+ scopeKeys,
1023
+ pushed.response.commitSeq,
1024
+ {
1025
+ excludeClientIds: [clientId],
1026
+ changes: pushed.emittedChanges,
1027
+ }
1028
+ );
1029
+
1030
+ if (realtimeBroadcaster) {
1031
+ realtimeBroadcaster
1032
+ .publish({
1033
+ type: 'commit',
1034
+ commitSeq: pushed.response.commitSeq,
1035
+ partitionId,
1036
+ scopeKeys,
1037
+ sourceInstanceId: instanceId,
1038
+ })
1039
+ .catch(() => {});
1040
+ }
1041
+ }
1042
+ }
1043
+
1044
+ conn.sendPushResponse({
1045
+ requestId,
1046
+ ok: pushed.response.ok,
1047
+ status: pushed.response.status,
1048
+ commitSeq: pushed.response.commitSeq,
1049
+ results: pushed.response.results,
1050
+ });
1051
+ } catch (err) {
1052
+ const message =
1053
+ err instanceof Error ? err.message : 'Internal server error';
1054
+ conn.sendPushResponse({
1055
+ requestId,
1056
+ ok: false,
1057
+ status: 'rejected',
1058
+ results: [{ opIndex: 0, status: 'error', error: message }],
1059
+ });
1060
+ }
1061
+ }
1062
+ }
1063
+
1064
+ export function getSyncWebSocketConnectionManager(
1065
+ routes: Hono
1066
+ ): WebSocketConnectionManager | undefined {
1067
+ return wsConnectionManagerMap.get(routes);
1068
+ }
1069
+
1070
+ export function getSyncRealtimeUnsubscribe(
1071
+ routes: Hono
1072
+ ): (() => void) | undefined {
1073
+ return realtimeUnsubscribeMap.get(routes);
1074
+ }
1075
+
1076
+ function clampInt(value: number, min: number, max: number): number {
1077
+ return Math.max(min, Math.min(max, value));
1078
+ }
1079
+
1080
+ function readTransportPath(
1081
+ c: Context,
1082
+ queryValue?: string | null
1083
+ ): 'direct' | 'relay' {
1084
+ if (queryValue === 'relay' || queryValue === 'direct') {
1085
+ return queryValue;
1086
+ }
1087
+
1088
+ const headerValue = c.req.header('x-syncular-transport-path');
1089
+ if (headerValue === 'relay' || headerValue === 'direct') {
1090
+ return headerValue;
1091
+ }
1092
+
1093
+ return 'direct';
1094
+ }
1095
+
1096
+ function scopeValuesToScopeKeys(scopes: unknown): string[] {
1097
+ if (!scopes || typeof scopes !== 'object') return [];
1098
+ const scopeKeys = new Set<string>();
1099
+
1100
+ for (const [key, value] of Object.entries(scopes)) {
1101
+ if (!value) continue;
1102
+ const prefix = key.replace(/_id$/, '');
1103
+
1104
+ if (Array.isArray(value)) {
1105
+ for (const v of value) {
1106
+ if (typeof v !== 'string') continue;
1107
+ if (!v) continue;
1108
+ scopeKeys.add(`${prefix}:${v}`);
1109
+ }
1110
+ continue;
1111
+ }
1112
+
1113
+ if (typeof value === 'string') {
1114
+ if (!value) continue;
1115
+ scopeKeys.add(`${prefix}:${value}`);
1116
+ continue;
1117
+ }
1118
+
1119
+ // Best-effort: stringify scalars.
1120
+ if (typeof value === 'number' || typeof value === 'bigint') {
1121
+ scopeKeys.add(`${prefix}:${String(value)}`);
1122
+ }
1123
+ }
1124
+
1125
+ return Array.from(scopeKeys);
1126
+ }
1127
+
1128
+ function partitionScopeKey(partitionId: string, scopeKey: string): string {
1129
+ return `${partitionId}::${scopeKey}`;
1130
+ }
1131
+
1132
+ function applyPartitionToScopeKeys(
1133
+ partitionId: string,
1134
+ scopeKeys: readonly string[]
1135
+ ): string[] {
1136
+ const prefixed = new Set<string>();
1137
+ for (const scopeKey of scopeKeys) {
1138
+ if (!scopeKey) continue;
1139
+ if (scopeKey.startsWith(`${partitionId}::`)) {
1140
+ prefixed.add(scopeKey);
1141
+ continue;
1142
+ }
1143
+ prefixed.add(partitionScopeKey(partitionId, scopeKey));
1144
+ }
1145
+ return Array.from(prefixed);
1146
+ }
1147
+
1148
+ function normalizeScopeKeyForPartition(
1149
+ partitionId: string,
1150
+ scopeKey: string
1151
+ ): string {
1152
+ if (scopeKey.startsWith(`${partitionId}::`)) return scopeKey;
1153
+ if (scopeKey.includes('::')) return '';
1154
+ return partitionScopeKey(partitionId, scopeKey);
1155
+ }
1156
+
1157
+ async function readCommitScopeKeys<DB extends SyncCoreDb>(
1158
+ db: Kysely<DB>,
1159
+ commitSeq: number,
1160
+ partitionId: string
1161
+ ): Promise<string[]> {
1162
+ // Read scopes from the JSONB column and convert to scope strings
1163
+ const rowsResult = await sql<{ scopes: unknown }>`
1164
+ select scopes
1165
+ from ${sql.table('sync_changes')}
1166
+ where commit_seq = ${commitSeq}
1167
+ and partition_id = ${partitionId}
1168
+ `.execute(db);
1169
+ const rows = rowsResult.rows;
1170
+
1171
+ const scopeKeys = new Set<string>();
1172
+
1173
+ for (const row of rows) {
1174
+ const scopes =
1175
+ typeof row.scopes === 'string' ? JSON.parse(row.scopes) : row.scopes;
1176
+
1177
+ for (const k of applyPartitionToScopeKeys(
1178
+ partitionId,
1179
+ scopeValuesToScopeKeys(scopes)
1180
+ )) {
1181
+ scopeKeys.add(k);
1182
+ }
1183
+ }
1184
+
1185
+ return Array.from(scopeKeys);
1186
+ }