@syncular/server 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +773 -13
- package/dist/admin.d.ts +136 -0
- package/dist/admin.js +168 -0
- package/dist/blob-handlers.d.ts +69 -0
- package/dist/blob-handlers.js +245 -0
- package/dist/blob-store.d.ts +111 -0
- package/dist/blob-store.js +0 -0
- package/dist/content-encoding.d.ts +27 -0
- package/dist/content-encoding.js +69 -0
- package/dist/context.d.ts +151 -0
- package/dist/context.js +21 -0
- package/dist/crdt-merger.d.ts +28 -0
- package/dist/crdt-merger.js +11 -0
- package/dist/d1-storage.d.ts +50 -0
- package/dist/d1-storage.js +0 -0
- package/dist/errors.d.ts +35 -9
- package/dist/errors.js +220 -11
- package/dist/events-ring.d.ts +55 -0
- package/dist/events-ring.js +96 -0
- package/dist/events.d.ts +231 -0
- package/dist/events.js +22 -0
- package/dist/frame-bytes.d.ts +20 -0
- package/dist/frame-bytes.js +75 -0
- package/dist/handler.d.ts +9 -0
- package/dist/handler.js +490 -0
- package/dist/index.d.ts +42 -33
- package/dist/index.js +45 -27
- package/dist/lease-store.d.ts +50 -0
- package/dist/lease-store.js +0 -0
- package/dist/pg-executor-pglite.d.ts +27 -0
- package/dist/pg-executor-pglite.js +33 -0
- package/dist/pg-executor.d.ts +67 -0
- package/dist/pg-executor.js +56 -0
- package/dist/postgres-fanout.d.ts +83 -0
- package/dist/postgres-fanout.js +71 -0
- package/dist/postgres-storage.d.ts +69 -0
- package/dist/postgres-storage.js +766 -0
- package/dist/prune.d.ts +24 -39
- package/dist/prune.js +44 -146
- package/dist/pull.d.ts +42 -67
- package/dist/pull.js +387 -1187
- package/dist/push.d.ts +31 -66
- package/dist/push.js +406 -655
- package/dist/realtime.d.ts +197 -0
- package/dist/realtime.js +848 -0
- package/dist/relational-rows.d.ts +229 -0
- package/dist/relational-rows.js +514 -0
- package/dist/s3-blob-store.d.ts +142 -0
- package/dist/s3-blob-store.js +458 -0
- package/dist/s3-segment-store.d.ts +95 -0
- package/dist/s3-segment-store.js +380 -0
- package/dist/schema.d.ts +82 -286
- package/dist/schema.js +133 -9
- package/dist/scopes.d.ts +48 -0
- package/dist/scopes.js +98 -0
- package/dist/segment-download.d.ts +14 -0
- package/dist/segment-download.js +125 -0
- package/dist/segment-store.d.ts +81 -0
- package/dist/segment-store.js +63 -0
- package/dist/signed-url.d.ts +139 -0
- package/dist/signed-url.js +143 -0
- package/dist/sigv4.d.ts +97 -0
- package/dist/sigv4.js +161 -0
- package/dist/sqlite-blob-store.d.ts +22 -0
- package/dist/sqlite-blob-store.js +86 -0
- package/dist/sqlite-dialect.d.ts +101 -0
- package/dist/sqlite-dialect.js +250 -0
- package/dist/sqlite-image.d.ts +27 -0
- package/dist/sqlite-image.js +83 -0
- package/dist/sqlite-lease-store.d.ts +21 -0
- package/dist/sqlite-lease-store.js +94 -0
- package/dist/sqlite-segment-store.d.ts +25 -0
- package/dist/sqlite-segment-store.js +115 -0
- package/dist/sqlite-storage.d.ts +47 -0
- package/dist/sqlite-storage.js +477 -0
- package/dist/storage.d.ts +252 -0
- package/dist/storage.js +1 -0
- package/dist/validate.d.ts +83 -0
- package/dist/validate.js +44 -0
- package/package.json +17 -325
- package/src/admin.ts +312 -0
- package/src/blob-handlers.ts +340 -0
- package/src/blob-store.ts +0 -0
- package/src/content-encoding.ts +94 -0
- package/src/context.ts +176 -0
- package/src/crdt-merger.ts +33 -0
- package/src/d1-storage.ts +0 -0
- package/src/errors.ts +241 -19
- package/src/events-ring.ts +116 -0
- package/src/events.ts +291 -0
- package/src/frame-bytes.ts +94 -0
- package/src/handler.ts +678 -0
- package/src/index.ts +44 -27
- package/src/lease-store.ts +0 -0
- package/src/pg-executor-pglite.ts +63 -0
- package/src/pg-executor.ts +87 -0
- package/src/postgres-fanout.ts +143 -0
- package/src/postgres-storage.ts +1182 -0
- package/src/prune.ts +63 -289
- package/src/pull.ts +505 -1805
- package/src/push.ts +623 -1006
- package/src/realtime.ts +1082 -0
- package/src/relational-rows.ts +633 -0
- package/src/s3-blob-store.ts +616 -0
- package/src/s3-segment-store.ts +519 -0
- package/src/schema.ts +242 -281
- package/src/scopes.ts +127 -0
- package/src/segment-download.ts +169 -0
- package/src/segment-store.ts +154 -0
- package/src/signed-url.ts +308 -0
- package/src/sigv4.ts +267 -0
- package/src/sqlite-blob-store.ts +125 -0
- package/src/sqlite-dialect.ts +337 -0
- package/src/sqlite-image.ts +124 -0
- package/src/sqlite-lease-store.ts +144 -0
- package/src/sqlite-segment-store.ts +204 -0
- package/src/sqlite-storage.ts +792 -0
- package/src/storage.ts +316 -0
- package/src/validate.ts +123 -0
- package/dist/auth-leases.d.ts +0 -85
- package/dist/auth-leases.d.ts.map +0 -1
- package/dist/auth-leases.js +0 -368
- package/dist/auth-leases.js.map +0 -1
- package/dist/better-sqlite3.d.ts +0 -22
- package/dist/better-sqlite3.d.ts.map +0 -1
- package/dist/better-sqlite3.js +0 -16
- package/dist/better-sqlite3.js.map +0 -1
- package/dist/blobs/access.d.ts +0 -35
- package/dist/blobs/access.d.ts.map +0 -1
- package/dist/blobs/access.js +0 -149
- package/dist/blobs/access.js.map +0 -1
- package/dist/blobs/adapters/database.d.ts +0 -100
- package/dist/blobs/adapters/database.d.ts.map +0 -1
- package/dist/blobs/adapters/database.js +0 -247
- package/dist/blobs/adapters/database.js.map +0 -1
- package/dist/blobs/index.d.ts +0 -9
- package/dist/blobs/index.d.ts.map +0 -1
- package/dist/blobs/index.js +0 -9
- package/dist/blobs/index.js.map +0 -1
- package/dist/blobs/manager.d.ts +0 -220
- package/dist/blobs/manager.d.ts.map +0 -1
- package/dist/blobs/manager.js +0 -625
- package/dist/blobs/manager.js.map +0 -1
- package/dist/blobs/migrate.d.ts +0 -27
- package/dist/blobs/migrate.d.ts.map +0 -1
- package/dist/blobs/migrate.js +0 -127
- package/dist/blobs/migrate.js.map +0 -1
- package/dist/blobs/types.d.ts +0 -58
- package/dist/blobs/types.d.ts.map +0 -1
- package/dist/blobs/types.js +0 -5
- package/dist/blobs/types.js.map +0 -1
- package/dist/bun-sqlite.d.ts +0 -19
- package/dist/bun-sqlite.d.ts.map +0 -1
- package/dist/bun-sqlite.js +0 -19
- package/dist/bun-sqlite.js.map +0 -1
- package/dist/clients.d.ts +0 -15
- package/dist/clients.d.ts.map +0 -1
- package/dist/clients.js +0 -7
- package/dist/clients.js.map +0 -1
- package/dist/cloudflare/durable-object.d.ts +0 -93
- package/dist/cloudflare/durable-object.d.ts.map +0 -1
- package/dist/cloudflare/durable-object.js +0 -210
- package/dist/cloudflare/durable-object.js.map +0 -1
- package/dist/cloudflare/index.d.ts +0 -22
- package/dist/cloudflare/index.d.ts.map +0 -1
- package/dist/cloudflare/index.js +0 -22
- package/dist/cloudflare/index.js.map +0 -1
- package/dist/cloudflare/r2.d.ts +0 -180
- package/dist/cloudflare/r2.d.ts.map +0 -1
- package/dist/cloudflare/r2.js +0 -258
- package/dist/cloudflare/r2.js.map +0 -1
- package/dist/cloudflare/scope-cache.d.ts +0 -54
- package/dist/cloudflare/scope-cache.d.ts.map +0 -1
- package/dist/cloudflare/scope-cache.js +0 -223
- package/dist/cloudflare/scope-cache.js.map +0 -1
- package/dist/cloudflare/sentry.d.ts +0 -47
- package/dist/cloudflare/sentry.d.ts.map +0 -1
- package/dist/cloudflare/sentry.js +0 -163
- package/dist/cloudflare/sentry.js.map +0 -1
- package/dist/cloudflare/worker.d.ts +0 -46
- package/dist/cloudflare/worker.d.ts.map +0 -1
- package/dist/cloudflare/worker.js +0 -63
- package/dist/cloudflare/worker.js.map +0 -1
- package/dist/commit-integrity.d.ts +0 -38
- package/dist/commit-integrity.d.ts.map +0 -1
- package/dist/commit-integrity.js +0 -260
- package/dist/commit-integrity.js.map +0 -1
- package/dist/compaction.d.ts +0 -27
- package/dist/compaction.d.ts.map +0 -1
- package/dist/compaction.js +0 -49
- package/dist/compaction.js.map +0 -1
- package/dist/crdt-yjs/index.d.ts +0 -99
- package/dist/crdt-yjs/index.d.ts.map +0 -1
- package/dist/crdt-yjs/index.js +0 -629
- package/dist/crdt-yjs/index.js.map +0 -1
- package/dist/d1.d.ts +0 -13
- package/dist/d1.d.ts.map +0 -1
- package/dist/d1.js +0 -14
- package/dist/d1.js.map +0 -1
- package/dist/dialect/base.d.ts +0 -93
- package/dist/dialect/base.d.ts.map +0 -1
- package/dist/dialect/base.js +0 -181
- package/dist/dialect/base.js.map +0 -1
- package/dist/dialect/helpers.d.ts +0 -16
- package/dist/dialect/helpers.d.ts.map +0 -1
- package/dist/dialect/helpers.js +0 -83
- package/dist/dialect/helpers.js.map +0 -1
- package/dist/dialect/index.d.ts +0 -7
- package/dist/dialect/index.d.ts.map +0 -1
- package/dist/dialect/index.js +0 -7
- package/dist/dialect/index.js.map +0 -1
- package/dist/dialect/types.d.ts +0 -187
- package/dist/dialect/types.d.ts.map +0 -1
- package/dist/dialect/types.js +0 -8
- package/dist/dialect/types.js.map +0 -1
- package/dist/encrypted-crdt.d.ts +0 -67
- package/dist/encrypted-crdt.d.ts.map +0 -1
- package/dist/encrypted-crdt.js +0 -426
- package/dist/encrypted-crdt.js.map +0 -1
- package/dist/errors.d.ts.map +0 -1
- package/dist/errors.js.map +0 -1
- package/dist/filesystem/index.d.ts +0 -44
- package/dist/filesystem/index.d.ts.map +0 -1
- package/dist/filesystem/index.js +0 -164
- package/dist/filesystem/index.js.map +0 -1
- package/dist/handlers/collection.d.ts +0 -20
- package/dist/handlers/collection.d.ts.map +0 -1
- package/dist/handlers/collection.js +0 -42
- package/dist/handlers/collection.js.map +0 -1
- package/dist/handlers/create-handler.d.ts +0 -156
- package/dist/handlers/create-handler.d.ts.map +0 -1
- package/dist/handlers/create-handler.js +0 -626
- package/dist/handlers/create-handler.js.map +0 -1
- package/dist/handlers/index.d.ts +0 -4
- package/dist/handlers/index.d.ts.map +0 -1
- package/dist/handlers/index.js +0 -4
- package/dist/handlers/index.js.map +0 -1
- package/dist/handlers/types.d.ts +0 -295
- package/dist/handlers/types.d.ts.map +0 -1
- package/dist/handlers/types.js +0 -2
- package/dist/handlers/types.js.map +0 -1
- package/dist/helpers/conflict.d.ts +0 -52
- package/dist/helpers/conflict.d.ts.map +0 -1
- package/dist/helpers/conflict.js +0 -49
- package/dist/helpers/conflict.js.map +0 -1
- package/dist/helpers/emitted-change.d.ts +0 -56
- package/dist/helpers/emitted-change.d.ts.map +0 -1
- package/dist/helpers/emitted-change.js +0 -46
- package/dist/helpers/emitted-change.js.map +0 -1
- package/dist/helpers/index.d.ts +0 -12
- package/dist/helpers/index.d.ts.map +0 -1
- package/dist/helpers/index.js +0 -12
- package/dist/helpers/index.js.map +0 -1
- package/dist/helpers/paginate.d.ts +0 -49
- package/dist/helpers/paginate.d.ts.map +0 -1
- package/dist/helpers/paginate.js +0 -54
- package/dist/helpers/paginate.js.map +0 -1
- package/dist/helpers/scope-authorization.d.ts +0 -7
- package/dist/helpers/scope-authorization.d.ts.map +0 -1
- package/dist/helpers/scope-authorization.js +0 -19
- package/dist/helpers/scope-authorization.js.map +0 -1
- package/dist/helpers/scope-commit-index.d.ts +0 -12
- package/dist/helpers/scope-commit-index.d.ts.map +0 -1
- package/dist/helpers/scope-commit-index.js +0 -35
- package/dist/helpers/scope-commit-index.js.map +0 -1
- package/dist/helpers/scope-strings.d.ts +0 -74
- package/dist/helpers/scope-strings.d.ts.map +0 -1
- package/dist/helpers/scope-strings.js +0 -82
- package/dist/helpers/scope-strings.js.map +0 -1
- package/dist/hono/api-key-auth.d.ts +0 -49
- package/dist/hono/api-key-auth.d.ts.map +0 -1
- package/dist/hono/api-key-auth.js +0 -108
- package/dist/hono/api-key-auth.js.map +0 -1
- package/dist/hono/audit-redaction.d.ts +0 -20
- package/dist/hono/audit-redaction.d.ts.map +0 -1
- package/dist/hono/audit-redaction.js +0 -85
- package/dist/hono/audit-redaction.js.map +0 -1
- package/dist/hono/blobs.d.ts +0 -75
- package/dist/hono/blobs.d.ts.map +0 -1
- package/dist/hono/blobs.js +0 -586
- package/dist/hono/blobs.js.map +0 -1
- package/dist/hono/console/gateway.d.ts +0 -42
- package/dist/hono/console/gateway.d.ts.map +0 -1
- package/dist/hono/console/gateway.js +0 -2158
- package/dist/hono/console/gateway.js.map +0 -1
- package/dist/hono/console/live-auth.d.ts +0 -7
- package/dist/hono/console/live-auth.d.ts.map +0 -1
- package/dist/hono/console/live-auth.js +0 -39
- package/dist/hono/console/live-auth.js.map +0 -1
- package/dist/hono/console/route-descriptor.d.ts +0 -6
- package/dist/hono/console/route-descriptor.d.ts.map +0 -1
- package/dist/hono/console/route-descriptor.js +0 -16
- package/dist/hono/console/route-descriptor.js.map +0 -1
- package/dist/hono/console/routes/api-keys.d.ts +0 -8
- package/dist/hono/console/routes/api-keys.d.ts.map +0 -1
- package/dist/hono/console/routes/api-keys.js +0 -575
- package/dist/hono/console/routes/api-keys.js.map +0 -1
- package/dist/hono/console/routes/clients.d.ts +0 -8
- package/dist/hono/console/routes/clients.d.ts.map +0 -1
- package/dist/hono/console/routes/clients.js +0 -322
- package/dist/hono/console/routes/clients.js.map +0 -1
- package/dist/hono/console/routes/commits.d.ts +0 -8
- package/dist/hono/console/routes/commits.d.ts.map +0 -1
- package/dist/hono/console/routes/commits.js +0 -882
- package/dist/hono/console/routes/commits.js.map +0 -1
- package/dist/hono/console/routes/context.d.ts +0 -188
- package/dist/hono/console/routes/context.d.ts.map +0 -1
- package/dist/hono/console/routes/context.js +0 -633
- package/dist/hono/console/routes/context.js.map +0 -1
- package/dist/hono/console/routes/events.d.ts +0 -8
- package/dist/hono/console/routes/events.d.ts.map +0 -1
- package/dist/hono/console/routes/events.js +0 -501
- package/dist/hono/console/routes/events.js.map +0 -1
- package/dist/hono/console/routes/maintenance.d.ts +0 -8
- package/dist/hono/console/routes/maintenance.d.ts.map +0 -1
- package/dist/hono/console/routes/maintenance.js +0 -341
- package/dist/hono/console/routes/maintenance.js.map +0 -1
- package/dist/hono/console/routes/shared.d.ts +0 -174
- package/dist/hono/console/routes/shared.d.ts.map +0 -1
- package/dist/hono/console/routes/shared.js +0 -583
- package/dist/hono/console/routes/shared.js.map +0 -1
- package/dist/hono/console/routes/stats.d.ts +0 -8
- package/dist/hono/console/routes/stats.d.ts.map +0 -1
- package/dist/hono/console/routes/stats.js +0 -295
- package/dist/hono/console/routes/stats.js.map +0 -1
- package/dist/hono/console/routes/storage.d.ts +0 -8
- package/dist/hono/console/routes/storage.d.ts.map +0 -1
- package/dist/hono/console/routes/storage.js +0 -121
- package/dist/hono/console/routes/storage.js.map +0 -1
- package/dist/hono/console/routes.d.ts +0 -36
- package/dist/hono/console/routes.d.ts.map +0 -1
- package/dist/hono/console/routes.js +0 -112
- package/dist/hono/console/routes.js.map +0 -1
- package/dist/hono/console/schema-errors.d.ts +0 -2
- package/dist/hono/console/schema-errors.d.ts.map +0 -1
- package/dist/hono/console/schema-errors.js +0 -17
- package/dist/hono/console/schema-errors.js.map +0 -1
- package/dist/hono/console/schemas.d.ts +0 -1515
- package/dist/hono/console/schemas.d.ts.map +0 -1
- package/dist/hono/console/schemas.js +0 -661
- package/dist/hono/console/schemas.js.map +0 -1
- package/dist/hono/console/types.d.ts +0 -213
- package/dist/hono/console/types.d.ts.map +0 -1
- package/dist/hono/console/types.js +0 -2
- package/dist/hono/console/types.js.map +0 -1
- package/dist/hono/console/ui.d.ts +0 -38
- package/dist/hono/console/ui.d.ts.map +0 -1
- package/dist/hono/console/ui.js +0 -43
- package/dist/hono/console/ui.js.map +0 -1
- package/dist/hono/create-server.d.ts +0 -71
- package/dist/hono/create-server.d.ts.map +0 -1
- package/dist/hono/create-server.js +0 -121
- package/dist/hono/create-server.js.map +0 -1
- package/dist/hono/errors.d.ts +0 -13
- package/dist/hono/errors.d.ts.map +0 -1
- package/dist/hono/errors.js +0 -21
- package/dist/hono/errors.js.map +0 -1
- package/dist/hono/index.d.ts +0 -20
- package/dist/hono/index.d.ts.map +0 -1
- package/dist/hono/index.js +0 -31
- package/dist/hono/index.js.map +0 -1
- package/dist/hono/openapi.d.ts +0 -72
- package/dist/hono/openapi.d.ts.map +0 -1
- package/dist/hono/openapi.js +0 -99
- package/dist/hono/openapi.js.map +0 -1
- package/dist/hono/proxy/connection-manager.d.ts +0 -78
- package/dist/hono/proxy/connection-manager.d.ts.map +0 -1
- package/dist/hono/proxy/connection-manager.js +0 -251
- package/dist/hono/proxy/connection-manager.js.map +0 -1
- package/dist/hono/proxy/index.d.ts +0 -8
- package/dist/hono/proxy/index.d.ts.map +0 -1
- package/dist/hono/proxy/index.js +0 -8
- package/dist/hono/proxy/index.js.map +0 -1
- package/dist/hono/proxy/routes.d.ts +0 -86
- package/dist/hono/proxy/routes.d.ts.map +0 -1
- package/dist/hono/proxy/routes.js +0 -183
- package/dist/hono/proxy/routes.js.map +0 -1
- package/dist/hono/rate-limit.d.ts +0 -101
- package/dist/hono/rate-limit.d.ts.map +0 -1
- package/dist/hono/rate-limit.js +0 -184
- package/dist/hono/rate-limit.js.map +0 -1
- package/dist/hono/realtime-sync-packs.d.ts +0 -43
- package/dist/hono/realtime-sync-packs.d.ts.map +0 -1
- package/dist/hono/realtime-sync-packs.js +0 -219
- package/dist/hono/realtime-sync-packs.js.map +0 -1
- package/dist/hono/routes/audit.d.ts +0 -12
- package/dist/hono/routes/audit.d.ts.map +0 -1
- package/dist/hono/routes/audit.js +0 -385
- package/dist/hono/routes/audit.js.map +0 -1
- package/dist/hono/routes/auth-leases.d.ts +0 -8
- package/dist/hono/routes/auth-leases.d.ts.map +0 -1
- package/dist/hono/routes/auth-leases.js +0 -88
- package/dist/hono/routes/auth-leases.js.map +0 -1
- package/dist/hono/routes/combined.d.ts +0 -8
- package/dist/hono/routes/combined.d.ts.map +0 -1
- package/dist/hono/routes/combined.js +0 -392
- package/dist/hono/routes/combined.js.map +0 -1
- package/dist/hono/routes/context.d.ts +0 -314
- package/dist/hono/routes/context.d.ts.map +0 -1
- package/dist/hono/routes/context.js +0 -1147
- package/dist/hono/routes/context.js.map +0 -1
- package/dist/hono/routes/health.d.ts +0 -8
- package/dist/hono/routes/health.d.ts.map +0 -1
- package/dist/hono/routes/health.js +0 -16
- package/dist/hono/routes/health.js.map +0 -1
- package/dist/hono/routes/realtime.d.ts +0 -9
- package/dist/hono/routes/realtime.d.ts.map +0 -1
- package/dist/hono/routes/realtime.js +0 -626
- package/dist/hono/routes/realtime.js.map +0 -1
- package/dist/hono/routes/shared.d.ts +0 -689
- package/dist/hono/routes/shared.d.ts.map +0 -1
- package/dist/hono/routes/shared.js +0 -972
- package/dist/hono/routes/shared.js.map +0 -1
- package/dist/hono/routes/snapshots.d.ts +0 -10
- package/dist/hono/routes/snapshots.d.ts.map +0 -1
- package/dist/hono/routes/snapshots.js +0 -281
- package/dist/hono/routes/snapshots.js.map +0 -1
- package/dist/hono/routes.d.ts +0 -19
- package/dist/hono/routes.d.ts.map +0 -1
- package/dist/hono/routes.js +0 -37
- package/dist/hono/routes.js.map +0 -1
- package/dist/hono/validation.d.ts +0 -5
- package/dist/hono/validation.d.ts.map +0 -1
- package/dist/hono/validation.js +0 -47
- package/dist/hono/validation.js.map +0 -1
- package/dist/hono/websocket-origin.d.ts +0 -9
- package/dist/hono/websocket-origin.d.ts.map +0 -1
- package/dist/hono/websocket-origin.js +0 -96
- package/dist/hono/websocket-origin.js.map +0 -1
- package/dist/hono/ws.d.ts +0 -341
- package/dist/hono/ws.d.ts.map +0 -1
- package/dist/hono/ws.js +0 -711
- package/dist/hono/ws.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/libsql.d.ts +0 -29
- package/dist/libsql.d.ts.map +0 -1
- package/dist/libsql.js +0 -25
- package/dist/libsql.js.map +0 -1
- package/dist/migrate.d.ts +0 -14
- package/dist/migrate.d.ts.map +0 -1
- package/dist/migrate.js +0 -13
- package/dist/migrate.js.map +0 -1
- package/dist/neon.d.ts +0 -21
- package/dist/neon.d.ts.map +0 -1
- package/dist/neon.js +0 -22
- package/dist/neon.js.map +0 -1
- package/dist/notify.d.ts +0 -77
- package/dist/notify.d.ts.map +0 -1
- package/dist/notify.js +0 -233
- package/dist/notify.js.map +0 -1
- package/dist/pglite.d.ts +0 -37
- package/dist/pglite.d.ts.map +0 -1
- package/dist/pglite.js +0 -37
- package/dist/pglite.js.map +0 -1
- package/dist/plugins/index.d.ts +0 -2
- package/dist/plugins/index.d.ts.map +0 -1
- package/dist/plugins/index.js +0 -2
- package/dist/plugins/index.js.map +0 -1
- package/dist/plugins/types.d.ts +0 -73
- package/dist/plugins/types.d.ts.map +0 -1
- package/dist/plugins/types.js +0 -30
- package/dist/plugins/types.js.map +0 -1
- package/dist/postgres/index.d.ts +0 -68
- package/dist/postgres/index.d.ts.map +0 -1
- package/dist/postgres/index.js +0 -924
- package/dist/postgres/index.js.map +0 -1
- package/dist/proxy/collection.d.ts +0 -7
- package/dist/proxy/collection.d.ts.map +0 -1
- package/dist/proxy/collection.js +0 -6
- package/dist/proxy/collection.js.map +0 -1
- package/dist/proxy/handler.d.ts +0 -42
- package/dist/proxy/handler.d.ts.map +0 -1
- package/dist/proxy/handler.js +0 -102
- package/dist/proxy/handler.js.map +0 -1
- package/dist/proxy/index.d.ts +0 -9
- package/dist/proxy/index.d.ts.map +0 -1
- package/dist/proxy/index.js +0 -14
- package/dist/proxy/index.js.map +0 -1
- package/dist/proxy/mutation-detector.d.ts +0 -35
- package/dist/proxy/mutation-detector.d.ts.map +0 -1
- package/dist/proxy/mutation-detector.js +0 -246
- package/dist/proxy/mutation-detector.js.map +0 -1
- package/dist/proxy/oplog.d.ts +0 -30
- package/dist/proxy/oplog.d.ts.map +0 -1
- package/dist/proxy/oplog.js +0 -137
- package/dist/proxy/oplog.js.map +0 -1
- package/dist/proxy/types.d.ts +0 -44
- package/dist/proxy/types.d.ts.map +0 -1
- package/dist/proxy/types.js +0 -7
- package/dist/proxy/types.js.map +0 -1
- package/dist/prune.d.ts.map +0 -1
- package/dist/prune.js.map +0 -1
- package/dist/pull.d.ts.map +0 -1
- package/dist/pull.js.map +0 -1
- package/dist/push.d.ts.map +0 -1
- package/dist/push.js.map +0 -1
- package/dist/realtime/in-memory.d.ts +0 -13
- package/dist/realtime/in-memory.d.ts.map +0 -1
- package/dist/realtime/in-memory.js +0 -28
- package/dist/realtime/in-memory.js.map +0 -1
- package/dist/realtime/index.d.ts +0 -4
- package/dist/realtime/index.d.ts.map +0 -1
- package/dist/realtime/index.js +0 -3
- package/dist/realtime/index.js.map +0 -1
- package/dist/realtime/types.d.ts +0 -62
- package/dist/realtime/types.d.ts.map +0 -1
- package/dist/realtime/types.js +0 -19
- package/dist/realtime/types.js.map +0 -1
- package/dist/relay/client-role/forward-engine.d.ts +0 -63
- package/dist/relay/client-role/forward-engine.d.ts.map +0 -1
- package/dist/relay/client-role/forward-engine.js +0 -267
- package/dist/relay/client-role/forward-engine.js.map +0 -1
- package/dist/relay/client-role/index.d.ts +0 -9
- package/dist/relay/client-role/index.d.ts.map +0 -1
- package/dist/relay/client-role/index.js +0 -9
- package/dist/relay/client-role/index.js.map +0 -1
- package/dist/relay/client-role/pull-engine.d.ts +0 -72
- package/dist/relay/client-role/pull-engine.d.ts.map +0 -1
- package/dist/relay/client-role/pull-engine.js +0 -249
- package/dist/relay/client-role/pull-engine.js.map +0 -1
- package/dist/relay/client-role/sequence-mapper.d.ts +0 -65
- package/dist/relay/client-role/sequence-mapper.d.ts.map +0 -1
- package/dist/relay/client-role/sequence-mapper.js +0 -161
- package/dist/relay/client-role/sequence-mapper.js.map +0 -1
- package/dist/relay/evaluation/relay-paths.d.ts +0 -41
- package/dist/relay/evaluation/relay-paths.d.ts.map +0 -1
- package/dist/relay/evaluation/relay-paths.js +0 -504
- package/dist/relay/evaluation/relay-paths.js.map +0 -1
- package/dist/relay/evaluation/rust-boundary.d.ts +0 -47
- package/dist/relay/evaluation/rust-boundary.d.ts.map +0 -1
- package/dist/relay/evaluation/rust-boundary.js +0 -220
- package/dist/relay/evaluation/rust-boundary.js.map +0 -1
- package/dist/relay/index.d.ts +0 -37
- package/dist/relay/index.d.ts.map +0 -1
- package/dist/relay/index.js +0 -44
- package/dist/relay/index.js.map +0 -1
- package/dist/relay/migrate.d.ts +0 -18
- package/dist/relay/migrate.d.ts.map +0 -1
- package/dist/relay/migrate.js +0 -99
- package/dist/relay/migrate.js.map +0 -1
- package/dist/relay/mode-manager.d.ts +0 -60
- package/dist/relay/mode-manager.d.ts.map +0 -1
- package/dist/relay/mode-manager.js +0 -114
- package/dist/relay/mode-manager.js.map +0 -1
- package/dist/relay/realtime.d.ts +0 -90
- package/dist/relay/realtime.d.ts.map +0 -1
- package/dist/relay/realtime.js +0 -147
- package/dist/relay/realtime.js.map +0 -1
- package/dist/relay/relay.d.ts +0 -190
- package/dist/relay/relay.d.ts.map +0 -1
- package/dist/relay/relay.js +0 -320
- package/dist/relay/relay.js.map +0 -1
- package/dist/relay/schema.d.ts +0 -158
- package/dist/relay/schema.d.ts.map +0 -1
- package/dist/relay/schema.js +0 -7
- package/dist/relay/schema.js.map +0 -1
- package/dist/relay/server-role/index.d.ts +0 -56
- package/dist/relay/server-role/index.d.ts.map +0 -1
- package/dist/relay/server-role/index.js +0 -193
- package/dist/relay/server-role/index.js.map +0 -1
- package/dist/relay/server-role/pull.d.ts +0 -27
- package/dist/relay/server-role/pull.d.ts.map +0 -1
- package/dist/relay/server-role/pull.js +0 -24
- package/dist/relay/server-role/pull.js.map +0 -1
- package/dist/relay/server-role/push.d.ts +0 -29
- package/dist/relay/server-role/push.d.ts.map +0 -1
- package/dist/relay/server-role/push.js +0 -122
- package/dist/relay/server-role/push.js.map +0 -1
- package/dist/s3/index.d.ts +0 -83
- package/dist/s3/index.d.ts.map +0 -1
- package/dist/s3/index.js +0 -223
- package/dist/s3/index.js.map +0 -1
- package/dist/schema.d.ts.map +0 -1
- package/dist/schema.js.map +0 -1
- package/dist/service-worker/index.d.ts +0 -114
- package/dist/service-worker/index.d.ts.map +0 -1
- package/dist/service-worker/index.js +0 -487
- package/dist/service-worker/index.js.map +0 -1
- package/dist/snapshot-artifacts/sqlite-bun.d.ts +0 -15
- package/dist/snapshot-artifacts/sqlite-bun.d.ts.map +0 -1
- package/dist/snapshot-artifacts/sqlite-bun.js +0 -128
- package/dist/snapshot-artifacts/sqlite-bun.js.map +0 -1
- package/dist/snapshot-artifacts.d.ts +0 -206
- package/dist/snapshot-artifacts.d.ts.map +0 -1
- package/dist/snapshot-artifacts.js +0 -545
- package/dist/snapshot-artifacts.js.map +0 -1
- package/dist/snapshot-chunks/db-metadata.d.ts +0 -56
- package/dist/snapshot-chunks/db-metadata.d.ts.map +0 -1
- package/dist/snapshot-chunks/db-metadata.js +0 -360
- package/dist/snapshot-chunks/db-metadata.js.map +0 -1
- package/dist/snapshot-chunks/index.d.ts +0 -8
- package/dist/snapshot-chunks/index.d.ts.map +0 -1
- package/dist/snapshot-chunks/index.js +0 -8
- package/dist/snapshot-chunks/index.js.map +0 -1
- package/dist/snapshot-chunks/types.d.ts +0 -80
- package/dist/snapshot-chunks/types.d.ts.map +0 -1
- package/dist/snapshot-chunks/types.js +0 -8
- package/dist/snapshot-chunks/types.js.map +0 -1
- package/dist/snapshot-chunks.d.ts +0 -90
- package/dist/snapshot-chunks.d.ts.map +0 -1
- package/dist/snapshot-chunks.js +0 -304
- package/dist/snapshot-chunks.js.map +0 -1
- package/dist/sqlite/index.d.ts +0 -53
- package/dist/sqlite/index.d.ts.map +0 -1
- package/dist/sqlite/index.js +0 -795
- package/dist/sqlite/index.js.map +0 -1
- package/dist/sqlite3.d.ts +0 -22
- package/dist/sqlite3.d.ts.map +0 -1
- package/dist/sqlite3.js +0 -99
- package/dist/sqlite3.js.map +0 -1
- package/dist/stats.d.ts +0 -28
- package/dist/stats.d.ts.map +0 -1
- package/dist/stats.js +0 -93
- package/dist/stats.js.map +0 -1
- package/dist/subscriptions/cache.d.ts +0 -58
- package/dist/subscriptions/cache.d.ts.map +0 -1
- package/dist/subscriptions/cache.js +0 -250
- package/dist/subscriptions/cache.js.map +0 -1
- package/dist/subscriptions/index.d.ts +0 -3
- package/dist/subscriptions/index.d.ts.map +0 -1
- package/dist/subscriptions/index.js +0 -3
- package/dist/subscriptions/index.js.map +0 -1
- package/dist/subscriptions/resolve.d.ts +0 -40
- package/dist/subscriptions/resolve.d.ts.map +0 -1
- package/dist/subscriptions/resolve.js +0 -275
- package/dist/subscriptions/resolve.js.map +0 -1
- package/dist/sync.d.ts +0 -24
- package/dist/sync.d.ts.map +0 -1
- package/dist/sync.js +0 -27
- package/dist/sync.js.map +0 -1
- package/src/auth-leases.ts +0 -649
- package/src/better-sqlite3.ts +0 -35
- package/src/blobs/access.ts +0 -244
- package/src/blobs/adapters/database.ts +0 -397
- package/src/blobs/index.ts +0 -9
- package/src/blobs/manager.ts +0 -901
- package/src/blobs/migrate.ts +0 -158
- package/src/blobs/types.ts +0 -74
- package/src/bun-sqlite-ambient.d.ts +0 -19
- package/src/bun-sqlite.ts +0 -27
- package/src/clients.ts +0 -22
- package/src/cloudflare/durable-object.ts +0 -289
- package/src/cloudflare/index.ts +0 -22
- package/src/cloudflare/r2.ts +0 -526
- package/src/cloudflare/scope-cache.ts +0 -341
- package/src/cloudflare/sentry.ts +0 -230
- package/src/cloudflare/worker.ts +0 -77
- package/src/commit-integrity.ts +0 -371
- package/src/compaction.ts +0 -77
- package/src/crdt-yjs/index.ts +0 -931
- package/src/d1.ts +0 -16
- package/src/dialect/base.ts +0 -360
- package/src/dialect/helpers.ts +0 -92
- package/src/dialect/index.ts +0 -7
- package/src/dialect/types.ts +0 -247
- package/src/encrypted-crdt.ts +0 -786
- package/src/filesystem/index.ts +0 -262
- package/src/handlers/collection.ts +0 -121
- package/src/handlers/create-handler.ts +0 -1134
- package/src/handlers/index.ts +0 -3
- package/src/handlers/types.ts +0 -403
- package/src/helpers/conflict.ts +0 -64
- package/src/helpers/emitted-change.ts +0 -69
- package/src/helpers/index.ts +0 -12
- package/src/helpers/paginate.ts +0 -82
- package/src/helpers/scope-authorization.ts +0 -27
- package/src/helpers/scope-commit-index.ts +0 -52
- package/src/helpers/scope-strings.ts +0 -101
- package/src/hono/api-key-auth.ts +0 -177
- package/src/hono/audit-redaction.ts +0 -135
- package/src/hono/blobs.ts +0 -851
- package/src/hono/console/gateway.ts +0 -3046
- package/src/hono/console/live-auth.ts +0 -46
- package/src/hono/console/route-descriptor.ts +0 -22
- package/src/hono/console/routes/api-keys.ts +0 -721
- package/src/hono/console/routes/clients.ts +0 -447
- package/src/hono/console/routes/commits.ts +0 -1137
- package/src/hono/console/routes/context.ts +0 -956
- package/src/hono/console/routes/events.ts +0 -669
- package/src/hono/console/routes/maintenance.ts +0 -461
- package/src/hono/console/routes/shared.ts +0 -816
- package/src/hono/console/routes/stats.ts +0 -392
- package/src/hono/console/routes/storage.ts +0 -159
- package/src/hono/console/routes.ts +0 -146
- package/src/hono/console/schema-errors.ts +0 -23
- package/src/hono/console/schemas.ts +0 -914
- package/src/hono/console/types.ts +0 -223
- package/src/hono/console/ui.ts +0 -100
- package/src/hono/create-server.ts +0 -230
- package/src/hono/errors.ts +0 -50
- package/src/hono/index.ts +0 -54
- package/src/hono/openapi.ts +0 -139
- package/src/hono/proxy/connection-manager.ts +0 -340
- package/src/hono/proxy/index.ts +0 -8
- package/src/hono/proxy/routes.ts +0 -272
- package/src/hono/rate-limit.ts +0 -319
- package/src/hono/realtime-sync-packs.ts +0 -354
- package/src/hono/routes/audit.ts +0 -499
- package/src/hono/routes/auth-leases.ts +0 -119
- package/src/hono/routes/combined.ts +0 -583
- package/src/hono/routes/context.ts +0 -1629
- package/src/hono/routes/health.ts +0 -26
- package/src/hono/routes/realtime.ts +0 -808
- package/src/hono/routes/shared.ts +0 -1626
- package/src/hono/routes/snapshots.ts +0 -345
- package/src/hono/routes.ts +0 -68
- package/src/hono/validation.ts +0 -81
- package/src/hono/websocket-origin.ts +0 -131
- package/src/hono/ws.ts +0 -1134
- package/src/libsql.ts +0 -51
- package/src/migrate.ts +0 -20
- package/src/neon.ts +0 -28
- package/src/notify.ts +0 -341
- package/src/pglite.ts +0 -68
- package/src/plugins/index.ts +0 -1
- package/src/plugins/types.ts +0 -144
- package/src/postgres/index.ts +0 -1291
- package/src/proxy/collection.ts +0 -17
- package/src/proxy/handler.ts +0 -159
- package/src/proxy/index.ts +0 -21
- package/src/proxy/mutation-detector.ts +0 -281
- package/src/proxy/oplog.ts +0 -181
- package/src/proxy/types.ts +0 -46
- package/src/realtime/in-memory.ts +0 -33
- package/src/realtime/index.ts +0 -7
- package/src/realtime/types.ts +0 -90
- package/src/relay/bun-types.d.ts +0 -50
- package/src/relay/client-role/forward-engine.ts +0 -355
- package/src/relay/client-role/index.ts +0 -9
- package/src/relay/client-role/pull-engine.ts +0 -329
- package/src/relay/client-role/sequence-mapper.ts +0 -201
- package/src/relay/evaluation/relay-paths.ts +0 -699
- package/src/relay/evaluation/rust-boundary.ts +0 -464
- package/src/relay/index.ts +0 -50
- package/src/relay/migrate.ts +0 -113
- package/src/relay/mode-manager.ts +0 -142
- package/src/relay/realtime.ts +0 -207
- package/src/relay/relay.ts +0 -431
- package/src/relay/schema.ts +0 -171
- package/src/relay/server-role/index.ts +0 -338
- package/src/relay/server-role/pull.ts +0 -43
- package/src/relay/server-role/push.ts +0 -164
- package/src/s3/index.ts +0 -346
- package/src/service-worker/index.ts +0 -773
- package/src/snapshot-artifacts/sqlite-bun.ts +0 -168
- package/src/snapshot-artifacts.ts +0 -896
- package/src/snapshot-chunks/db-metadata.ts +0 -537
- package/src/snapshot-chunks/index.ts +0 -8
- package/src/snapshot-chunks/types.ts +0 -105
- package/src/snapshot-chunks.ts +0 -453
- package/src/sqlite/index.ts +0 -1064
- package/src/sqlite3.ts +0 -137
- package/src/stats.ts +0 -180
- package/src/subscriptions/cache.ts +0 -376
- package/src/subscriptions/index.ts +0 -2
- package/src/subscriptions/resolve.ts +0 -357
- package/src/sync.ts +0 -111
package/src/pull.ts
CHANGED
|
@@ -1,1865 +1,565 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pull: incremental commit delivery, cursors, bootstrap segments
|
|
3
|
+
* (SPEC.md §4, §5).
|
|
4
|
+
*/
|
|
1
5
|
import {
|
|
2
|
-
type
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
distributionSyncMetric,
|
|
11
|
-
encodeBinarySnapshotTable,
|
|
12
|
-
gzipBytes,
|
|
13
|
-
randomId,
|
|
14
|
-
type ScopeValues,
|
|
15
|
-
SYNC_SCOPED_SNAPSHOT_ARTIFACT_KIND_SQLITE_V1,
|
|
16
|
-
SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
17
|
-
SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
18
|
-
type SyncBootstrapState,
|
|
19
|
-
type SyncChange,
|
|
20
|
-
type SyncCommit,
|
|
21
|
-
type SyncCrdtStateVectorHint,
|
|
22
|
-
type SyncPullRequest,
|
|
23
|
-
type SyncPullResponse,
|
|
24
|
-
type SyncPullSubscriptionResponse,
|
|
25
|
-
type SyncScopedSnapshotArtifactKind,
|
|
26
|
-
type SyncSnapshot,
|
|
27
|
-
type SyncSnapshotArtifactCompression,
|
|
28
|
-
type SyncSnapshotArtifactsRequest,
|
|
29
|
-
type SyncSnapshotChunkRef,
|
|
30
|
-
sha256Hex,
|
|
31
|
-
snapshotScopeDigestFromCacheKey,
|
|
32
|
-
startSyncSpan,
|
|
6
|
+
type CommitChange,
|
|
7
|
+
decodeRow,
|
|
8
|
+
encodeRowsSegment,
|
|
9
|
+
type PullHeaderFrame,
|
|
10
|
+
type ResponseFrame,
|
|
11
|
+
type ScopeMap,
|
|
12
|
+
type SegmentRow,
|
|
13
|
+
type SubscriptionFrame,
|
|
33
14
|
} from '@syncular/core';
|
|
34
|
-
import type {
|
|
35
|
-
import {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
} from './
|
|
39
|
-
import type {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
} from './dialect/types';
|
|
44
|
-
import {
|
|
45
|
-
getServerBootstrapOrderFor,
|
|
46
|
-
type ServerHandlerCollection,
|
|
47
|
-
} from './handlers/collection';
|
|
48
|
-
import type { ServerTableHandler, SyncServerAuth } from './handlers/types';
|
|
49
|
-
import { EXTERNAL_CLIENT_ID } from './notify';
|
|
50
|
-
import {
|
|
51
|
-
type SyncServerPullPlugin,
|
|
52
|
-
sortServerPullPlugins,
|
|
53
|
-
} from './plugins/types';
|
|
54
|
-
import type { SyncCoreDb } from './schema';
|
|
55
|
-
import {
|
|
56
|
-
createScopedSnapshotArtifactScopeCacheKey,
|
|
57
|
-
readBestScopedSnapshotArtifactRefForPageCapacity,
|
|
58
|
-
} from './snapshot-artifacts';
|
|
59
|
-
import {
|
|
60
|
-
createSnapshotChunkScopeCacheKey,
|
|
61
|
-
insertSnapshotChunk,
|
|
62
|
-
readSnapshotChunkRefByPageKey,
|
|
63
|
-
type SnapshotChunkRefWithContinuation,
|
|
64
|
-
} from './snapshot-chunks';
|
|
65
|
-
import type { SnapshotChunkStorage } from './snapshot-chunks/types';
|
|
66
|
-
import {
|
|
67
|
-
createMemoryScopeCache,
|
|
68
|
-
type ScopeCacheBackend,
|
|
69
|
-
} from './subscriptions/cache';
|
|
70
|
-
import { resolveEffectiveScopesForSubscriptions } from './subscriptions/resolve';
|
|
71
|
-
|
|
72
|
-
const defaultScopeCache = createMemoryScopeCache();
|
|
73
|
-
const DEFAULT_MAX_BINARY_SNAPSHOT_BUNDLE_ROWS = 50_000;
|
|
74
|
-
const DEFAULT_SNAPSHOT_CHUNK_GZIP_LEVEL = 1;
|
|
75
|
-
const MAX_PULL_TRANSACTION_RETRIES = 2;
|
|
76
|
-
const PULL_TRANSACTION_RETRY_DELAY_MS = 15;
|
|
77
|
-
|
|
78
|
-
interface PullBootstrapTimings {
|
|
79
|
-
snapshotQueryMs: number;
|
|
80
|
-
binaryEncodeMs: number;
|
|
81
|
-
chunkCacheLookupMs: number;
|
|
82
|
-
artifactCacheLookupMs: number;
|
|
83
|
-
chunkGzipMs: number;
|
|
84
|
-
chunkHashMs: number;
|
|
85
|
-
chunkPersistMs: number;
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
interface SnapshotChunkEncodeResult {
|
|
89
|
-
body: Uint8Array;
|
|
90
|
-
sha256: string;
|
|
91
|
-
gzipMs: number;
|
|
92
|
-
hashMs: number;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
function toResponseChunkRef(ref: SyncSnapshotChunkRef): SyncSnapshotChunkRef {
|
|
96
|
-
return {
|
|
97
|
-
id: ref.id,
|
|
98
|
-
byteLength: ref.byteLength,
|
|
99
|
-
sha256: ref.sha256,
|
|
100
|
-
encoding: ref.encoding,
|
|
101
|
-
compression: ref.compression,
|
|
102
|
-
};
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
async function createChunkedSnapshotManifest(args: {
|
|
106
|
-
table: string;
|
|
107
|
-
asOfCommitSeq: number;
|
|
108
|
-
scopeKey: string;
|
|
109
|
-
rowCursor: string | null;
|
|
110
|
-
rowLimit: number;
|
|
111
|
-
nextRowCursor: string | null;
|
|
112
|
-
isFirstPage: boolean;
|
|
113
|
-
isLastPage: boolean;
|
|
114
|
-
chunks: readonly SyncSnapshotChunkRef[];
|
|
115
|
-
}): Promise<SyncSnapshot['manifest']> {
|
|
116
|
-
return createSnapshotManifest({
|
|
117
|
-
version: 1,
|
|
118
|
-
table: args.table,
|
|
119
|
-
asOfCommitSeq: args.asOfCommitSeq,
|
|
120
|
-
scopeDigest: snapshotScopeDigestFromCacheKey(args.scopeKey),
|
|
121
|
-
rowCursor: args.rowCursor,
|
|
122
|
-
rowLimit: args.rowLimit,
|
|
123
|
-
nextRowCursor: args.nextRowCursor,
|
|
124
|
-
isFirstPage: args.isFirstPage,
|
|
125
|
-
isLastPage: args.isLastPage,
|
|
126
|
-
chunks: args.chunks.map(toResponseChunkRef),
|
|
127
|
-
});
|
|
128
|
-
}
|
|
15
|
+
import type { SyncRequestContext } from './context';
|
|
16
|
+
import { clockOf, limitsOf } from './context';
|
|
17
|
+
import type { PullSegmentSummary } from './events';
|
|
18
|
+
import type { CompiledSchema, CompiledTable } from './schema';
|
|
19
|
+
import { scopeDigest } from './scopes';
|
|
20
|
+
import type { SegmentRecord } from './segment-store';
|
|
21
|
+
import { issueSegmentUrl } from './signed-url';
|
|
22
|
+
import type { SqliteImageBuilder } from './sqlite-image';
|
|
23
|
+
import type { StoredCommit, StoredRow } from './storage';
|
|
129
24
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
function
|
|
138
|
-
|
|
139
|
-
):
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
): SnapshotArtifactSelection | null {
|
|
147
|
-
if (!request) return null;
|
|
148
|
-
if (
|
|
149
|
-
!request.artifactKinds.includes(
|
|
150
|
-
SYNC_SCOPED_SNAPSHOT_ARTIFACT_KIND_SQLITE_V1
|
|
151
|
-
)
|
|
152
|
-
) {
|
|
153
|
-
return null;
|
|
154
|
-
}
|
|
155
|
-
if (!request.compressions?.includes(SYNC_SNAPSHOT_CHUNK_COMPRESSION)) {
|
|
156
|
-
return null;
|
|
25
|
+
/**
|
|
26
|
+
* Resolve the §5.3 image builder: the host-injected one if present, else the
|
|
27
|
+
* in-tree `buildSqliteImage` on a Bun runtime (dynamic import so `bun:sqlite`
|
|
28
|
+
* is never a static dep of the neutral core), else `undefined` (rows lane).
|
|
29
|
+
* Memoized so the dynamic import happens at most once per process.
|
|
30
|
+
*/
|
|
31
|
+
let cachedDefaultBuilder: SqliteImageBuilder | null | undefined;
|
|
32
|
+
async function resolveImageBuilder(
|
|
33
|
+
ctx: SyncRequestContext,
|
|
34
|
+
): Promise<SqliteImageBuilder | undefined> {
|
|
35
|
+
if (ctx.sqliteImageBuilder !== undefined) return ctx.sqliteImageBuilder;
|
|
36
|
+
if (cachedDefaultBuilder === undefined) {
|
|
37
|
+
const hasBun = (globalThis as { Bun?: unknown }).Bun !== undefined;
|
|
38
|
+
cachedDefaultBuilder = hasBun
|
|
39
|
+
? (await import('./sqlite-image')).buildSqliteImage
|
|
40
|
+
: null;
|
|
157
41
|
}
|
|
158
|
-
return
|
|
159
|
-
artifactKind: SYNC_SCOPED_SNAPSHOT_ARTIFACT_KIND_SQLITE_V1,
|
|
160
|
-
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
161
|
-
schemaVersion: String(clientSchemaVersion),
|
|
162
|
-
featureSet: normalizeFeatureSet(request.featureSet),
|
|
163
|
-
};
|
|
42
|
+
return cachedDefaultBuilder ?? undefined;
|
|
164
43
|
}
|
|
165
44
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
45
|
+
/** §4.2 accept bitmask. */
|
|
46
|
+
export const ACCEPT_INLINE_ROWS = 1 << 0;
|
|
47
|
+
export const ACCEPT_EXTERNAL_ROWS = 1 << 1;
|
|
48
|
+
export const ACCEPT_SQLITE = 1 << 2;
|
|
49
|
+
export const ACCEPT_SIGNED_URLS = 1 << 3;
|
|
50
|
+
|
|
51
|
+
export interface PullLimits {
|
|
52
|
+
readonly limitCommits: number;
|
|
53
|
+
readonly limitSnapshotRows: number;
|
|
54
|
+
readonly maxSnapshotPages: number;
|
|
55
|
+
readonly accept: number;
|
|
177
56
|
}
|
|
178
57
|
|
|
179
|
-
function
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
>(
|
|
183
|
-
handler: ServerTableHandler<DB, Auth>,
|
|
184
|
-
schemaVersion: number
|
|
185
|
-
): BinarySnapshotRowsEncoder | undefined {
|
|
186
|
-
const versioned = handler.snapshotBinaryEncoderForVersion?.(schemaVersion);
|
|
187
|
-
return versioned === undefined
|
|
188
|
-
? handler.snapshotBinaryEncoder
|
|
189
|
-
: (versioned ?? undefined);
|
|
58
|
+
function clamp(value: number, min: number, max: number, dflt: number): number {
|
|
59
|
+
if (value === 0) return dflt;
|
|
60
|
+
return Math.min(max, Math.max(min, value));
|
|
190
61
|
}
|
|
191
62
|
|
|
192
|
-
|
|
63
|
+
/** §4.2 defaults and silent clamps (the v1 values). */
|
|
64
|
+
export function clampPullLimits(header: PullHeaderFrame): PullLimits {
|
|
193
65
|
return {
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
chunkGzipMs: 0,
|
|
199
|
-
chunkHashMs: 0,
|
|
200
|
-
chunkPersistMs: 0,
|
|
66
|
+
limitCommits: clamp(header.limitCommits, 1, 1000, 1000),
|
|
67
|
+
limitSnapshotRows: clamp(header.limitSnapshotRows, 1, 50000, 1000),
|
|
68
|
+
maxSnapshotPages: clamp(header.maxSnapshotPages, 1, 50, 4),
|
|
69
|
+
accept: header.accept,
|
|
201
70
|
};
|
|
202
71
|
}
|
|
203
72
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
const pagesRemaining = Math.max(1, args.pagesRemaining);
|
|
210
|
-
const maxBundlePages = Math.max(
|
|
211
|
-
1,
|
|
212
|
-
Math.ceil(DEFAULT_MAX_BINARY_SNAPSHOT_BUNDLE_ROWS / pageSize)
|
|
213
|
-
);
|
|
214
|
-
return pageSize * Math.min(pagesRemaining, maxBundlePages);
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
async function gzipByteChunks(
|
|
218
|
-
chunks: readonly Uint8Array[],
|
|
219
|
-
gzipLevel: number
|
|
220
|
-
): Promise<Uint8Array> {
|
|
221
|
-
return gzipBytes(concatByteChunks(chunks), {
|
|
222
|
-
level: gzipLevel,
|
|
223
|
-
});
|
|
73
|
+
export interface SubscriptionPlan {
|
|
74
|
+
readonly frame: SubscriptionFrame;
|
|
75
|
+
readonly table: CompiledTable;
|
|
76
|
+
readonly status: 'active' | 'revoked';
|
|
77
|
+
readonly effective: ScopeMap;
|
|
224
78
|
}
|
|
225
79
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
): Promise<SnapshotChunkEncodeResult> {
|
|
230
|
-
const gzipStartedAt = Date.now();
|
|
231
|
-
const body = await gzipByteChunks(chunks, gzipLevel);
|
|
232
|
-
const gzipMs = Math.max(0, Date.now() - gzipStartedAt);
|
|
233
|
-
const hashStartedAt = Date.now();
|
|
234
|
-
const sha256 = await sha256Hex(body);
|
|
235
|
-
const hashMs = Math.max(0, Date.now() - hashStartedAt);
|
|
236
|
-
return { body, sha256, gzipMs, hashMs };
|
|
80
|
+
export interface SubscriptionResult {
|
|
81
|
+
readonly nextCursor: number;
|
|
82
|
+
readonly active: boolean;
|
|
237
83
|
}
|
|
238
84
|
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
)
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
gzipMs: number;
|
|
247
|
-
hashMs: number;
|
|
248
|
-
}> {
|
|
249
|
-
const encoded = await encodeCompressedSnapshotChunk(chunks, gzipLevel);
|
|
250
|
-
return {
|
|
251
|
-
stream: bytesToReadableStream(encoded.body),
|
|
252
|
-
byteLength: encoded.body.length,
|
|
253
|
-
sha256: encoded.sha256,
|
|
254
|
-
gzipMs: encoded.gzipMs,
|
|
255
|
-
hashMs: encoded.hashMs,
|
|
256
|
-
};
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
interface SnapshotColumnInference {
|
|
260
|
-
name: string;
|
|
261
|
-
type: BinarySnapshotColumnType | null;
|
|
262
|
-
nullable: boolean;
|
|
263
|
-
presentCount: number;
|
|
85
|
+
/**
|
|
86
|
+
* Mutable per-section collector for the `pull.served` event. Only created
|
|
87
|
+
* when an events sink is configured — with `trace` undefined the pull path
|
|
88
|
+
* does zero extra work (`trace?.…` short-circuits argument evaluation).
|
|
89
|
+
*/
|
|
90
|
+
export interface PullSectionTrace {
|
|
91
|
+
readonly segments: PullSegmentSummary[];
|
|
264
92
|
}
|
|
265
93
|
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
const recordRows = rows.map((row) => toSnapshotRecordRow(table, row));
|
|
272
|
-
return encodeBinarySnapshotTable({
|
|
273
|
-
table,
|
|
274
|
-
columns: columns ?? inferBinarySnapshotColumns(recordRows),
|
|
275
|
-
rows: recordRows,
|
|
276
|
-
});
|
|
94
|
+
interface BootstrapToken {
|
|
95
|
+
asOfCommitSeq: number;
|
|
96
|
+
tables: string[];
|
|
97
|
+
tableIndex: number;
|
|
98
|
+
rowCursor: string | null;
|
|
277
99
|
}
|
|
278
100
|
|
|
279
|
-
function
|
|
101
|
+
function parseBootstrapToken(
|
|
102
|
+
raw: string | undefined,
|
|
280
103
|
table: string,
|
|
281
|
-
|
|
282
|
-
)
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
}
|
|
294
|
-
return row as Record<string, unknown>;
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
function inferBinarySnapshotColumns(
|
|
298
|
-
rows: readonly Record<string, unknown>[]
|
|
299
|
-
): BinarySnapshotColumn[] {
|
|
300
|
-
const columns: SnapshotColumnInference[] = [];
|
|
301
|
-
const columnsByName = new Map<string, SnapshotColumnInference>();
|
|
302
|
-
|
|
303
|
-
for (const row of rows) {
|
|
304
|
-
for (const name in row) {
|
|
305
|
-
if (!Object.hasOwn(row, name)) continue;
|
|
306
|
-
const value = row[name];
|
|
307
|
-
let column = columnsByName.get(name);
|
|
308
|
-
if (!column) {
|
|
309
|
-
column = { name, type: null, nullable: false, presentCount: 0 };
|
|
310
|
-
columnsByName.set(name, column);
|
|
311
|
-
columns.push(column);
|
|
312
|
-
}
|
|
313
|
-
column.presentCount += 1;
|
|
314
|
-
if (value == null) {
|
|
315
|
-
column.nullable = true;
|
|
316
|
-
continue;
|
|
317
|
-
}
|
|
318
|
-
column.type = mergeBinarySnapshotColumnTypes(
|
|
319
|
-
column.type,
|
|
320
|
-
inferBinarySnapshotColumnType(value)
|
|
321
|
-
);
|
|
104
|
+
): BootstrapToken | undefined {
|
|
105
|
+
if (raw === undefined) return undefined;
|
|
106
|
+
try {
|
|
107
|
+
const parsed = JSON.parse(raw) as Partial<BootstrapToken>;
|
|
108
|
+
if (
|
|
109
|
+
typeof parsed.asOfCommitSeq !== 'number' ||
|
|
110
|
+
!Array.isArray(parsed.tables) ||
|
|
111
|
+
typeof parsed.tableIndex !== 'number' ||
|
|
112
|
+
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string') ||
|
|
113
|
+
parsed.tables[0] !== table
|
|
114
|
+
) {
|
|
115
|
+
return undefined;
|
|
322
116
|
}
|
|
117
|
+
return {
|
|
118
|
+
asOfCommitSeq: parsed.asOfCommitSeq,
|
|
119
|
+
tables: parsed.tables as string[],
|
|
120
|
+
tableIndex: parsed.tableIndex,
|
|
121
|
+
rowCursor: parsed.rowCursor ?? null,
|
|
122
|
+
};
|
|
123
|
+
} catch {
|
|
124
|
+
return undefined;
|
|
323
125
|
}
|
|
126
|
+
}
|
|
324
127
|
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
128
|
+
function commitFrame(table: string, commit: StoredCommit): ResponseFrame {
|
|
129
|
+
const changes: CommitChange[] = commit.changes.map((change) => ({
|
|
130
|
+
tableIndex: 0,
|
|
131
|
+
rowId: change.rowId,
|
|
132
|
+
op: change.op,
|
|
133
|
+
...(change.rowVersion !== undefined
|
|
134
|
+
? { rowVersion: change.rowVersion }
|
|
330
135
|
: {}),
|
|
136
|
+
scopes: change.scopes,
|
|
137
|
+
...(change.payload !== undefined ? { row: change.payload } : {}),
|
|
331
138
|
}));
|
|
139
|
+
return {
|
|
140
|
+
type: 'COMMIT',
|
|
141
|
+
commitSeq: commit.commitSeq,
|
|
142
|
+
createdAtMs: commit.createdAtMs,
|
|
143
|
+
actorId: commit.actorId,
|
|
144
|
+
tables: [table],
|
|
145
|
+
changes,
|
|
146
|
+
};
|
|
332
147
|
}
|
|
333
148
|
|
|
334
|
-
function
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
if (typeof value === 'bigint') return 'integer';
|
|
340
|
-
if (typeof value === 'number') {
|
|
341
|
-
return Number.isSafeInteger(value) ? 'integer' : 'float';
|
|
342
|
-
}
|
|
343
|
-
if (value instanceof Uint8Array || value instanceof ArrayBuffer) {
|
|
344
|
-
return 'bytes';
|
|
345
|
-
}
|
|
346
|
-
return 'json';
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
function mergeBinarySnapshotColumnTypes(
|
|
350
|
-
current: BinarySnapshotColumnType | null,
|
|
351
|
-
next: BinarySnapshotColumnType
|
|
352
|
-
): BinarySnapshotColumnType {
|
|
353
|
-
if (!current || current === next) return next;
|
|
354
|
-
if (
|
|
355
|
-
(current === 'integer' && next === 'float') ||
|
|
356
|
-
(current === 'float' && next === 'integer')
|
|
357
|
-
) {
|
|
358
|
-
return 'float';
|
|
149
|
+
function chunkRows(rows: SegmentRow[], size: number): SegmentRow[][] {
|
|
150
|
+
if (rows.length === 0) return [];
|
|
151
|
+
const blocks: SegmentRow[][] = [];
|
|
152
|
+
for (let i = 0; i < rows.length; i += size) {
|
|
153
|
+
blocks.push(rows.slice(i, i + size));
|
|
359
154
|
}
|
|
360
|
-
return
|
|
361
|
-
}
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
155
|
+
return blocks;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** §5.4 signed-URL fields for a descriptor, when the client asked (bit 3). */
|
|
159
|
+
async function signedUrlFields(
|
|
160
|
+
ctx: SyncRequestContext,
|
|
161
|
+
limits: PullLimits,
|
|
162
|
+
segmentId: string,
|
|
163
|
+
digest: string,
|
|
164
|
+
now: number,
|
|
165
|
+
): Promise<{ url?: string; urlExpiresAtMs?: number }> {
|
|
166
|
+
if ((limits.accept & ACCEPT_SIGNED_URLS) === 0 || !ctx.signedUrls) return {};
|
|
167
|
+
return issueSegmentUrl(ctx.signedUrls, {
|
|
168
|
+
segmentId,
|
|
169
|
+
partition: ctx.partition,
|
|
170
|
+
scopeDigest: digest,
|
|
171
|
+
nowMs: now,
|
|
172
|
+
});
|
|
374
173
|
}
|
|
375
174
|
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
175
|
+
function segmentRefFrame(
|
|
176
|
+
record: SegmentRecord,
|
|
177
|
+
extra: { url?: string; urlExpiresAtMs?: number },
|
|
178
|
+
): ResponseFrame {
|
|
179
|
+
return {
|
|
180
|
+
type: 'SEGMENT_REF',
|
|
181
|
+
segmentId: record.segmentId,
|
|
182
|
+
mediaType: record.mediaType,
|
|
183
|
+
table: record.table,
|
|
184
|
+
byteLength: record.byteLength,
|
|
185
|
+
rowCount: record.rowCount,
|
|
186
|
+
asOfCommitSeq: record.asOfCommitSeq,
|
|
187
|
+
scopeDigest: record.scopeDigest,
|
|
188
|
+
...(record.rowCursor !== null ? { rowCursor: record.rowCursor } : {}),
|
|
189
|
+
...(record.nextRowCursor !== null
|
|
190
|
+
? { nextRowCursor: record.nextRowCursor }
|
|
191
|
+
: {}),
|
|
192
|
+
...(extra.url !== undefined ? { url: extra.url } : {}),
|
|
193
|
+
...(extra.urlExpiresAtMs !== undefined
|
|
194
|
+
? { urlExpiresAtMs: extra.urlExpiresAtMs }
|
|
195
|
+
: {}),
|
|
387
196
|
};
|
|
388
|
-
payloadParts: Uint8Array[];
|
|
389
|
-
expiresAt: string;
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
async function runWithConcurrency<T>(
|
|
393
|
-
items: readonly T[],
|
|
394
|
-
concurrency: number,
|
|
395
|
-
worker: (item: T) => Promise<void>
|
|
396
|
-
): Promise<void> {
|
|
397
|
-
if (items.length === 0) return;
|
|
398
|
-
|
|
399
|
-
const workerCount = Math.max(1, Math.min(concurrency, items.length));
|
|
400
|
-
let nextIndex = 0;
|
|
401
|
-
|
|
402
|
-
async function runWorker(): Promise<void> {
|
|
403
|
-
while (nextIndex < items.length) {
|
|
404
|
-
const index = nextIndex;
|
|
405
|
-
nextIndex += 1;
|
|
406
|
-
const item = items[index];
|
|
407
|
-
if (item === undefined) continue;
|
|
408
|
-
await worker(item);
|
|
409
|
-
}
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
await Promise.all(Array.from({ length: workerCount }, () => runWorker()));
|
|
413
197
|
}
|
|
414
198
|
|
|
415
199
|
/**
|
|
416
|
-
*
|
|
417
|
-
*
|
|
200
|
+
* §5.3 sqlite-image lane: only at the start of a table, only when the
|
|
201
|
+
* client advertised bit 2, and only when the snapshot exceeds one rows
|
|
202
|
+
* page. Reuses an unexpired stored image for the same (partition, table,
|
|
203
|
+
* schemaVersion, scope digest, pin) instead of rebuilding — the
|
|
204
|
+
* bootstrap-storm rule. Returns false when the table is not eligible
|
|
205
|
+
* (the rows lane takes over).
|
|
418
206
|
*/
|
|
419
|
-
function
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
withCode.code === '40001' ||
|
|
441
|
-
error.message.toLowerCase().includes('could not serialize access')
|
|
207
|
+
async function* sqliteImageSegment(
|
|
208
|
+
ctx: SyncRequestContext,
|
|
209
|
+
schema: CompiledSchema,
|
|
210
|
+
limits: PullLimits,
|
|
211
|
+
plan: SubscriptionPlan,
|
|
212
|
+
asOf: number,
|
|
213
|
+
digest: string,
|
|
214
|
+
trace: PullSectionTrace | undefined,
|
|
215
|
+
): AsyncGenerator<ResponseFrame, boolean> {
|
|
216
|
+
const { storage, segments, partition } = ctx;
|
|
217
|
+
const now = clockOf(ctx)();
|
|
218
|
+
const existing = await segments.find(
|
|
219
|
+
{
|
|
220
|
+
partition,
|
|
221
|
+
table: plan.table.name,
|
|
222
|
+
schemaVersion: schema.version,
|
|
223
|
+
mediaType: 'sqlite',
|
|
224
|
+
scopeDigest: digest,
|
|
225
|
+
asOfCommitSeq: asOf,
|
|
226
|
+
},
|
|
227
|
+
now,
|
|
442
228
|
);
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
function mergeScopes(subscriptions: { scopes: ScopeValues }[]): ScopeValues {
|
|
455
|
-
const result: Record<string, Set<string>> = {};
|
|
456
|
-
|
|
457
|
-
for (const sub of subscriptions) {
|
|
458
|
-
for (const [key, value] of Object.entries(sub.scopes)) {
|
|
459
|
-
if (!result[key]) result[key] = new Set();
|
|
460
|
-
const arr = Array.isArray(value) ? value : [value];
|
|
461
|
-
for (const v of arr) result[key].add(v);
|
|
462
|
-
}
|
|
463
|
-
}
|
|
464
|
-
|
|
465
|
-
const merged: ScopeValues = {};
|
|
466
|
-
for (const [key, set] of Object.entries(result)) {
|
|
467
|
-
const arr = Array.from(set);
|
|
468
|
-
if (arr.length === 0) continue;
|
|
469
|
-
merged[key] = arr.length === 1 ? arr[0]! : arr;
|
|
470
|
-
}
|
|
471
|
-
return merged;
|
|
472
|
-
}
|
|
473
|
-
|
|
474
|
-
interface PullResponseStats {
|
|
475
|
-
subscriptionCount: number;
|
|
476
|
-
activeSubscriptionCount: number;
|
|
477
|
-
revokedSubscriptionCount: number;
|
|
478
|
-
bootstrapSubscriptionCount: number;
|
|
479
|
-
commitCount: number;
|
|
480
|
-
changeCount: number;
|
|
481
|
-
snapshotPageCount: number;
|
|
482
|
-
}
|
|
483
|
-
|
|
484
|
-
function assertPullChangeIdentityUnchanged(
|
|
485
|
-
source: string,
|
|
486
|
-
before: SyncChange,
|
|
487
|
-
after: SyncChange
|
|
488
|
-
): void {
|
|
489
|
-
if (before.table !== after.table) {
|
|
490
|
-
throw new Error(
|
|
491
|
-
`${source} cannot change change.table (${before.table} -> ${after.table})`
|
|
229
|
+
if (existing !== undefined) {
|
|
230
|
+
trace?.segments.push({
|
|
231
|
+
mediaType: 'sqlite',
|
|
232
|
+
delivery: 'ref',
|
|
233
|
+
origin: 'reused',
|
|
234
|
+
bytes: existing.byteLength,
|
|
235
|
+
rows: existing.rowCount,
|
|
236
|
+
});
|
|
237
|
+
yield segmentRefFrame(
|
|
238
|
+
existing,
|
|
239
|
+
await signedUrlFields(ctx, limits, existing.segmentId, digest, now),
|
|
492
240
|
);
|
|
241
|
+
return true;
|
|
493
242
|
}
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
243
|
+
// Eligibility probe (§5.3): image only when the snapshot exceeds one
|
|
244
|
+
// rows page — smaller tables stay on the (typically inline) rows lane.
|
|
245
|
+
const probe = await storage.scanRows(partition, {
|
|
246
|
+
table: plan.table.name,
|
|
247
|
+
scopeFilter: plan.effective,
|
|
248
|
+
afterRowId: null,
|
|
249
|
+
limit: limits.limitSnapshotRows + 1,
|
|
250
|
+
});
|
|
251
|
+
if (probe.length <= limits.limitSnapshotRows) return false;
|
|
252
|
+
// §5.3: building an image needs a SQLite engine. The host injects the
|
|
253
|
+
// builder through `sqliteImageBuilder`; when omitted we default to the
|
|
254
|
+
// in-tree `buildSqliteImage` ONLY on a Bun runtime, reached by a *dynamic*
|
|
255
|
+
// import so `bun:sqlite` is never a static dependency of the pull path
|
|
256
|
+
// (TODO §4.2 neutrality — enforced by test/runtime-neutrality.test.ts). On
|
|
257
|
+
// Workers/edge (no `Bun`) this yields the rows lane for a bit-2 client — a
|
|
258
|
+
// support floor, not a fallback (§5.3: sqlite is an *accept*, not a demand).
|
|
259
|
+
const buildImage = await resolveImageBuilder(ctx);
|
|
260
|
+
if (buildImage === undefined) return false;
|
|
261
|
+
// The probe rows are the snapshot's first page — keep them and scan on
|
|
262
|
+
// from the probe's cursor instead of re-reading the whole prefix (the
|
|
263
|
+
// scan is keyset-ordered by rowId, so the concatenation is exactly the
|
|
264
|
+
// rows a single full scan would return).
|
|
265
|
+
const rows: StoredRow[] = [...probe];
|
|
266
|
+
let afterRowId: string | null = probe[probe.length - 1]?.rowId ?? null;
|
|
267
|
+
for (;;) {
|
|
268
|
+
const scanned: StoredRow[] = await storage.scanRows(partition, {
|
|
269
|
+
table: plan.table.name,
|
|
270
|
+
scopeFilter: plan.effective,
|
|
271
|
+
afterRowId,
|
|
272
|
+
limit: 50_000,
|
|
273
|
+
});
|
|
274
|
+
rows.push(...scanned);
|
|
275
|
+
const last = scanned[scanned.length - 1];
|
|
276
|
+
if (scanned.length < 50_000 || last === undefined) break;
|
|
277
|
+
afterRowId = last.rowId;
|
|
498
278
|
}
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
279
|
+
const bytes = buildImage({
|
|
280
|
+
table: plan.table,
|
|
281
|
+
schemaVersion: schema.version,
|
|
282
|
+
asOfCommitSeq: asOf,
|
|
283
|
+
scopeDigest: digest,
|
|
284
|
+
rows,
|
|
285
|
+
});
|
|
286
|
+
const record = await segments.put(
|
|
287
|
+
{
|
|
288
|
+
partition,
|
|
289
|
+
table: plan.table.name,
|
|
290
|
+
schemaVersion: schema.version,
|
|
291
|
+
mediaType: 'sqlite',
|
|
292
|
+
scopeDigest: digest,
|
|
293
|
+
asOfCommitSeq: asOf,
|
|
294
|
+
rowCount: rows.length,
|
|
295
|
+
rowCursor: null,
|
|
296
|
+
nextRowCursor: null,
|
|
297
|
+
},
|
|
298
|
+
bytes,
|
|
299
|
+
now,
|
|
300
|
+
);
|
|
301
|
+
trace?.segments.push({
|
|
302
|
+
mediaType: 'sqlite',
|
|
303
|
+
delivery: 'ref',
|
|
304
|
+
origin: 'built',
|
|
305
|
+
bytes: record.byteLength,
|
|
306
|
+
rows: record.rowCount,
|
|
307
|
+
});
|
|
308
|
+
yield segmentRefFrame(
|
|
309
|
+
record,
|
|
310
|
+
await signedUrlFields(ctx, limits, record.segmentId, digest, now),
|
|
311
|
+
);
|
|
312
|
+
return true;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
async function* bootstrapSegments(
|
|
316
|
+
ctx: SyncRequestContext,
|
|
317
|
+
schema: CompiledSchema,
|
|
318
|
+
limits: PullLimits,
|
|
319
|
+
plan: SubscriptionPlan,
|
|
320
|
+
asOf: number,
|
|
321
|
+
startRowCursor: string | null,
|
|
322
|
+
trace: PullSectionTrace | undefined,
|
|
323
|
+
): AsyncGenerator<
|
|
324
|
+
ResponseFrame,
|
|
325
|
+
{ complete: boolean; rowCursor: string | null }
|
|
326
|
+
> {
|
|
327
|
+
const { storage, segments, partition } = ctx;
|
|
328
|
+
const serverLimits = limitsOf(ctx);
|
|
329
|
+
const digest = await scopeDigest(plan.effective);
|
|
330
|
+
const now = clockOf(ctx)();
|
|
331
|
+
// §5.3: the sqlite-image lane — whole-table, chosen only at the start
|
|
332
|
+
// of a table (a mid-table resume stays on the rows lane, never
|
|
333
|
+
// switching lanes), and only when the client advertised bit 2.
|
|
334
|
+
// §5.11: a table with any encrypted column is image-INELIGIBLE — an image
|
|
335
|
+
// copies ciphertext wholesale with no per-row decrypt pass, so it MUST be
|
|
336
|
+
// served on the rows lane (which decrypts per row on the client).
|
|
337
|
+
if (
|
|
338
|
+
(limits.accept & ACCEPT_SQLITE) !== 0 &&
|
|
339
|
+
startRowCursor === null &&
|
|
340
|
+
plan.table.encryptedColumnIndices.length === 0
|
|
341
|
+
) {
|
|
342
|
+
const imaged = yield* sqliteImageSegment(
|
|
343
|
+
ctx,
|
|
344
|
+
schema,
|
|
345
|
+
limits,
|
|
346
|
+
plan,
|
|
347
|
+
asOf,
|
|
348
|
+
digest,
|
|
349
|
+
trace,
|
|
502
350
|
);
|
|
351
|
+
if (imaged) return { complete: true, rowCursor: null };
|
|
503
352
|
}
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
async function transformPullChanges<
|
|
526
|
-
DB extends SyncCoreDb,
|
|
527
|
-
Auth extends SyncServerAuth,
|
|
528
|
-
>(args: {
|
|
529
|
-
plugins: readonly SyncServerPullPlugin<DB, Auth>[];
|
|
530
|
-
ctx: { db: DbExecutor<DB>; actorId: string; auth: Auth };
|
|
531
|
-
tableHandler: ServerTableHandler<DB, Auth>;
|
|
532
|
-
subscription: {
|
|
533
|
-
id: string;
|
|
534
|
-
table: string;
|
|
535
|
-
scopes: ScopeValues;
|
|
536
|
-
params: Record<string, unknown> | undefined;
|
|
537
|
-
cursor: number;
|
|
538
|
-
crdtStateVectors: readonly SyncCrdtStateVectorHint[];
|
|
539
|
-
};
|
|
540
|
-
changes: readonly SyncChange[];
|
|
541
|
-
}): Promise<SyncChange[]> {
|
|
542
|
-
let changes = [...args.changes];
|
|
543
|
-
for (const plugin of args.plugins) {
|
|
544
|
-
if (!plugin.transformPullChanges) continue;
|
|
545
|
-
const nextChanges = await plugin.transformPullChanges({
|
|
546
|
-
ctx: args.ctx,
|
|
547
|
-
tableHandler: args.tableHandler,
|
|
548
|
-
subscription: args.subscription,
|
|
549
|
-
changes,
|
|
353
|
+
let rowCursor = startRowCursor;
|
|
354
|
+
for (let page = 0; page < limits.maxSnapshotPages; page++) {
|
|
355
|
+
const scanned = await storage.scanRows(partition, {
|
|
356
|
+
table: plan.table.name,
|
|
357
|
+
scopeFilter: plan.effective,
|
|
358
|
+
afterRowId: rowCursor,
|
|
359
|
+
limit: limits.limitSnapshotRows + 1,
|
|
360
|
+
});
|
|
361
|
+
const pageRows = scanned.slice(0, limits.limitSnapshotRows);
|
|
362
|
+
const hasMore = scanned.length > limits.limitSnapshotRows;
|
|
363
|
+
// §5.2: every row record carries the row's current server_version.
|
|
364
|
+
const decoded = pageRows.map((row: StoredRow) => ({
|
|
365
|
+
serverVersion: row.serverVersion,
|
|
366
|
+
values: decodeRow(plan.table.columns, row.payload),
|
|
367
|
+
}));
|
|
368
|
+
const bytes = encodeRowsSegment({
|
|
369
|
+
table: plan.table.name,
|
|
370
|
+
schemaVersion: schema.version,
|
|
371
|
+
columns: plan.table.columns,
|
|
372
|
+
blocks: chunkRows(decoded, 1000),
|
|
550
373
|
});
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
374
|
+
const lastRow = pageRows[pageRows.length - 1];
|
|
375
|
+
const nextRowCursor =
|
|
376
|
+
hasMore && lastRow !== undefined ? lastRow.rowId : null;
|
|
377
|
+
|
|
378
|
+
const canInline = (limits.accept & ACCEPT_INLINE_ROWS) !== 0;
|
|
379
|
+
const canExternal = (limits.accept & ACCEPT_EXTERNAL_ROWS) !== 0;
|
|
380
|
+
const inline =
|
|
381
|
+
canInline &&
|
|
382
|
+
(bytes.length <= serverLimits.inlineSegmentMaxBytes || !canExternal);
|
|
383
|
+
if (inline) {
|
|
384
|
+
trace?.segments.push({
|
|
385
|
+
mediaType: 'rows',
|
|
386
|
+
delivery: 'inline',
|
|
387
|
+
origin: 'built',
|
|
388
|
+
bytes: bytes.length,
|
|
389
|
+
rows: pageRows.length,
|
|
390
|
+
});
|
|
391
|
+
yield { type: 'SEGMENT_INLINE', payload: bytes };
|
|
392
|
+
} else {
|
|
393
|
+
const record = await segments.put(
|
|
394
|
+
{
|
|
395
|
+
partition,
|
|
396
|
+
table: plan.table.name,
|
|
397
|
+
schemaVersion: schema.version,
|
|
398
|
+
mediaType: 'rows',
|
|
399
|
+
scopeDigest: digest,
|
|
400
|
+
asOfCommitSeq: asOf,
|
|
401
|
+
rowCount: pageRows.length,
|
|
402
|
+
rowCursor,
|
|
403
|
+
nextRowCursor,
|
|
404
|
+
},
|
|
405
|
+
bytes,
|
|
406
|
+
now,
|
|
554
407
|
);
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
408
|
+
trace?.segments.push({
|
|
409
|
+
mediaType: 'rows',
|
|
410
|
+
delivery: 'ref',
|
|
411
|
+
origin: 'built',
|
|
412
|
+
bytes: record.byteLength,
|
|
413
|
+
rows: record.rowCount,
|
|
414
|
+
});
|
|
415
|
+
yield segmentRefFrame(
|
|
416
|
+
record,
|
|
417
|
+
await signedUrlFields(ctx, limits, record.segmentId, digest, now),
|
|
561
418
|
);
|
|
562
419
|
}
|
|
563
|
-
|
|
420
|
+
if (!hasMore) return { complete: true, rowCursor: null };
|
|
421
|
+
rowCursor = nextRowCursor;
|
|
564
422
|
}
|
|
565
|
-
return
|
|
423
|
+
return { complete: false, rowCursor };
|
|
566
424
|
}
|
|
567
425
|
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
426
|
+
/**
|
|
427
|
+
* Produce the `SUB_START … SUB_END` section for one subscription (§1.6),
|
|
428
|
+
* returning the cursor recorded for the retention watermark (§4.5).
|
|
429
|
+
*/
|
|
430
|
+
export async function* subscriptionSection(
|
|
431
|
+
ctx: SyncRequestContext,
|
|
432
|
+
schema: CompiledSchema,
|
|
433
|
+
limits: PullLimits,
|
|
434
|
+
plan: SubscriptionPlan,
|
|
435
|
+
maxSeq: number,
|
|
436
|
+
horizonSeq: number,
|
|
437
|
+
trace?: PullSectionTrace,
|
|
438
|
+
): AsyncGenerator<ResponseFrame, SubscriptionResult> {
|
|
439
|
+
const sub = plan.frame;
|
|
440
|
+
|
|
441
|
+
if (plan.status === 'revoked') {
|
|
442
|
+
yield {
|
|
443
|
+
type: 'SUB_START',
|
|
444
|
+
id: sub.id,
|
|
445
|
+
status: 'revoked',
|
|
446
|
+
reasonCode: 'sync.scope_revoked',
|
|
447
|
+
effectiveScopes: {},
|
|
448
|
+
bootstrap: false,
|
|
449
|
+
};
|
|
450
|
+
yield { type: 'SUB_END', nextCursor: sub.cursor };
|
|
451
|
+
return { nextCursor: sub.cursor, active: false };
|
|
452
|
+
}
|
|
583
453
|
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
454
|
+
const token = parseBootstrapToken(sub.bootstrapState, sub.table);
|
|
455
|
+
|
|
456
|
+
// §4.6: a cursor behind the horizon (and not resuming a bootstrap)
|
|
457
|
+
// cannot compute deltas — answer `reset` and echo the cursor.
|
|
458
|
+
if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
|
|
459
|
+
yield {
|
|
460
|
+
type: 'SUB_START',
|
|
461
|
+
id: sub.id,
|
|
462
|
+
status: 'reset',
|
|
463
|
+
reasonCode: 'sync.cursor_expired',
|
|
464
|
+
effectiveScopes: {},
|
|
465
|
+
bootstrap: false,
|
|
466
|
+
};
|
|
467
|
+
yield { type: 'SUB_END', nextCursor: sub.cursor };
|
|
468
|
+
return { nextCursor: sub.cursor, active: false };
|
|
469
|
+
}
|
|
587
470
|
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
471
|
+
const bootstrapping =
|
|
472
|
+
token !== undefined || sub.cursor < 0 || sub.cursor > maxSeq;
|
|
473
|
+
|
|
474
|
+
if (bootstrapping) {
|
|
475
|
+
// §4.7: resume at the pinned point unless the pin fell behind the
|
|
476
|
+
// horizon (or the token is unusable) — then restart from a fresh pin.
|
|
477
|
+
const resume =
|
|
478
|
+
token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
479
|
+
? token
|
|
480
|
+
: undefined;
|
|
481
|
+
const asOf = resume?.asOfCommitSeq ?? maxSeq;
|
|
482
|
+
const startCursor = resume?.rowCursor ?? null;
|
|
483
|
+
yield {
|
|
484
|
+
type: 'SUB_START',
|
|
485
|
+
id: sub.id,
|
|
486
|
+
status: 'active',
|
|
487
|
+
reasonCode: '',
|
|
488
|
+
effectiveScopes: plan.effective,
|
|
489
|
+
bootstrap: true,
|
|
490
|
+
};
|
|
491
|
+
const outcome = yield* bootstrapSegments(
|
|
492
|
+
ctx,
|
|
493
|
+
schema,
|
|
494
|
+
limits,
|
|
495
|
+
plan,
|
|
496
|
+
asOf,
|
|
497
|
+
startCursor,
|
|
498
|
+
trace,
|
|
499
|
+
);
|
|
500
|
+
if (outcome.complete) {
|
|
501
|
+
yield { type: 'SUB_END', nextCursor: asOf };
|
|
502
|
+
} else {
|
|
503
|
+
const nextToken: BootstrapToken = {
|
|
504
|
+
asOfCommitSeq: asOf,
|
|
505
|
+
tables: [sub.table],
|
|
506
|
+
tableIndex: 0,
|
|
507
|
+
rowCursor: outcome.rowCursor,
|
|
508
|
+
};
|
|
509
|
+
yield {
|
|
510
|
+
type: 'SUB_END',
|
|
511
|
+
nextCursor: asOf,
|
|
512
|
+
bootstrapState: JSON.stringify(nextToken),
|
|
513
|
+
};
|
|
592
514
|
}
|
|
593
|
-
|
|
594
|
-
snapshotPageCount += sub.snapshots?.length ?? 0;
|
|
515
|
+
return { nextCursor: asOf, active: true };
|
|
595
516
|
}
|
|
596
517
|
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
}
|
|
607
|
-
|
|
608
|
-
function recordPullMetrics(args: {
|
|
609
|
-
status: string;
|
|
610
|
-
dedupeRows: boolean;
|
|
611
|
-
durationMs: number;
|
|
612
|
-
stats: PullResponseStats;
|
|
613
|
-
}): void {
|
|
614
|
-
const { status, dedupeRows, durationMs, stats } = args;
|
|
615
|
-
const attributes = {
|
|
616
|
-
status,
|
|
617
|
-
dedupe_rows: dedupeRows,
|
|
518
|
+
// Incremental (§4.5): window cursor < commitSeq <= maxSeq, oldest first,
|
|
519
|
+
// cut off at limitCommits total changes, never splitting a commit.
|
|
520
|
+
yield {
|
|
521
|
+
type: 'SUB_START',
|
|
522
|
+
id: sub.id,
|
|
523
|
+
status: 'active',
|
|
524
|
+
reasonCode: '',
|
|
525
|
+
effectiveScopes: plan.effective,
|
|
526
|
+
bootstrap: false,
|
|
618
527
|
};
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
distributionSyncMetric(
|
|
626
|
-
'sync.server.pull.subscriptions',
|
|
627
|
-
stats.subscriptionCount,
|
|
628
|
-
{ attributes }
|
|
629
|
-
);
|
|
630
|
-
distributionSyncMetric(
|
|
631
|
-
'sync.server.pull.active_subscriptions',
|
|
632
|
-
stats.activeSubscriptionCount,
|
|
633
|
-
{ attributes }
|
|
634
|
-
);
|
|
635
|
-
distributionSyncMetric(
|
|
636
|
-
'sync.server.pull.revoked_subscriptions',
|
|
637
|
-
stats.revokedSubscriptionCount,
|
|
638
|
-
{ attributes }
|
|
639
|
-
);
|
|
640
|
-
distributionSyncMetric(
|
|
641
|
-
'sync.server.pull.bootstrap_subscriptions',
|
|
642
|
-
stats.bootstrapSubscriptionCount,
|
|
643
|
-
{ attributes }
|
|
644
|
-
);
|
|
645
|
-
distributionSyncMetric('sync.server.pull.commits', stats.commitCount, {
|
|
646
|
-
attributes,
|
|
528
|
+
const commits = await ctx.storage.readCommitWindow(ctx.partition, {
|
|
529
|
+
table: sub.table,
|
|
530
|
+
scopeFilter: plan.effective,
|
|
531
|
+
afterSeq: sub.cursor,
|
|
532
|
+
throughSeq: maxSeq,
|
|
533
|
+
limitChanges: limits.limitCommits + 1,
|
|
647
534
|
});
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
async function readLatestExternalCommitByTable<DB extends SyncCoreDb>(
|
|
659
|
-
trx: DbExecutor<DB>,
|
|
660
|
-
args: { partitionId: string; afterCursor: number; tables: string[] }
|
|
661
|
-
): Promise<Map<string, number>> {
|
|
662
|
-
const tableNames = Array.from(
|
|
663
|
-
new Set(args.tables.filter((table) => typeof table === 'string'))
|
|
664
|
-
);
|
|
665
|
-
const latestByTable = new Map<string, number>();
|
|
666
|
-
if (tableNames.length === 0) {
|
|
667
|
-
return latestByTable;
|
|
668
|
-
}
|
|
669
|
-
|
|
670
|
-
type SyncExecutor = Pick<Kysely<SyncCoreDb>, 'selectFrom'>;
|
|
671
|
-
const executor = trx as SyncExecutor;
|
|
672
|
-
const rows = await executor
|
|
673
|
-
.selectFrom('sync_table_commits as tc')
|
|
674
|
-
.innerJoin('sync_commits as cm', (join) =>
|
|
675
|
-
join
|
|
676
|
-
.onRef('cm.commit_seq', '=', 'tc.commit_seq')
|
|
677
|
-
.onRef('cm.partition_id', '=', 'tc.partition_id')
|
|
678
|
-
)
|
|
679
|
-
.select(['tc.table as table'])
|
|
680
|
-
.select((eb) => eb.fn.max('tc.commit_seq').as('latest_commit_seq'))
|
|
681
|
-
.where('tc.partition_id', '=', args.partitionId)
|
|
682
|
-
.where('cm.client_id', '=', EXTERNAL_CLIENT_ID)
|
|
683
|
-
.where('cm.change_count', '=', 0)
|
|
684
|
-
.where('tc.commit_seq', '>', args.afterCursor)
|
|
685
|
-
.where('tc.table', 'in', tableNames)
|
|
686
|
-
.groupBy('tc.table')
|
|
687
|
-
.execute();
|
|
688
|
-
|
|
689
|
-
for (const row of rows) {
|
|
690
|
-
const commitSeq = Number(row.latest_commit_seq ?? -1);
|
|
691
|
-
if (!Number.isFinite(commitSeq) || commitSeq < 0) continue;
|
|
692
|
-
latestByTable.set(row.table, commitSeq);
|
|
693
|
-
}
|
|
694
|
-
|
|
695
|
-
return latestByTable;
|
|
696
|
-
}
|
|
697
|
-
|
|
698
|
-
export async function pull<
|
|
699
|
-
DB extends SyncCoreDb,
|
|
700
|
-
Auth extends SyncServerAuth,
|
|
701
|
-
>(args: {
|
|
702
|
-
db: Kysely<DB>;
|
|
703
|
-
dialect: ServerSyncDialect;
|
|
704
|
-
handlers: ServerHandlerCollection<DB, Auth>;
|
|
705
|
-
auth: Auth;
|
|
706
|
-
request: SyncPullRequest;
|
|
707
|
-
/**
|
|
708
|
-
* Optional snapshot chunk storage adapter.
|
|
709
|
-
* When provided, stores chunk bodies in external storage (S3, etc.)
|
|
710
|
-
* instead of inline in the database.
|
|
711
|
-
*/
|
|
712
|
-
chunkStorage?: SnapshotChunkStorage;
|
|
713
|
-
/**
|
|
714
|
-
* Optional shared scope cache backend.
|
|
715
|
-
* Request-local memoization is always applied, even with custom backends.
|
|
716
|
-
* Defaults to process-local memory cache.
|
|
717
|
-
*/
|
|
718
|
-
scopeCache?: ScopeCacheBackend;
|
|
719
|
-
/**
|
|
720
|
-
* Gzip compression level for generated snapshot chunks. The protocol remains
|
|
721
|
-
* gzip-only; this tunes CPU/size tradeoffs for deployments that know their
|
|
722
|
-
* network constraints.
|
|
723
|
-
*
|
|
724
|
-
* Default: 1, range: 0-9.
|
|
725
|
-
*/
|
|
726
|
-
snapshotChunkGzipLevel?: number;
|
|
727
|
-
/**
|
|
728
|
-
* Schema/cache semantic version included in generated snapshot chunk cache
|
|
729
|
-
* keys. Changing this value invalidates cached bootstrap chunks without
|
|
730
|
-
* requiring table data to change.
|
|
731
|
-
*/
|
|
732
|
-
snapshotChunkCacheSchemaVersion?: number | string | null;
|
|
733
|
-
/**
|
|
734
|
-
* Optional server plugins for protocol-level pull transforms.
|
|
735
|
-
*/
|
|
736
|
-
plugins?: readonly SyncServerPullPlugin<DB, Auth>[];
|
|
737
|
-
}): Promise<PullResult> {
|
|
738
|
-
const { request, dialect } = args;
|
|
739
|
-
const db = args.db;
|
|
740
|
-
const pullPlugins = sortServerPullPlugins(args.plugins);
|
|
741
|
-
const partitionId = args.auth.partitionId ?? 'default';
|
|
742
|
-
const snapshotChunkGzipLevel = sanitizeGzipLevel(args.snapshotChunkGzipLevel);
|
|
743
|
-
const clientSchemaVersion = request.schemaVersion;
|
|
744
|
-
if (!Number.isInteger(clientSchemaVersion) || clientSchemaVersion < 1) {
|
|
745
|
-
throw new Error('Pull request schemaVersion must be a positive integer');
|
|
746
|
-
}
|
|
747
|
-
const snapshotChunkCacheSchemaVersion =
|
|
748
|
-
args.snapshotChunkCacheSchemaVersion === null ||
|
|
749
|
-
args.snapshotChunkCacheSchemaVersion === undefined
|
|
750
|
-
? clientSchemaVersion
|
|
751
|
-
: `${clientSchemaVersion}:${args.snapshotChunkCacheSchemaVersion}`;
|
|
752
|
-
const requestedSubscriptionCount = Array.isArray(request.subscriptions)
|
|
753
|
-
? request.subscriptions.length
|
|
754
|
-
: 0;
|
|
755
|
-
const startedAtMs = Date.now();
|
|
756
|
-
|
|
757
|
-
return startSyncSpan(
|
|
758
|
-
{
|
|
759
|
-
name: 'sync.server.pull',
|
|
760
|
-
op: 'sync.pull',
|
|
761
|
-
attributes: {
|
|
762
|
-
requested_subscription_count: requestedSubscriptionCount,
|
|
763
|
-
dedupe_rows: request.dedupeRows === true,
|
|
764
|
-
},
|
|
765
|
-
},
|
|
766
|
-
async (span) => {
|
|
767
|
-
try {
|
|
768
|
-
// Validate and sanitize request limits
|
|
769
|
-
const limitCommits = sanitizeLimit(request.limitCommits, 1000, 1, 1000);
|
|
770
|
-
const limitSnapshotRows = sanitizeLimit(
|
|
771
|
-
request.limitSnapshotRows,
|
|
772
|
-
1000,
|
|
773
|
-
1,
|
|
774
|
-
50000
|
|
775
|
-
);
|
|
776
|
-
const maxSnapshotPages = sanitizeLimit(
|
|
777
|
-
request.maxSnapshotPages,
|
|
778
|
-
4,
|
|
779
|
-
1,
|
|
780
|
-
50
|
|
781
|
-
);
|
|
782
|
-
const dedupeRows = request.dedupeRows === true;
|
|
783
|
-
const snapshotChunkEncoding = SYNC_SNAPSHOT_CHUNK_ENCODING;
|
|
784
|
-
const snapshotArtifactSelection = resolveSnapshotArtifactSelection(
|
|
785
|
-
request.snapshotArtifacts,
|
|
786
|
-
clientSchemaVersion
|
|
787
|
-
);
|
|
788
|
-
const snapshotArtifactSchemaVersion =
|
|
789
|
-
snapshotArtifactSelection?.schemaVersion ?? null;
|
|
790
|
-
// Resolve effective scopes for each subscription
|
|
791
|
-
const resolved = await resolveEffectiveScopesForSubscriptions({
|
|
792
|
-
db,
|
|
793
|
-
auth: args.auth,
|
|
794
|
-
subscriptions: request.subscriptions ?? [],
|
|
795
|
-
handlers: args.handlers,
|
|
796
|
-
scopeCache: args.scopeCache ?? defaultScopeCache,
|
|
797
|
-
});
|
|
798
|
-
|
|
799
|
-
for (
|
|
800
|
-
let attemptIndex = 0;
|
|
801
|
-
attemptIndex < MAX_PULL_TRANSACTION_RETRIES;
|
|
802
|
-
attemptIndex += 1
|
|
803
|
-
) {
|
|
804
|
-
const pendingExternalChunkWrites: PendingExternalChunkWrite[] = [];
|
|
805
|
-
const bootstrapTimings = createPullBootstrapTimings();
|
|
806
|
-
|
|
807
|
-
try {
|
|
808
|
-
const result = await dialect.executeInTransaction(
|
|
809
|
-
db,
|
|
810
|
-
async (trx) => {
|
|
811
|
-
await dialect.setRepeatableRead(trx);
|
|
812
|
-
|
|
813
|
-
const maxCommitSeq = await dialect.readMaxCommitSeq(trx, {
|
|
814
|
-
partitionId,
|
|
815
|
-
});
|
|
816
|
-
const minCommitSeq = await dialect.readMinCommitSeq(trx, {
|
|
817
|
-
partitionId,
|
|
818
|
-
});
|
|
819
|
-
|
|
820
|
-
const subResponses: SyncPullSubscriptionResponse[] = [];
|
|
821
|
-
const activeSubscriptions: { scopes: ScopeValues }[] = [];
|
|
822
|
-
const nextCursors: number[] = [];
|
|
823
|
-
|
|
824
|
-
// Detect external data changes (synthetic commits from notifyExternalDataChange)
|
|
825
|
-
// Compute minimum cursor across all active subscriptions to scope the query.
|
|
826
|
-
let minSubCursor = Number.MAX_SAFE_INTEGER;
|
|
827
|
-
const activeTables = new Set<string>();
|
|
828
|
-
for (const sub of resolved) {
|
|
829
|
-
if (
|
|
830
|
-
sub.status === 'revoked' ||
|
|
831
|
-
Object.keys(sub.scopes).length === 0
|
|
832
|
-
)
|
|
833
|
-
continue;
|
|
834
|
-
activeTables.add(sub.table);
|
|
835
|
-
const cursor = Math.max(-1, sub.cursor ?? -1);
|
|
836
|
-
if (cursor >= 0 && cursor < minSubCursor) {
|
|
837
|
-
minSubCursor = cursor;
|
|
838
|
-
}
|
|
839
|
-
}
|
|
840
|
-
|
|
841
|
-
const maxExternalCommitByTable =
|
|
842
|
-
minSubCursor < Number.MAX_SAFE_INTEGER && minSubCursor >= 0
|
|
843
|
-
? await readLatestExternalCommitByTable(trx, {
|
|
844
|
-
partitionId,
|
|
845
|
-
afterCursor: minSubCursor,
|
|
846
|
-
tables: Array.from(activeTables),
|
|
847
|
-
})
|
|
848
|
-
: new Map<string, number>();
|
|
849
|
-
|
|
850
|
-
for (const sub of resolved) {
|
|
851
|
-
const cursor = Math.max(-1, sub.cursor ?? -1);
|
|
852
|
-
// Validate table handler exists (throws if not registered)
|
|
853
|
-
if (!args.handlers.byTable.has(sub.table)) {
|
|
854
|
-
throw new Error(`Unknown table: ${sub.table}`);
|
|
855
|
-
}
|
|
856
|
-
|
|
857
|
-
if (
|
|
858
|
-
sub.status === 'revoked' ||
|
|
859
|
-
Object.keys(sub.scopes).length === 0
|
|
860
|
-
) {
|
|
861
|
-
subResponses.push({
|
|
862
|
-
id: sub.id,
|
|
863
|
-
status: 'revoked',
|
|
864
|
-
scopes: {},
|
|
865
|
-
bootstrap: false,
|
|
866
|
-
nextCursor: cursor,
|
|
867
|
-
commits: [],
|
|
868
|
-
});
|
|
869
|
-
continue;
|
|
870
|
-
}
|
|
871
|
-
|
|
872
|
-
const effectiveScopes = sub.scopes;
|
|
873
|
-
activeSubscriptions.push({ scopes: effectiveScopes });
|
|
874
|
-
const latestExternalCommitForTable =
|
|
875
|
-
maxExternalCommitByTable.get(sub.table);
|
|
876
|
-
|
|
877
|
-
const needsBootstrap =
|
|
878
|
-
sub.bootstrapState != null ||
|
|
879
|
-
cursor < 0 ||
|
|
880
|
-
cursor > maxCommitSeq ||
|
|
881
|
-
(minCommitSeq > 0 && cursor < minCommitSeq - 1) ||
|
|
882
|
-
(latestExternalCommitForTable !== undefined &&
|
|
883
|
-
latestExternalCommitForTable > cursor);
|
|
884
|
-
|
|
885
|
-
if (needsBootstrap) {
|
|
886
|
-
const tables = getServerBootstrapOrderFor(
|
|
887
|
-
args.handlers,
|
|
888
|
-
sub.table
|
|
889
|
-
).map((handler) => handler.table);
|
|
890
|
-
const initState: SyncBootstrapState = {
|
|
891
|
-
asOfCommitSeq: maxCommitSeq,
|
|
892
|
-
tables,
|
|
893
|
-
tableIndex: 0,
|
|
894
|
-
rowCursor: null,
|
|
895
|
-
};
|
|
896
|
-
|
|
897
|
-
const requestedState = sub.bootstrapState ?? null;
|
|
898
|
-
const state =
|
|
899
|
-
requestedState &&
|
|
900
|
-
typeof requestedState.asOfCommitSeq === 'number' &&
|
|
901
|
-
Array.isArray(requestedState.tables) &&
|
|
902
|
-
typeof requestedState.tableIndex === 'number'
|
|
903
|
-
? (requestedState as SyncBootstrapState)
|
|
904
|
-
: initState;
|
|
905
|
-
|
|
906
|
-
// If the bootstrap state's asOfCommitSeq is no longer catch-up-able, restart bootstrap.
|
|
907
|
-
const effectiveState =
|
|
908
|
-
state.asOfCommitSeq < minCommitSeq - 1
|
|
909
|
-
? initState
|
|
910
|
-
: state;
|
|
911
|
-
|
|
912
|
-
const tableName =
|
|
913
|
-
effectiveState.tables[effectiveState.tableIndex];
|
|
914
|
-
|
|
915
|
-
// No tables (or ran past the end): treat bootstrap as complete.
|
|
916
|
-
if (!tableName) {
|
|
917
|
-
subResponses.push({
|
|
918
|
-
id: sub.id,
|
|
919
|
-
status: 'active',
|
|
920
|
-
scopes: effectiveScopes,
|
|
921
|
-
bootstrap: true,
|
|
922
|
-
bootstrapState: null,
|
|
923
|
-
nextCursor: effectiveState.asOfCommitSeq,
|
|
924
|
-
commits: [],
|
|
925
|
-
snapshots: [],
|
|
926
|
-
});
|
|
927
|
-
nextCursors.push(effectiveState.asOfCommitSeq);
|
|
928
|
-
continue;
|
|
929
|
-
}
|
|
930
|
-
|
|
931
|
-
const snapshots: SyncSnapshot[] = [];
|
|
932
|
-
let nextState: SyncBootstrapState | null = effectiveState;
|
|
933
|
-
const cacheKey = await createSnapshotChunkScopeCacheKey({
|
|
934
|
-
partitionId,
|
|
935
|
-
scopes: effectiveScopes,
|
|
936
|
-
schemaVersion: snapshotChunkCacheSchemaVersion,
|
|
937
|
-
encoding: snapshotChunkEncoding,
|
|
938
|
-
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
939
|
-
gzipLevel: snapshotChunkGzipLevel,
|
|
940
|
-
});
|
|
941
|
-
const artifactScopeKey =
|
|
942
|
-
snapshotArtifactSelection && snapshotArtifactSchemaVersion
|
|
943
|
-
? await createScopedSnapshotArtifactScopeCacheKey({
|
|
944
|
-
partitionId,
|
|
945
|
-
subscriptionId: sub.id,
|
|
946
|
-
scopes: effectiveScopes,
|
|
947
|
-
schemaVersion: snapshotArtifactSchemaVersion,
|
|
948
|
-
artifactKind:
|
|
949
|
-
snapshotArtifactSelection.artifactKind,
|
|
950
|
-
compression: snapshotArtifactSelection.compression,
|
|
951
|
-
features: snapshotArtifactSelection.featureSet,
|
|
952
|
-
})
|
|
953
|
-
: null;
|
|
954
|
-
|
|
955
|
-
interface SnapshotBundle {
|
|
956
|
-
table: string;
|
|
957
|
-
tableIndex: number;
|
|
958
|
-
startCursor: string | null;
|
|
959
|
-
nextRowCursor: string | null;
|
|
960
|
-
isFirstPage: boolean;
|
|
961
|
-
isLastPage: boolean;
|
|
962
|
-
pageCount: number;
|
|
963
|
-
cacheRowLimit: number | null;
|
|
964
|
-
ttlMs: number;
|
|
965
|
-
binaryColumns?: readonly BinarySnapshotColumn[];
|
|
966
|
-
binaryEncoder?: BinarySnapshotRowsEncoder;
|
|
967
|
-
binaryRows: unknown[];
|
|
968
|
-
}
|
|
969
|
-
|
|
970
|
-
const createSnapshotBundle = (
|
|
971
|
-
table: string,
|
|
972
|
-
tableIndex: number,
|
|
973
|
-
rowCursor: string | null,
|
|
974
|
-
ttlMs: number,
|
|
975
|
-
binaryColumns?: readonly BinarySnapshotColumn[],
|
|
976
|
-
binaryEncoder?: BinarySnapshotRowsEncoder
|
|
977
|
-
): SnapshotBundle => {
|
|
978
|
-
return {
|
|
979
|
-
table,
|
|
980
|
-
tableIndex,
|
|
981
|
-
startCursor: rowCursor,
|
|
982
|
-
nextRowCursor: null,
|
|
983
|
-
isFirstPage: rowCursor == null,
|
|
984
|
-
isLastPage: false,
|
|
985
|
-
pageCount: 0,
|
|
986
|
-
cacheRowLimit: null,
|
|
987
|
-
ttlMs,
|
|
988
|
-
binaryColumns,
|
|
989
|
-
binaryEncoder,
|
|
990
|
-
binaryRows: [],
|
|
991
|
-
};
|
|
992
|
-
};
|
|
993
|
-
|
|
994
|
-
const snapshotBootstrapStateAfter = (args: {
|
|
995
|
-
tableIndex: number;
|
|
996
|
-
nextRowCursor: string | null;
|
|
997
|
-
isLastPage: boolean;
|
|
998
|
-
}): SyncBootstrapState | null => {
|
|
999
|
-
if (!args.isLastPage) {
|
|
1000
|
-
return {
|
|
1001
|
-
...effectiveState,
|
|
1002
|
-
tableIndex: args.tableIndex,
|
|
1003
|
-
rowCursor: args.nextRowCursor,
|
|
1004
|
-
};
|
|
1005
|
-
}
|
|
1006
|
-
if (args.tableIndex + 1 < effectiveState.tables.length) {
|
|
1007
|
-
return {
|
|
1008
|
-
...effectiveState,
|
|
1009
|
-
tableIndex: args.tableIndex + 1,
|
|
1010
|
-
rowCursor: null,
|
|
1011
|
-
};
|
|
1012
|
-
}
|
|
1013
|
-
return null;
|
|
1014
|
-
};
|
|
1015
|
-
|
|
1016
|
-
const encodeSnapshotBundlePayload = (
|
|
1017
|
-
bundle: SnapshotBundle
|
|
1018
|
-
): Uint8Array[] => {
|
|
1019
|
-
const encodeStartedAt = Date.now();
|
|
1020
|
-
const payload = bundle.binaryEncoder
|
|
1021
|
-
? bundle.binaryEncoder(bundle.binaryRows)
|
|
1022
|
-
: encodeBinarySnapshotRows(
|
|
1023
|
-
bundle.table,
|
|
1024
|
-
bundle.binaryRows,
|
|
1025
|
-
bundle.binaryColumns
|
|
1026
|
-
);
|
|
1027
|
-
bootstrapTimings.binaryEncodeMs += Math.max(
|
|
1028
|
-
0,
|
|
1029
|
-
Date.now() - encodeStartedAt
|
|
1030
|
-
);
|
|
1031
|
-
return [payload];
|
|
1032
|
-
};
|
|
1033
|
-
|
|
1034
|
-
const flushSnapshotBundle = async (
|
|
1035
|
-
bundle: SnapshotBundle
|
|
1036
|
-
): Promise<void> => {
|
|
1037
|
-
const nowIso = new Date().toISOString();
|
|
1038
|
-
const bundleRowLimit = Math.max(
|
|
1039
|
-
1,
|
|
1040
|
-
bundle.cacheRowLimit ??
|
|
1041
|
-
limitSnapshotRows * bundle.pageCount
|
|
1042
|
-
);
|
|
1043
|
-
|
|
1044
|
-
const cacheLookupStartedAt = Date.now();
|
|
1045
|
-
const cached = await readSnapshotChunkRefByPageKey(trx, {
|
|
1046
|
-
partitionId,
|
|
1047
|
-
scopeKey: cacheKey,
|
|
1048
|
-
scope: bundle.table,
|
|
1049
|
-
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
1050
|
-
rowCursor: bundle.startCursor,
|
|
1051
|
-
rowLimit: bundleRowLimit,
|
|
1052
|
-
encoding: snapshotChunkEncoding,
|
|
1053
|
-
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
1054
|
-
nowIso,
|
|
1055
|
-
});
|
|
1056
|
-
bootstrapTimings.chunkCacheLookupMs += Math.max(
|
|
1057
|
-
0,
|
|
1058
|
-
Date.now() - cacheLookupStartedAt
|
|
1059
|
-
);
|
|
1060
|
-
|
|
1061
|
-
let chunkRef: SyncSnapshotChunkRef | null = cached;
|
|
1062
|
-
if (!chunkRef) {
|
|
1063
|
-
const expiresAt = new Date(
|
|
1064
|
-
Date.now() + Math.max(1000, bundle.ttlMs)
|
|
1065
|
-
).toISOString();
|
|
1066
|
-
|
|
1067
|
-
if (args.chunkStorage) {
|
|
1068
|
-
const snapshot: SyncSnapshot = {
|
|
1069
|
-
table: bundle.table,
|
|
1070
|
-
rows: [],
|
|
1071
|
-
chunks: [],
|
|
1072
|
-
isFirstPage: bundle.isFirstPage,
|
|
1073
|
-
isLastPage: bundle.isLastPage,
|
|
1074
|
-
bootstrapStateAfter: snapshotBootstrapStateAfter({
|
|
1075
|
-
tableIndex: bundle.tableIndex,
|
|
1076
|
-
nextRowCursor: bundle.nextRowCursor,
|
|
1077
|
-
isLastPage: bundle.isLastPage,
|
|
1078
|
-
}),
|
|
1079
|
-
};
|
|
1080
|
-
snapshots.push(snapshot);
|
|
1081
|
-
pendingExternalChunkWrites.push({
|
|
1082
|
-
snapshot,
|
|
1083
|
-
cacheLookup: {
|
|
1084
|
-
partitionId,
|
|
1085
|
-
scopeKey: cacheKey,
|
|
1086
|
-
scope: bundle.table,
|
|
1087
|
-
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
1088
|
-
rowCursor: bundle.startCursor,
|
|
1089
|
-
rowLimit: bundleRowLimit,
|
|
1090
|
-
nextRowCursor: bundle.nextRowCursor,
|
|
1091
|
-
isLastPage: bundle.isLastPage,
|
|
1092
|
-
},
|
|
1093
|
-
payloadParts: encodeSnapshotBundlePayload(bundle),
|
|
1094
|
-
expiresAt,
|
|
1095
|
-
});
|
|
1096
|
-
return;
|
|
1097
|
-
}
|
|
1098
|
-
const encodedChunk =
|
|
1099
|
-
await encodeCompressedSnapshotChunk(
|
|
1100
|
-
encodeSnapshotBundlePayload(bundle),
|
|
1101
|
-
snapshotChunkGzipLevel
|
|
1102
|
-
);
|
|
1103
|
-
bootstrapTimings.chunkGzipMs += encodedChunk.gzipMs;
|
|
1104
|
-
bootstrapTimings.chunkHashMs += encodedChunk.hashMs;
|
|
1105
|
-
const chunkId = randomId();
|
|
1106
|
-
const chunkPersistStartedAt = Date.now();
|
|
1107
|
-
chunkRef = await insertSnapshotChunk(trx, {
|
|
1108
|
-
chunkId,
|
|
1109
|
-
partitionId,
|
|
1110
|
-
scopeKey: cacheKey,
|
|
1111
|
-
scope: bundle.table,
|
|
1112
|
-
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
1113
|
-
rowCursor: bundle.startCursor,
|
|
1114
|
-
rowLimit: bundleRowLimit,
|
|
1115
|
-
nextRowCursor: bundle.nextRowCursor,
|
|
1116
|
-
isLastPage: bundle.isLastPage,
|
|
1117
|
-
encoding: snapshotChunkEncoding,
|
|
1118
|
-
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
1119
|
-
sha256: encodedChunk.sha256,
|
|
1120
|
-
body: encodedChunk.body,
|
|
1121
|
-
expiresAt,
|
|
1122
|
-
});
|
|
1123
|
-
bootstrapTimings.chunkPersistMs += Math.max(
|
|
1124
|
-
0,
|
|
1125
|
-
Date.now() - chunkPersistStartedAt
|
|
1126
|
-
);
|
|
1127
|
-
}
|
|
1128
|
-
|
|
1129
|
-
const chunk = toResponseChunkRef(chunkRef);
|
|
1130
|
-
snapshots.push({
|
|
1131
|
-
table: bundle.table,
|
|
1132
|
-
rows: [],
|
|
1133
|
-
chunks: [chunk],
|
|
1134
|
-
manifest: await createChunkedSnapshotManifest({
|
|
1135
|
-
table: bundle.table,
|
|
1136
|
-
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
1137
|
-
scopeKey: cacheKey,
|
|
1138
|
-
rowCursor: bundle.startCursor,
|
|
1139
|
-
rowLimit: bundleRowLimit,
|
|
1140
|
-
nextRowCursor: bundle.nextRowCursor,
|
|
1141
|
-
isFirstPage: bundle.isFirstPage,
|
|
1142
|
-
isLastPage: bundle.isLastPage,
|
|
1143
|
-
chunks: [chunk],
|
|
1144
|
-
}),
|
|
1145
|
-
isFirstPage: bundle.isFirstPage,
|
|
1146
|
-
isLastPage: bundle.isLastPage,
|
|
1147
|
-
bootstrapStateAfter: snapshotBootstrapStateAfter({
|
|
1148
|
-
tableIndex: bundle.tableIndex,
|
|
1149
|
-
nextRowCursor: bundle.nextRowCursor,
|
|
1150
|
-
isLastPage: bundle.isLastPage,
|
|
1151
|
-
}),
|
|
1152
|
-
});
|
|
1153
|
-
};
|
|
1154
|
-
|
|
1155
|
-
let activeBundle: SnapshotBundle | null = null;
|
|
1156
|
-
|
|
1157
|
-
for (
|
|
1158
|
-
let pageIndex = 0;
|
|
1159
|
-
pageIndex < maxSnapshotPages;
|
|
1160
|
-
pageIndex++
|
|
1161
|
-
) {
|
|
1162
|
-
if (!nextState) break;
|
|
1163
|
-
|
|
1164
|
-
const nextTableName: string | undefined =
|
|
1165
|
-
nextState.tables[nextState.tableIndex];
|
|
1166
|
-
if (!nextTableName) {
|
|
1167
|
-
if (activeBundle) {
|
|
1168
|
-
activeBundle.isLastPage = true;
|
|
1169
|
-
await flushSnapshotBundle(activeBundle);
|
|
1170
|
-
activeBundle = null;
|
|
1171
|
-
}
|
|
1172
|
-
nextState = null;
|
|
1173
|
-
break;
|
|
1174
|
-
}
|
|
1175
|
-
|
|
1176
|
-
const tableHandler =
|
|
1177
|
-
args.handlers.byTable.get(nextTableName);
|
|
1178
|
-
if (!tableHandler) {
|
|
1179
|
-
throw new Error(`Unknown table: ${nextTableName}`);
|
|
1180
|
-
}
|
|
1181
|
-
if (
|
|
1182
|
-
!activeBundle ||
|
|
1183
|
-
activeBundle.table !== nextTableName
|
|
1184
|
-
) {
|
|
1185
|
-
if (activeBundle) {
|
|
1186
|
-
await flushSnapshotBundle(activeBundle);
|
|
1187
|
-
}
|
|
1188
|
-
activeBundle = createSnapshotBundle(
|
|
1189
|
-
nextTableName,
|
|
1190
|
-
nextState.tableIndex,
|
|
1191
|
-
nextState.rowCursor,
|
|
1192
|
-
tableHandler.snapshotChunkTtlMs ??
|
|
1193
|
-
24 * 60 * 60 * 1000,
|
|
1194
|
-
resolveSnapshotBinaryColumns(
|
|
1195
|
-
tableHandler,
|
|
1196
|
-
clientSchemaVersion
|
|
1197
|
-
),
|
|
1198
|
-
resolveSnapshotBinaryEncoder(
|
|
1199
|
-
tableHandler,
|
|
1200
|
-
clientSchemaVersion
|
|
1201
|
-
)
|
|
1202
|
-
);
|
|
1203
|
-
}
|
|
1204
|
-
|
|
1205
|
-
if (artifactScopeKey && activeBundle.pageCount === 0) {
|
|
1206
|
-
const pagesRemaining = Math.max(
|
|
1207
|
-
1,
|
|
1208
|
-
maxSnapshotPages - pageIndex
|
|
1209
|
-
);
|
|
1210
|
-
const artifactRowLimit =
|
|
1211
|
-
resolveBinarySnapshotBundleRowLimit({
|
|
1212
|
-
limitSnapshotRows,
|
|
1213
|
-
pagesRemaining,
|
|
1214
|
-
});
|
|
1215
|
-
const artifactLookupStartedAt = Date.now();
|
|
1216
|
-
const artifact =
|
|
1217
|
-
await readBestScopedSnapshotArtifactRefForPageCapacity(
|
|
1218
|
-
trx,
|
|
1219
|
-
{
|
|
1220
|
-
partitionId,
|
|
1221
|
-
scopeKey: artifactScopeKey,
|
|
1222
|
-
subscriptionId: sub.id,
|
|
1223
|
-
table: nextTableName,
|
|
1224
|
-
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
1225
|
-
rowCursor: nextState.rowCursor,
|
|
1226
|
-
maxRowLimit: artifactRowLimit,
|
|
1227
|
-
artifactKind:
|
|
1228
|
-
snapshotArtifactSelection!.artifactKind,
|
|
1229
|
-
schemaVersion: snapshotArtifactSchemaVersion!,
|
|
1230
|
-
compression:
|
|
1231
|
-
snapshotArtifactSelection!.compression,
|
|
1232
|
-
}
|
|
1233
|
-
);
|
|
1234
|
-
bootstrapTimings.artifactCacheLookupMs += Math.max(
|
|
1235
|
-
0,
|
|
1236
|
-
Date.now() - artifactLookupStartedAt
|
|
1237
|
-
);
|
|
1238
|
-
|
|
1239
|
-
if (
|
|
1240
|
-
artifact &&
|
|
1241
|
-
(artifact.isLastPage ||
|
|
1242
|
-
artifact.nextRowCursor !== null)
|
|
1243
|
-
) {
|
|
1244
|
-
snapshots.push({
|
|
1245
|
-
table: nextTableName,
|
|
1246
|
-
rows: [],
|
|
1247
|
-
artifacts: [artifact],
|
|
1248
|
-
isFirstPage: artifact.isFirstPage,
|
|
1249
|
-
isLastPage: artifact.isLastPage,
|
|
1250
|
-
bootstrapStateAfter: snapshotBootstrapStateAfter({
|
|
1251
|
-
tableIndex: nextState.tableIndex,
|
|
1252
|
-
nextRowCursor: artifact.nextRowCursor,
|
|
1253
|
-
isLastPage: artifact.isLastPage,
|
|
1254
|
-
}),
|
|
1255
|
-
});
|
|
1256
|
-
activeBundle = null;
|
|
1257
|
-
const selectedArtifactRowLimit =
|
|
1258
|
-
artifact.manifest.rowLimit;
|
|
1259
|
-
pageIndex +=
|
|
1260
|
-
Math.max(
|
|
1261
|
-
1,
|
|
1262
|
-
Math.ceil(
|
|
1263
|
-
selectedArtifactRowLimit / limitSnapshotRows
|
|
1264
|
-
)
|
|
1265
|
-
) - 1;
|
|
1266
|
-
|
|
1267
|
-
nextState = snapshotBootstrapStateAfter({
|
|
1268
|
-
tableIndex: nextState.tableIndex,
|
|
1269
|
-
nextRowCursor: artifact.nextRowCursor,
|
|
1270
|
-
isLastPage: artifact.isLastPage,
|
|
1271
|
-
});
|
|
1272
|
-
continue;
|
|
1273
|
-
}
|
|
1274
|
-
}
|
|
1275
|
-
|
|
1276
|
-
if (activeBundle.pageCount === 0) {
|
|
1277
|
-
const pagesRemaining = Math.max(
|
|
1278
|
-
1,
|
|
1279
|
-
maxSnapshotPages - pageIndex
|
|
1280
|
-
);
|
|
1281
|
-
const cachedRowLimit =
|
|
1282
|
-
resolveBinarySnapshotBundleRowLimit({
|
|
1283
|
-
limitSnapshotRows,
|
|
1284
|
-
pagesRemaining,
|
|
1285
|
-
});
|
|
1286
|
-
activeBundle.cacheRowLimit = cachedRowLimit;
|
|
1287
|
-
const cacheLookupStartedAt = Date.now();
|
|
1288
|
-
const cached: SnapshotChunkRefWithContinuation | null =
|
|
1289
|
-
await readSnapshotChunkRefByPageKey(trx, {
|
|
1290
|
-
partitionId,
|
|
1291
|
-
scopeKey: cacheKey,
|
|
1292
|
-
scope: nextTableName,
|
|
1293
|
-
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
1294
|
-
rowCursor: nextState.rowCursor,
|
|
1295
|
-
rowLimit: cachedRowLimit,
|
|
1296
|
-
encoding: snapshotChunkEncoding,
|
|
1297
|
-
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
1298
|
-
});
|
|
1299
|
-
bootstrapTimings.chunkCacheLookupMs += Math.max(
|
|
1300
|
-
0,
|
|
1301
|
-
Date.now() - cacheLookupStartedAt
|
|
1302
|
-
);
|
|
1303
|
-
|
|
1304
|
-
if (
|
|
1305
|
-
cached &&
|
|
1306
|
-
(cached.isLastPage || cached.nextRowCursor !== null)
|
|
1307
|
-
) {
|
|
1308
|
-
const chunk = toResponseChunkRef(cached);
|
|
1309
|
-
snapshots.push({
|
|
1310
|
-
table: nextTableName,
|
|
1311
|
-
rows: [],
|
|
1312
|
-
chunks: [chunk],
|
|
1313
|
-
manifest: await createChunkedSnapshotManifest({
|
|
1314
|
-
table: nextTableName,
|
|
1315
|
-
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
1316
|
-
scopeKey: cacheKey,
|
|
1317
|
-
rowCursor: nextState.rowCursor,
|
|
1318
|
-
rowLimit: cachedRowLimit,
|
|
1319
|
-
nextRowCursor: cached.nextRowCursor,
|
|
1320
|
-
isFirstPage: nextState.rowCursor == null,
|
|
1321
|
-
isLastPage: cached.isLastPage,
|
|
1322
|
-
chunks: [chunk],
|
|
1323
|
-
}),
|
|
1324
|
-
isFirstPage: nextState.rowCursor == null,
|
|
1325
|
-
isLastPage: cached.isLastPage,
|
|
1326
|
-
bootstrapStateAfter: snapshotBootstrapStateAfter({
|
|
1327
|
-
tableIndex: nextState.tableIndex,
|
|
1328
|
-
nextRowCursor: cached.nextRowCursor,
|
|
1329
|
-
isLastPage: cached.isLastPage,
|
|
1330
|
-
}),
|
|
1331
|
-
});
|
|
1332
|
-
activeBundle = null;
|
|
1333
|
-
pageIndex +=
|
|
1334
|
-
Math.max(
|
|
1335
|
-
1,
|
|
1336
|
-
Math.ceil(cachedRowLimit / limitSnapshotRows)
|
|
1337
|
-
) - 1;
|
|
1338
|
-
|
|
1339
|
-
nextState = snapshotBootstrapStateAfter({
|
|
1340
|
-
tableIndex: nextState.tableIndex,
|
|
1341
|
-
nextRowCursor: cached.nextRowCursor,
|
|
1342
|
-
isLastPage: cached.isLastPage,
|
|
1343
|
-
});
|
|
1344
|
-
continue;
|
|
1345
|
-
}
|
|
1346
|
-
}
|
|
1347
|
-
|
|
1348
|
-
const snapshotQueryStartedAt = Date.now();
|
|
1349
|
-
const page: {
|
|
1350
|
-
rows: unknown[];
|
|
1351
|
-
nextCursor: string | null;
|
|
1352
|
-
} = await tableHandler.snapshot(
|
|
1353
|
-
{
|
|
1354
|
-
db: trx,
|
|
1355
|
-
actorId: args.auth.actorId,
|
|
1356
|
-
auth: args.auth,
|
|
1357
|
-
scopeValues: effectiveScopes,
|
|
1358
|
-
cursor: nextState.rowCursor,
|
|
1359
|
-
limit: limitSnapshotRows,
|
|
1360
|
-
schemaVersion: clientSchemaVersion,
|
|
1361
|
-
},
|
|
1362
|
-
sub.params
|
|
1363
|
-
);
|
|
1364
|
-
bootstrapTimings.snapshotQueryMs += Math.max(
|
|
1365
|
-
0,
|
|
1366
|
-
Date.now() - snapshotQueryStartedAt
|
|
1367
|
-
);
|
|
1368
|
-
|
|
1369
|
-
const pageRows = page.rows ?? [];
|
|
1370
|
-
activeBundle.nextRowCursor = page.nextCursor;
|
|
1371
|
-
activeBundle.binaryRows.push(...pageRows);
|
|
1372
|
-
activeBundle.pageCount += 1;
|
|
1373
|
-
|
|
1374
|
-
if (page.nextCursor != null) {
|
|
1375
|
-
const shouldFlushBinaryBundle =
|
|
1376
|
-
activeBundle.binaryRows.length >=
|
|
1377
|
-
DEFAULT_MAX_BINARY_SNAPSHOT_BUNDLE_ROWS;
|
|
1378
|
-
if (shouldFlushBinaryBundle) {
|
|
1379
|
-
await flushSnapshotBundle(activeBundle);
|
|
1380
|
-
activeBundle = null;
|
|
1381
|
-
}
|
|
1382
|
-
nextState = {
|
|
1383
|
-
...nextState,
|
|
1384
|
-
rowCursor: page.nextCursor,
|
|
1385
|
-
};
|
|
1386
|
-
continue;
|
|
1387
|
-
}
|
|
1388
|
-
|
|
1389
|
-
activeBundle.isLastPage = true;
|
|
1390
|
-
await flushSnapshotBundle(activeBundle);
|
|
1391
|
-
activeBundle = null;
|
|
1392
|
-
|
|
1393
|
-
if (nextState.tableIndex + 1 < nextState.tables.length) {
|
|
1394
|
-
nextState = {
|
|
1395
|
-
...nextState,
|
|
1396
|
-
tableIndex: nextState.tableIndex + 1,
|
|
1397
|
-
rowCursor: null,
|
|
1398
|
-
};
|
|
1399
|
-
continue;
|
|
1400
|
-
}
|
|
1401
|
-
|
|
1402
|
-
nextState = null;
|
|
1403
|
-
break;
|
|
1404
|
-
}
|
|
1405
|
-
|
|
1406
|
-
if (activeBundle) {
|
|
1407
|
-
await flushSnapshotBundle(activeBundle);
|
|
1408
|
-
}
|
|
1409
|
-
|
|
1410
|
-
subResponses.push({
|
|
1411
|
-
id: sub.id,
|
|
1412
|
-
status: 'active',
|
|
1413
|
-
scopes: effectiveScopes,
|
|
1414
|
-
bootstrap: true,
|
|
1415
|
-
bootstrapState: nextState,
|
|
1416
|
-
nextCursor: effectiveState.asOfCommitSeq,
|
|
1417
|
-
commits: [],
|
|
1418
|
-
snapshots,
|
|
1419
|
-
});
|
|
1420
|
-
nextCursors.push(effectiveState.asOfCommitSeq);
|
|
1421
|
-
continue;
|
|
1422
|
-
}
|
|
1423
|
-
|
|
1424
|
-
// Incremental pull for this subscription. The dialect row query
|
|
1425
|
-
// carries the scanned commit-window max when matching rows exist,
|
|
1426
|
-
// so we only need a separate commit-window scan when the row query
|
|
1427
|
-
// returns no matches at all.
|
|
1428
|
-
const incrementalRows: IncrementalPullRow[] = [];
|
|
1429
|
-
let maxScannedCommitSeq = cursor;
|
|
1430
|
-
|
|
1431
|
-
for await (const row of dialect.iterateIncrementalPullRows(
|
|
1432
|
-
trx,
|
|
1433
|
-
{
|
|
1434
|
-
partitionId,
|
|
1435
|
-
table: sub.table,
|
|
1436
|
-
scopes: effectiveScopes,
|
|
1437
|
-
cursor,
|
|
1438
|
-
limitCommits,
|
|
1439
|
-
}
|
|
1440
|
-
)) {
|
|
1441
|
-
incrementalRows.push(row);
|
|
1442
|
-
maxScannedCommitSeq = Math.max(
|
|
1443
|
-
maxScannedCommitSeq,
|
|
1444
|
-
row.scanned_max_commit_seq ?? row.commit_seq
|
|
1445
|
-
);
|
|
1446
|
-
}
|
|
1447
|
-
|
|
1448
|
-
if (incrementalRows.length === 0) {
|
|
1449
|
-
const scannedCommitSeqs =
|
|
1450
|
-
await dialect.readCommitSeqsForPull(trx, {
|
|
1451
|
-
partitionId,
|
|
1452
|
-
cursor,
|
|
1453
|
-
limitCommits,
|
|
1454
|
-
tables: [sub.table],
|
|
1455
|
-
});
|
|
1456
|
-
maxScannedCommitSeq =
|
|
1457
|
-
scannedCommitSeqs.length > 0
|
|
1458
|
-
? scannedCommitSeqs[scannedCommitSeqs.length - 1]!
|
|
1459
|
-
: cursor;
|
|
1460
|
-
|
|
1461
|
-
if (scannedCommitSeqs.length === 0) {
|
|
1462
|
-
subResponses.push({
|
|
1463
|
-
id: sub.id,
|
|
1464
|
-
status: 'active',
|
|
1465
|
-
scopes: effectiveScopes,
|
|
1466
|
-
bootstrap: false,
|
|
1467
|
-
nextCursor: cursor,
|
|
1468
|
-
commits: [],
|
|
1469
|
-
});
|
|
1470
|
-
nextCursors.push(cursor);
|
|
1471
|
-
continue;
|
|
1472
|
-
}
|
|
1473
|
-
}
|
|
1474
|
-
|
|
1475
|
-
const tableHandler = args.handlers.byTable.get(sub.table);
|
|
1476
|
-
if (!tableHandler) {
|
|
1477
|
-
throw new Error(`Unknown table: ${sub.table}`);
|
|
1478
|
-
}
|
|
1479
|
-
|
|
1480
|
-
const incrementalItems = incrementalRows.map((r) => ({
|
|
1481
|
-
commitSeq: r.commit_seq,
|
|
1482
|
-
createdAt: r.created_at,
|
|
1483
|
-
actorId: r.actor_id,
|
|
1484
|
-
change: projectPullChangeForClientSchema({
|
|
1485
|
-
handler: tableHandler,
|
|
1486
|
-
schemaVersion: clientSchemaVersion,
|
|
1487
|
-
change: {
|
|
1488
|
-
table: r.table,
|
|
1489
|
-
row_id: r.row_id,
|
|
1490
|
-
op: r.op,
|
|
1491
|
-
row_json: r.row_json,
|
|
1492
|
-
row_version: r.row_version,
|
|
1493
|
-
scopes: r.scopes,
|
|
1494
|
-
} satisfies SyncChange,
|
|
1495
|
-
}),
|
|
1496
|
-
}));
|
|
1497
|
-
|
|
1498
|
-
if (pullPlugins.length > 0 && incrementalItems.length > 0) {
|
|
1499
|
-
const transformedChanges = await transformPullChanges({
|
|
1500
|
-
plugins: pullPlugins,
|
|
1501
|
-
ctx: {
|
|
1502
|
-
db: trx,
|
|
1503
|
-
actorId: args.auth.actorId,
|
|
1504
|
-
auth: args.auth,
|
|
1505
|
-
},
|
|
1506
|
-
tableHandler,
|
|
1507
|
-
subscription: {
|
|
1508
|
-
id: sub.id,
|
|
1509
|
-
table: sub.table,
|
|
1510
|
-
scopes: effectiveScopes,
|
|
1511
|
-
params: sub.params,
|
|
1512
|
-
cursor,
|
|
1513
|
-
crdtStateVectors: sub.crdtStateVectors,
|
|
1514
|
-
},
|
|
1515
|
-
changes: incrementalItems.map((item) => item.change),
|
|
1516
|
-
});
|
|
1517
|
-
for (let i = 0; i < incrementalItems.length; i += 1) {
|
|
1518
|
-
incrementalItems[i]!.change = transformedChanges[i]!;
|
|
1519
|
-
}
|
|
1520
|
-
}
|
|
1521
|
-
|
|
1522
|
-
let nextCursor = cursor;
|
|
1523
|
-
|
|
1524
|
-
if (dedupeRows) {
|
|
1525
|
-
const latestByRowKey = new Map<
|
|
1526
|
-
string,
|
|
1527
|
-
{
|
|
1528
|
-
commitSeq: number;
|
|
1529
|
-
createdAt: string;
|
|
1530
|
-
actorId: string;
|
|
1531
|
-
change: SyncChange;
|
|
1532
|
-
}
|
|
1533
|
-
>();
|
|
1534
|
-
|
|
1535
|
-
for (const item of incrementalItems) {
|
|
1536
|
-
nextCursor = Math.max(nextCursor, item.commitSeq);
|
|
1537
|
-
const rowKey = `${item.change.table}\u0000${item.change.row_id}`;
|
|
1538
|
-
|
|
1539
|
-
// Move row keys to insertion tail so Map iteration yields
|
|
1540
|
-
// "latest change wins" order without a full array sort.
|
|
1541
|
-
if (latestByRowKey.has(rowKey)) {
|
|
1542
|
-
latestByRowKey.delete(rowKey);
|
|
1543
|
-
}
|
|
1544
|
-
latestByRowKey.set(rowKey, {
|
|
1545
|
-
commitSeq: item.commitSeq,
|
|
1546
|
-
createdAt: item.createdAt,
|
|
1547
|
-
actorId: item.actorId,
|
|
1548
|
-
change: item.change,
|
|
1549
|
-
});
|
|
1550
|
-
}
|
|
1551
|
-
|
|
1552
|
-
nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
|
|
1553
|
-
|
|
1554
|
-
if (latestByRowKey.size === 0) {
|
|
1555
|
-
subResponses.push({
|
|
1556
|
-
id: sub.id,
|
|
1557
|
-
status: 'active',
|
|
1558
|
-
scopes: effectiveScopes,
|
|
1559
|
-
bootstrap: false,
|
|
1560
|
-
nextCursor,
|
|
1561
|
-
commits: [],
|
|
1562
|
-
});
|
|
1563
|
-
nextCursors.push(nextCursor);
|
|
1564
|
-
continue;
|
|
1565
|
-
}
|
|
1566
|
-
|
|
1567
|
-
const commits: SyncCommit[] = [];
|
|
1568
|
-
for (const item of latestByRowKey.values()) {
|
|
1569
|
-
const lastCommit = commits[commits.length - 1];
|
|
1570
|
-
if (
|
|
1571
|
-
!lastCommit ||
|
|
1572
|
-
lastCommit.commitSeq !== item.commitSeq
|
|
1573
|
-
) {
|
|
1574
|
-
commits.push({
|
|
1575
|
-
commitSeq: item.commitSeq,
|
|
1576
|
-
createdAt: item.createdAt,
|
|
1577
|
-
actorId: item.actorId,
|
|
1578
|
-
changes: [item.change],
|
|
1579
|
-
});
|
|
1580
|
-
continue;
|
|
1581
|
-
}
|
|
1582
|
-
lastCommit.changes.push(item.change);
|
|
1583
|
-
}
|
|
1584
|
-
const integrity = await createWireSubscriptionIntegrity({
|
|
1585
|
-
partitionId,
|
|
1586
|
-
subscriptionId: sub.id,
|
|
1587
|
-
previousRoot:
|
|
1588
|
-
typeof sub.verifiedRoot === 'string'
|
|
1589
|
-
? sub.verifiedRoot
|
|
1590
|
-
: SYNCULAR_COMMIT_GENESIS_ROOT,
|
|
1591
|
-
commits,
|
|
1592
|
-
});
|
|
1593
|
-
|
|
1594
|
-
subResponses.push({
|
|
1595
|
-
id: sub.id,
|
|
1596
|
-
status: 'active',
|
|
1597
|
-
scopes: effectiveScopes,
|
|
1598
|
-
bootstrap: false,
|
|
1599
|
-
nextCursor,
|
|
1600
|
-
...(integrity ? { integrity } : {}),
|
|
1601
|
-
commits,
|
|
1602
|
-
});
|
|
1603
|
-
nextCursors.push(nextCursor);
|
|
1604
|
-
continue;
|
|
1605
|
-
}
|
|
1606
|
-
|
|
1607
|
-
const commits: SyncCommit[] = [];
|
|
1608
|
-
|
|
1609
|
-
for (const item of incrementalItems) {
|
|
1610
|
-
nextCursor = Math.max(nextCursor, item.commitSeq);
|
|
1611
|
-
const seq = item.commitSeq;
|
|
1612
|
-
let commit = commits[commits.length - 1];
|
|
1613
|
-
if (!commit || commit.commitSeq !== seq) {
|
|
1614
|
-
commit = {
|
|
1615
|
-
commitSeq: seq,
|
|
1616
|
-
createdAt: item.createdAt,
|
|
1617
|
-
actorId: item.actorId,
|
|
1618
|
-
changes: [],
|
|
1619
|
-
};
|
|
1620
|
-
commits.push(commit);
|
|
1621
|
-
}
|
|
1622
|
-
|
|
1623
|
-
commit.changes.push(item.change);
|
|
1624
|
-
}
|
|
1625
|
-
|
|
1626
|
-
const integrity = await createWireSubscriptionIntegrity({
|
|
1627
|
-
partitionId,
|
|
1628
|
-
subscriptionId: sub.id,
|
|
1629
|
-
previousRoot:
|
|
1630
|
-
typeof sub.verifiedRoot === 'string'
|
|
1631
|
-
? sub.verifiedRoot
|
|
1632
|
-
: SYNCULAR_COMMIT_GENESIS_ROOT,
|
|
1633
|
-
commits,
|
|
1634
|
-
});
|
|
1635
|
-
|
|
1636
|
-
nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
|
|
1637
|
-
|
|
1638
|
-
if (commits.length === 0) {
|
|
1639
|
-
subResponses.push({
|
|
1640
|
-
id: sub.id,
|
|
1641
|
-
status: 'active',
|
|
1642
|
-
scopes: effectiveScopes,
|
|
1643
|
-
bootstrap: false,
|
|
1644
|
-
nextCursor,
|
|
1645
|
-
commits: [],
|
|
1646
|
-
});
|
|
1647
|
-
nextCursors.push(nextCursor);
|
|
1648
|
-
continue;
|
|
1649
|
-
}
|
|
1650
|
-
|
|
1651
|
-
subResponses.push({
|
|
1652
|
-
id: sub.id,
|
|
1653
|
-
status: 'active',
|
|
1654
|
-
scopes: effectiveScopes,
|
|
1655
|
-
bootstrap: false,
|
|
1656
|
-
nextCursor,
|
|
1657
|
-
...(integrity ? { integrity } : {}),
|
|
1658
|
-
commits,
|
|
1659
|
-
});
|
|
1660
|
-
nextCursors.push(nextCursor);
|
|
1661
|
-
}
|
|
1662
|
-
|
|
1663
|
-
const effectiveScopes = mergeScopes(activeSubscriptions);
|
|
1664
|
-
const clientCursor =
|
|
1665
|
-
nextCursors.length > 0
|
|
1666
|
-
? Math.min(...nextCursors)
|
|
1667
|
-
: maxCommitSeq;
|
|
1668
|
-
|
|
1669
|
-
return {
|
|
1670
|
-
response: {
|
|
1671
|
-
ok: true as const,
|
|
1672
|
-
subscriptions: subResponses,
|
|
1673
|
-
},
|
|
1674
|
-
effectiveScopes,
|
|
1675
|
-
clientCursor,
|
|
1676
|
-
};
|
|
1677
|
-
}
|
|
1678
|
-
);
|
|
1679
|
-
|
|
1680
|
-
const chunkStorage = args.chunkStorage;
|
|
1681
|
-
if (chunkStorage && pendingExternalChunkWrites.length > 0) {
|
|
1682
|
-
await runWithConcurrency(
|
|
1683
|
-
pendingExternalChunkWrites,
|
|
1684
|
-
4,
|
|
1685
|
-
async (pending) => {
|
|
1686
|
-
const cacheLookupStartedAt = Date.now();
|
|
1687
|
-
let chunkRef: SyncSnapshotChunkRef | null =
|
|
1688
|
-
await readSnapshotChunkRefByPageKey(db, {
|
|
1689
|
-
partitionId: pending.cacheLookup.partitionId,
|
|
1690
|
-
scopeKey: pending.cacheLookup.scopeKey,
|
|
1691
|
-
scope: pending.cacheLookup.scope,
|
|
1692
|
-
asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
|
|
1693
|
-
rowCursor: pending.cacheLookup.rowCursor,
|
|
1694
|
-
rowLimit: pending.cacheLookup.rowLimit,
|
|
1695
|
-
encoding: snapshotChunkEncoding,
|
|
1696
|
-
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
1697
|
-
});
|
|
1698
|
-
bootstrapTimings.chunkCacheLookupMs += Math.max(
|
|
1699
|
-
0,
|
|
1700
|
-
Date.now() - cacheLookupStartedAt
|
|
1701
|
-
);
|
|
1702
|
-
|
|
1703
|
-
if (!chunkRef) {
|
|
1704
|
-
if (chunkStorage.storeChunkStream) {
|
|
1705
|
-
const {
|
|
1706
|
-
stream: bodyStream,
|
|
1707
|
-
byteLength,
|
|
1708
|
-
sha256,
|
|
1709
|
-
gzipMs,
|
|
1710
|
-
hashMs,
|
|
1711
|
-
} = await encodeCompressedSnapshotChunkToStream(
|
|
1712
|
-
pending.payloadParts,
|
|
1713
|
-
snapshotChunkGzipLevel
|
|
1714
|
-
);
|
|
1715
|
-
bootstrapTimings.chunkGzipMs += gzipMs;
|
|
1716
|
-
bootstrapTimings.chunkHashMs += hashMs;
|
|
1717
|
-
const chunkPersistStartedAt = Date.now();
|
|
1718
|
-
chunkRef = await chunkStorage.storeChunkStream({
|
|
1719
|
-
partitionId: pending.cacheLookup.partitionId,
|
|
1720
|
-
scopeKey: pending.cacheLookup.scopeKey,
|
|
1721
|
-
scope: pending.cacheLookup.scope,
|
|
1722
|
-
asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
|
|
1723
|
-
rowCursor: pending.cacheLookup.rowCursor,
|
|
1724
|
-
rowLimit: pending.cacheLookup.rowLimit,
|
|
1725
|
-
nextRowCursor: pending.cacheLookup.nextRowCursor,
|
|
1726
|
-
isLastPage: pending.cacheLookup.isLastPage,
|
|
1727
|
-
encoding: snapshotChunkEncoding,
|
|
1728
|
-
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
1729
|
-
sha256,
|
|
1730
|
-
byteLength,
|
|
1731
|
-
bodyStream,
|
|
1732
|
-
expiresAt: pending.expiresAt,
|
|
1733
|
-
});
|
|
1734
|
-
bootstrapTimings.chunkPersistMs += Math.max(
|
|
1735
|
-
0,
|
|
1736
|
-
Date.now() - chunkPersistStartedAt
|
|
1737
|
-
);
|
|
1738
|
-
} else {
|
|
1739
|
-
const encodedChunk = await encodeCompressedSnapshotChunk(
|
|
1740
|
-
pending.payloadParts,
|
|
1741
|
-
snapshotChunkGzipLevel
|
|
1742
|
-
);
|
|
1743
|
-
bootstrapTimings.chunkGzipMs += encodedChunk.gzipMs;
|
|
1744
|
-
bootstrapTimings.chunkHashMs += encodedChunk.hashMs;
|
|
1745
|
-
const chunkPersistStartedAt = Date.now();
|
|
1746
|
-
chunkRef = await chunkStorage.storeChunk({
|
|
1747
|
-
partitionId: pending.cacheLookup.partitionId,
|
|
1748
|
-
scopeKey: pending.cacheLookup.scopeKey,
|
|
1749
|
-
scope: pending.cacheLookup.scope,
|
|
1750
|
-
asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
|
|
1751
|
-
rowCursor: pending.cacheLookup.rowCursor,
|
|
1752
|
-
rowLimit: pending.cacheLookup.rowLimit,
|
|
1753
|
-
nextRowCursor: pending.cacheLookup.nextRowCursor,
|
|
1754
|
-
isLastPage: pending.cacheLookup.isLastPage,
|
|
1755
|
-
encoding: snapshotChunkEncoding,
|
|
1756
|
-
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
1757
|
-
sha256: encodedChunk.sha256,
|
|
1758
|
-
body: encodedChunk.body,
|
|
1759
|
-
expiresAt: pending.expiresAt,
|
|
1760
|
-
});
|
|
1761
|
-
bootstrapTimings.chunkPersistMs += Math.max(
|
|
1762
|
-
0,
|
|
1763
|
-
Date.now() - chunkPersistStartedAt
|
|
1764
|
-
);
|
|
1765
|
-
}
|
|
1766
|
-
}
|
|
1767
|
-
|
|
1768
|
-
const chunk = toResponseChunkRef(chunkRef);
|
|
1769
|
-
pending.snapshot.chunks = [chunk];
|
|
1770
|
-
pending.snapshot.manifest =
|
|
1771
|
-
await createChunkedSnapshotManifest({
|
|
1772
|
-
table: pending.snapshot.table,
|
|
1773
|
-
asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
|
|
1774
|
-
scopeKey: pending.cacheLookup.scopeKey,
|
|
1775
|
-
rowCursor: pending.cacheLookup.rowCursor,
|
|
1776
|
-
rowLimit: pending.cacheLookup.rowLimit,
|
|
1777
|
-
nextRowCursor: pending.cacheLookup.nextRowCursor,
|
|
1778
|
-
isFirstPage: pending.snapshot.isFirstPage,
|
|
1779
|
-
isLastPage: pending.cacheLookup.isLastPage,
|
|
1780
|
-
chunks: [chunk],
|
|
1781
|
-
});
|
|
1782
|
-
}
|
|
1783
|
-
);
|
|
1784
|
-
}
|
|
1785
|
-
|
|
1786
|
-
const durationMs = Math.max(0, Date.now() - startedAtMs);
|
|
1787
|
-
const stats = summarizePullResponse(result.response);
|
|
1788
|
-
|
|
1789
|
-
span.setAttribute('status', 'ok');
|
|
1790
|
-
span.setAttribute('duration_ms', durationMs);
|
|
1791
|
-
span.setAttribute('subscription_count', stats.subscriptionCount);
|
|
1792
|
-
span.setAttribute('commit_count', stats.commitCount);
|
|
1793
|
-
span.setAttribute('change_count', stats.changeCount);
|
|
1794
|
-
span.setAttribute('snapshot_page_count', stats.snapshotPageCount);
|
|
1795
|
-
span.setAttributes({
|
|
1796
|
-
bootstrap_snapshot_query_ms: bootstrapTimings.snapshotQueryMs,
|
|
1797
|
-
bootstrap_snapshot_binary_encode_ms:
|
|
1798
|
-
bootstrapTimings.binaryEncodeMs,
|
|
1799
|
-
bootstrap_chunk_cache_lookup_ms:
|
|
1800
|
-
bootstrapTimings.chunkCacheLookupMs,
|
|
1801
|
-
bootstrap_artifact_cache_lookup_ms:
|
|
1802
|
-
bootstrapTimings.artifactCacheLookupMs,
|
|
1803
|
-
bootstrap_chunk_gzip_ms: bootstrapTimings.chunkGzipMs,
|
|
1804
|
-
bootstrap_chunk_hash_ms: bootstrapTimings.chunkHashMs,
|
|
1805
|
-
bootstrap_chunk_persist_ms: bootstrapTimings.chunkPersistMs,
|
|
1806
|
-
});
|
|
1807
|
-
span.setStatus('ok');
|
|
1808
|
-
|
|
1809
|
-
recordPullMetrics({
|
|
1810
|
-
status: 'ok',
|
|
1811
|
-
dedupeRows,
|
|
1812
|
-
durationMs,
|
|
1813
|
-
stats,
|
|
1814
|
-
});
|
|
1815
|
-
|
|
1816
|
-
return {
|
|
1817
|
-
...result,
|
|
1818
|
-
bootstrapTimings,
|
|
1819
|
-
};
|
|
1820
|
-
} catch (error) {
|
|
1821
|
-
if (
|
|
1822
|
-
error instanceof Error &&
|
|
1823
|
-
attemptIndex < MAX_PULL_TRANSACTION_RETRIES - 1 &&
|
|
1824
|
-
isSerializablePullError(error)
|
|
1825
|
-
) {
|
|
1826
|
-
await delay(PULL_TRANSACTION_RETRY_DELAY_MS * (attemptIndex + 1));
|
|
1827
|
-
continue;
|
|
1828
|
-
}
|
|
1829
|
-
throw error;
|
|
1830
|
-
}
|
|
1831
|
-
}
|
|
1832
|
-
|
|
1833
|
-
throw new Error('Pull transaction retry loop exhausted unexpectedly');
|
|
1834
|
-
} catch (error) {
|
|
1835
|
-
const durationMs = Math.max(0, Date.now() - startedAtMs);
|
|
1836
|
-
|
|
1837
|
-
span.setAttribute('status', 'error');
|
|
1838
|
-
span.setAttribute('duration_ms', durationMs);
|
|
1839
|
-
span.setStatus('error');
|
|
1840
|
-
|
|
1841
|
-
recordPullMetrics({
|
|
1842
|
-
status: 'error',
|
|
1843
|
-
dedupeRows: request.dedupeRows === true,
|
|
1844
|
-
durationMs,
|
|
1845
|
-
stats: {
|
|
1846
|
-
subscriptionCount: 0,
|
|
1847
|
-
activeSubscriptionCount: 0,
|
|
1848
|
-
revokedSubscriptionCount: 0,
|
|
1849
|
-
bootstrapSubscriptionCount: 0,
|
|
1850
|
-
commitCount: 0,
|
|
1851
|
-
changeCount: 0,
|
|
1852
|
-
snapshotPageCount: 0,
|
|
1853
|
-
},
|
|
1854
|
-
});
|
|
1855
|
-
|
|
1856
|
-
captureSyncException(error, {
|
|
1857
|
-
event: 'sync.server.pull',
|
|
1858
|
-
requestedSubscriptionCount,
|
|
1859
|
-
dedupeRows: request.dedupeRows === true,
|
|
1860
|
-
});
|
|
1861
|
-
throw error;
|
|
1862
|
-
}
|
|
535
|
+
let delivered = 0;
|
|
536
|
+
let deliveredCommits = 0;
|
|
537
|
+
let lastDeliveredSeq = sub.cursor;
|
|
538
|
+
for (const commit of commits) {
|
|
539
|
+
if (
|
|
540
|
+
delivered > 0 &&
|
|
541
|
+
delivered + commit.changes.length > limits.limitCommits
|
|
542
|
+
) {
|
|
543
|
+
break;
|
|
1863
544
|
}
|
|
1864
|
-
|
|
545
|
+
yield commitFrame(sub.table, commit);
|
|
546
|
+
delivered += commit.changes.length;
|
|
547
|
+
deliveredCommits += 1;
|
|
548
|
+
lastDeliveredSeq = commit.commitSeq;
|
|
549
|
+
if (delivered >= limits.limitCommits) break;
|
|
550
|
+
}
|
|
551
|
+
const totalReturned = commits.reduce((n, c) => n + c.changes.length, 0);
|
|
552
|
+
// The window is proven exhausted only when the storage lookahead came
|
|
553
|
+
// back under budget (it scanned to `throughSeq`) and every returned
|
|
554
|
+
// commit was delivered.
|
|
555
|
+
const exhausted =
|
|
556
|
+
totalReturned <= limits.limitCommits && deliveredCommits === commits.length;
|
|
557
|
+
// §4.5: the cursor advances even when no matching changes exist; when
|
|
558
|
+
// the change limit truncated the window it stops at the last fully
|
|
559
|
+
// delivered commit.
|
|
560
|
+
const nextCursor = exhausted
|
|
561
|
+
? Math.max(sub.cursor, maxSeq)
|
|
562
|
+
: Math.max(sub.cursor, lastDeliveredSeq);
|
|
563
|
+
yield { type: 'SUB_END', nextCursor };
|
|
564
|
+
return { nextCursor, active: true };
|
|
1865
565
|
}
|