@syncular/server 0.1.3 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +773 -13
- package/dist/admin.d.ts +136 -0
- package/dist/admin.js +168 -0
- package/dist/blob-handlers.d.ts +69 -0
- package/dist/blob-handlers.js +245 -0
- package/dist/blob-store.d.ts +111 -0
- package/dist/blob-store.js +0 -0
- package/dist/content-encoding.d.ts +27 -0
- package/dist/content-encoding.js +69 -0
- package/dist/context.d.ts +151 -0
- package/dist/context.js +21 -0
- package/dist/crdt-merger.d.ts +28 -0
- package/dist/crdt-merger.js +11 -0
- package/dist/d1-storage.d.ts +50 -0
- package/dist/d1-storage.js +0 -0
- package/dist/errors.d.ts +35 -9
- package/dist/errors.js +220 -11
- package/dist/events-ring.d.ts +55 -0
- package/dist/events-ring.js +96 -0
- package/dist/events.d.ts +231 -0
- package/dist/events.js +22 -0
- package/dist/frame-bytes.d.ts +20 -0
- package/dist/frame-bytes.js +75 -0
- package/dist/handler.d.ts +9 -0
- package/dist/handler.js +490 -0
- package/dist/index.d.ts +42 -33
- package/dist/index.js +45 -27
- package/dist/lease-store.d.ts +50 -0
- package/dist/lease-store.js +0 -0
- package/dist/pg-executor-pglite.d.ts +27 -0
- package/dist/pg-executor-pglite.js +33 -0
- package/dist/pg-executor.d.ts +67 -0
- package/dist/pg-executor.js +56 -0
- package/dist/postgres-fanout.d.ts +83 -0
- package/dist/postgres-fanout.js +71 -0
- package/dist/postgres-storage.d.ts +69 -0
- package/dist/postgres-storage.js +766 -0
- package/dist/prune.d.ts +24 -39
- package/dist/prune.js +44 -146
- package/dist/pull.d.ts +42 -67
- package/dist/pull.js +387 -1187
- package/dist/push.d.ts +31 -66
- package/dist/push.js +406 -655
- package/dist/realtime.d.ts +197 -0
- package/dist/realtime.js +848 -0
- package/dist/relational-rows.d.ts +229 -0
- package/dist/relational-rows.js +514 -0
- package/dist/s3-blob-store.d.ts +142 -0
- package/dist/s3-blob-store.js +458 -0
- package/dist/s3-segment-store.d.ts +95 -0
- package/dist/s3-segment-store.js +380 -0
- package/dist/schema.d.ts +82 -286
- package/dist/schema.js +133 -9
- package/dist/scopes.d.ts +48 -0
- package/dist/scopes.js +98 -0
- package/dist/segment-download.d.ts +14 -0
- package/dist/segment-download.js +125 -0
- package/dist/segment-store.d.ts +81 -0
- package/dist/segment-store.js +63 -0
- package/dist/signed-url.d.ts +139 -0
- package/dist/signed-url.js +143 -0
- package/dist/sigv4.d.ts +97 -0
- package/dist/sigv4.js +161 -0
- package/dist/sqlite-blob-store.d.ts +22 -0
- package/dist/sqlite-blob-store.js +86 -0
- package/dist/sqlite-dialect.d.ts +101 -0
- package/dist/sqlite-dialect.js +250 -0
- package/dist/sqlite-image.d.ts +27 -0
- package/dist/sqlite-image.js +83 -0
- package/dist/sqlite-lease-store.d.ts +21 -0
- package/dist/sqlite-lease-store.js +94 -0
- package/dist/sqlite-segment-store.d.ts +25 -0
- package/dist/sqlite-segment-store.js +115 -0
- package/dist/sqlite-storage.d.ts +47 -0
- package/dist/sqlite-storage.js +477 -0
- package/dist/storage.d.ts +252 -0
- package/dist/storage.js +1 -0
- package/dist/validate.d.ts +83 -0
- package/dist/validate.js +44 -0
- package/package.json +17 -325
- package/src/admin.ts +312 -0
- package/src/blob-handlers.ts +340 -0
- package/src/blob-store.ts +0 -0
- package/src/content-encoding.ts +94 -0
- package/src/context.ts +176 -0
- package/src/crdt-merger.ts +33 -0
- package/src/d1-storage.ts +0 -0
- package/src/errors.ts +241 -19
- package/src/events-ring.ts +116 -0
- package/src/events.ts +291 -0
- package/src/frame-bytes.ts +94 -0
- package/src/handler.ts +678 -0
- package/src/index.ts +44 -27
- package/src/lease-store.ts +0 -0
- package/src/pg-executor-pglite.ts +63 -0
- package/src/pg-executor.ts +87 -0
- package/src/postgres-fanout.ts +143 -0
- package/src/postgres-storage.ts +1182 -0
- package/src/prune.ts +63 -289
- package/src/pull.ts +505 -1805
- package/src/push.ts +623 -1006
- package/src/realtime.ts +1082 -0
- package/src/relational-rows.ts +633 -0
- package/src/s3-blob-store.ts +616 -0
- package/src/s3-segment-store.ts +519 -0
- package/src/schema.ts +242 -281
- package/src/scopes.ts +127 -0
- package/src/segment-download.ts +169 -0
- package/src/segment-store.ts +154 -0
- package/src/signed-url.ts +308 -0
- package/src/sigv4.ts +267 -0
- package/src/sqlite-blob-store.ts +125 -0
- package/src/sqlite-dialect.ts +337 -0
- package/src/sqlite-image.ts +124 -0
- package/src/sqlite-lease-store.ts +144 -0
- package/src/sqlite-segment-store.ts +204 -0
- package/src/sqlite-storage.ts +792 -0
- package/src/storage.ts +316 -0
- package/src/validate.ts +123 -0
- package/dist/auth-leases.d.ts +0 -85
- package/dist/auth-leases.d.ts.map +0 -1
- package/dist/auth-leases.js +0 -368
- package/dist/auth-leases.js.map +0 -1
- package/dist/better-sqlite3.d.ts +0 -22
- package/dist/better-sqlite3.d.ts.map +0 -1
- package/dist/better-sqlite3.js +0 -16
- package/dist/better-sqlite3.js.map +0 -1
- package/dist/blobs/access.d.ts +0 -35
- package/dist/blobs/access.d.ts.map +0 -1
- package/dist/blobs/access.js +0 -149
- package/dist/blobs/access.js.map +0 -1
- package/dist/blobs/adapters/database.d.ts +0 -100
- package/dist/blobs/adapters/database.d.ts.map +0 -1
- package/dist/blobs/adapters/database.js +0 -247
- package/dist/blobs/adapters/database.js.map +0 -1
- package/dist/blobs/index.d.ts +0 -9
- package/dist/blobs/index.d.ts.map +0 -1
- package/dist/blobs/index.js +0 -9
- package/dist/blobs/index.js.map +0 -1
- package/dist/blobs/manager.d.ts +0 -220
- package/dist/blobs/manager.d.ts.map +0 -1
- package/dist/blobs/manager.js +0 -625
- package/dist/blobs/manager.js.map +0 -1
- package/dist/blobs/migrate.d.ts +0 -27
- package/dist/blobs/migrate.d.ts.map +0 -1
- package/dist/blobs/migrate.js +0 -127
- package/dist/blobs/migrate.js.map +0 -1
- package/dist/blobs/types.d.ts +0 -58
- package/dist/blobs/types.d.ts.map +0 -1
- package/dist/blobs/types.js +0 -5
- package/dist/blobs/types.js.map +0 -1
- package/dist/bun-sqlite.d.ts +0 -19
- package/dist/bun-sqlite.d.ts.map +0 -1
- package/dist/bun-sqlite.js +0 -19
- package/dist/bun-sqlite.js.map +0 -1
- package/dist/clients.d.ts +0 -15
- package/dist/clients.d.ts.map +0 -1
- package/dist/clients.js +0 -7
- package/dist/clients.js.map +0 -1
- package/dist/cloudflare/durable-object.d.ts +0 -93
- package/dist/cloudflare/durable-object.d.ts.map +0 -1
- package/dist/cloudflare/durable-object.js +0 -210
- package/dist/cloudflare/durable-object.js.map +0 -1
- package/dist/cloudflare/index.d.ts +0 -22
- package/dist/cloudflare/index.d.ts.map +0 -1
- package/dist/cloudflare/index.js +0 -22
- package/dist/cloudflare/index.js.map +0 -1
- package/dist/cloudflare/r2.d.ts +0 -180
- package/dist/cloudflare/r2.d.ts.map +0 -1
- package/dist/cloudflare/r2.js +0 -258
- package/dist/cloudflare/r2.js.map +0 -1
- package/dist/cloudflare/scope-cache.d.ts +0 -54
- package/dist/cloudflare/scope-cache.d.ts.map +0 -1
- package/dist/cloudflare/scope-cache.js +0 -223
- package/dist/cloudflare/scope-cache.js.map +0 -1
- package/dist/cloudflare/sentry.d.ts +0 -47
- package/dist/cloudflare/sentry.d.ts.map +0 -1
- package/dist/cloudflare/sentry.js +0 -163
- package/dist/cloudflare/sentry.js.map +0 -1
- package/dist/cloudflare/worker.d.ts +0 -46
- package/dist/cloudflare/worker.d.ts.map +0 -1
- package/dist/cloudflare/worker.js +0 -63
- package/dist/cloudflare/worker.js.map +0 -1
- package/dist/commit-integrity.d.ts +0 -38
- package/dist/commit-integrity.d.ts.map +0 -1
- package/dist/commit-integrity.js +0 -260
- package/dist/commit-integrity.js.map +0 -1
- package/dist/compaction.d.ts +0 -27
- package/dist/compaction.d.ts.map +0 -1
- package/dist/compaction.js +0 -49
- package/dist/compaction.js.map +0 -1
- package/dist/crdt-yjs/index.d.ts +0 -99
- package/dist/crdt-yjs/index.d.ts.map +0 -1
- package/dist/crdt-yjs/index.js +0 -629
- package/dist/crdt-yjs/index.js.map +0 -1
- package/dist/d1.d.ts +0 -13
- package/dist/d1.d.ts.map +0 -1
- package/dist/d1.js +0 -14
- package/dist/d1.js.map +0 -1
- package/dist/dialect/base.d.ts +0 -93
- package/dist/dialect/base.d.ts.map +0 -1
- package/dist/dialect/base.js +0 -181
- package/dist/dialect/base.js.map +0 -1
- package/dist/dialect/helpers.d.ts +0 -16
- package/dist/dialect/helpers.d.ts.map +0 -1
- package/dist/dialect/helpers.js +0 -83
- package/dist/dialect/helpers.js.map +0 -1
- package/dist/dialect/index.d.ts +0 -7
- package/dist/dialect/index.d.ts.map +0 -1
- package/dist/dialect/index.js +0 -7
- package/dist/dialect/index.js.map +0 -1
- package/dist/dialect/types.d.ts +0 -187
- package/dist/dialect/types.d.ts.map +0 -1
- package/dist/dialect/types.js +0 -8
- package/dist/dialect/types.js.map +0 -1
- package/dist/encrypted-crdt.d.ts +0 -67
- package/dist/encrypted-crdt.d.ts.map +0 -1
- package/dist/encrypted-crdt.js +0 -426
- package/dist/encrypted-crdt.js.map +0 -1
- package/dist/errors.d.ts.map +0 -1
- package/dist/errors.js.map +0 -1
- package/dist/filesystem/index.d.ts +0 -44
- package/dist/filesystem/index.d.ts.map +0 -1
- package/dist/filesystem/index.js +0 -164
- package/dist/filesystem/index.js.map +0 -1
- package/dist/handlers/collection.d.ts +0 -20
- package/dist/handlers/collection.d.ts.map +0 -1
- package/dist/handlers/collection.js +0 -42
- package/dist/handlers/collection.js.map +0 -1
- package/dist/handlers/create-handler.d.ts +0 -156
- package/dist/handlers/create-handler.d.ts.map +0 -1
- package/dist/handlers/create-handler.js +0 -626
- package/dist/handlers/create-handler.js.map +0 -1
- package/dist/handlers/index.d.ts +0 -4
- package/dist/handlers/index.d.ts.map +0 -1
- package/dist/handlers/index.js +0 -4
- package/dist/handlers/index.js.map +0 -1
- package/dist/handlers/types.d.ts +0 -295
- package/dist/handlers/types.d.ts.map +0 -1
- package/dist/handlers/types.js +0 -2
- package/dist/handlers/types.js.map +0 -1
- package/dist/helpers/conflict.d.ts +0 -52
- package/dist/helpers/conflict.d.ts.map +0 -1
- package/dist/helpers/conflict.js +0 -49
- package/dist/helpers/conflict.js.map +0 -1
- package/dist/helpers/emitted-change.d.ts +0 -56
- package/dist/helpers/emitted-change.d.ts.map +0 -1
- package/dist/helpers/emitted-change.js +0 -46
- package/dist/helpers/emitted-change.js.map +0 -1
- package/dist/helpers/index.d.ts +0 -12
- package/dist/helpers/index.d.ts.map +0 -1
- package/dist/helpers/index.js +0 -12
- package/dist/helpers/index.js.map +0 -1
- package/dist/helpers/paginate.d.ts +0 -49
- package/dist/helpers/paginate.d.ts.map +0 -1
- package/dist/helpers/paginate.js +0 -54
- package/dist/helpers/paginate.js.map +0 -1
- package/dist/helpers/scope-authorization.d.ts +0 -7
- package/dist/helpers/scope-authorization.d.ts.map +0 -1
- package/dist/helpers/scope-authorization.js +0 -19
- package/dist/helpers/scope-authorization.js.map +0 -1
- package/dist/helpers/scope-commit-index.d.ts +0 -12
- package/dist/helpers/scope-commit-index.d.ts.map +0 -1
- package/dist/helpers/scope-commit-index.js +0 -35
- package/dist/helpers/scope-commit-index.js.map +0 -1
- package/dist/helpers/scope-strings.d.ts +0 -74
- package/dist/helpers/scope-strings.d.ts.map +0 -1
- package/dist/helpers/scope-strings.js +0 -82
- package/dist/helpers/scope-strings.js.map +0 -1
- package/dist/hono/api-key-auth.d.ts +0 -49
- package/dist/hono/api-key-auth.d.ts.map +0 -1
- package/dist/hono/api-key-auth.js +0 -108
- package/dist/hono/api-key-auth.js.map +0 -1
- package/dist/hono/audit-redaction.d.ts +0 -20
- package/dist/hono/audit-redaction.d.ts.map +0 -1
- package/dist/hono/audit-redaction.js +0 -85
- package/dist/hono/audit-redaction.js.map +0 -1
- package/dist/hono/blobs.d.ts +0 -75
- package/dist/hono/blobs.d.ts.map +0 -1
- package/dist/hono/blobs.js +0 -586
- package/dist/hono/blobs.js.map +0 -1
- package/dist/hono/console/gateway.d.ts +0 -42
- package/dist/hono/console/gateway.d.ts.map +0 -1
- package/dist/hono/console/gateway.js +0 -2158
- package/dist/hono/console/gateway.js.map +0 -1
- package/dist/hono/console/live-auth.d.ts +0 -7
- package/dist/hono/console/live-auth.d.ts.map +0 -1
- package/dist/hono/console/live-auth.js +0 -39
- package/dist/hono/console/live-auth.js.map +0 -1
- package/dist/hono/console/route-descriptor.d.ts +0 -6
- package/dist/hono/console/route-descriptor.d.ts.map +0 -1
- package/dist/hono/console/route-descriptor.js +0 -16
- package/dist/hono/console/route-descriptor.js.map +0 -1
- package/dist/hono/console/routes/api-keys.d.ts +0 -8
- package/dist/hono/console/routes/api-keys.d.ts.map +0 -1
- package/dist/hono/console/routes/api-keys.js +0 -575
- package/dist/hono/console/routes/api-keys.js.map +0 -1
- package/dist/hono/console/routes/clients.d.ts +0 -8
- package/dist/hono/console/routes/clients.d.ts.map +0 -1
- package/dist/hono/console/routes/clients.js +0 -322
- package/dist/hono/console/routes/clients.js.map +0 -1
- package/dist/hono/console/routes/commits.d.ts +0 -8
- package/dist/hono/console/routes/commits.d.ts.map +0 -1
- package/dist/hono/console/routes/commits.js +0 -882
- package/dist/hono/console/routes/commits.js.map +0 -1
- package/dist/hono/console/routes/context.d.ts +0 -188
- package/dist/hono/console/routes/context.d.ts.map +0 -1
- package/dist/hono/console/routes/context.js +0 -633
- package/dist/hono/console/routes/context.js.map +0 -1
- package/dist/hono/console/routes/events.d.ts +0 -8
- package/dist/hono/console/routes/events.d.ts.map +0 -1
- package/dist/hono/console/routes/events.js +0 -501
- package/dist/hono/console/routes/events.js.map +0 -1
- package/dist/hono/console/routes/maintenance.d.ts +0 -8
- package/dist/hono/console/routes/maintenance.d.ts.map +0 -1
- package/dist/hono/console/routes/maintenance.js +0 -341
- package/dist/hono/console/routes/maintenance.js.map +0 -1
- package/dist/hono/console/routes/shared.d.ts +0 -174
- package/dist/hono/console/routes/shared.d.ts.map +0 -1
- package/dist/hono/console/routes/shared.js +0 -583
- package/dist/hono/console/routes/shared.js.map +0 -1
- package/dist/hono/console/routes/stats.d.ts +0 -8
- package/dist/hono/console/routes/stats.d.ts.map +0 -1
- package/dist/hono/console/routes/stats.js +0 -295
- package/dist/hono/console/routes/stats.js.map +0 -1
- package/dist/hono/console/routes/storage.d.ts +0 -8
- package/dist/hono/console/routes/storage.d.ts.map +0 -1
- package/dist/hono/console/routes/storage.js +0 -121
- package/dist/hono/console/routes/storage.js.map +0 -1
- package/dist/hono/console/routes.d.ts +0 -36
- package/dist/hono/console/routes.d.ts.map +0 -1
- package/dist/hono/console/routes.js +0 -112
- package/dist/hono/console/routes.js.map +0 -1
- package/dist/hono/console/schema-errors.d.ts +0 -2
- package/dist/hono/console/schema-errors.d.ts.map +0 -1
- package/dist/hono/console/schema-errors.js +0 -17
- package/dist/hono/console/schema-errors.js.map +0 -1
- package/dist/hono/console/schemas.d.ts +0 -1515
- package/dist/hono/console/schemas.d.ts.map +0 -1
- package/dist/hono/console/schemas.js +0 -661
- package/dist/hono/console/schemas.js.map +0 -1
- package/dist/hono/console/types.d.ts +0 -213
- package/dist/hono/console/types.d.ts.map +0 -1
- package/dist/hono/console/types.js +0 -2
- package/dist/hono/console/types.js.map +0 -1
- package/dist/hono/console/ui.d.ts +0 -38
- package/dist/hono/console/ui.d.ts.map +0 -1
- package/dist/hono/console/ui.js +0 -43
- package/dist/hono/console/ui.js.map +0 -1
- package/dist/hono/create-server.d.ts +0 -71
- package/dist/hono/create-server.d.ts.map +0 -1
- package/dist/hono/create-server.js +0 -121
- package/dist/hono/create-server.js.map +0 -1
- package/dist/hono/errors.d.ts +0 -13
- package/dist/hono/errors.d.ts.map +0 -1
- package/dist/hono/errors.js +0 -21
- package/dist/hono/errors.js.map +0 -1
- package/dist/hono/index.d.ts +0 -20
- package/dist/hono/index.d.ts.map +0 -1
- package/dist/hono/index.js +0 -31
- package/dist/hono/index.js.map +0 -1
- package/dist/hono/openapi.d.ts +0 -72
- package/dist/hono/openapi.d.ts.map +0 -1
- package/dist/hono/openapi.js +0 -99
- package/dist/hono/openapi.js.map +0 -1
- package/dist/hono/proxy/connection-manager.d.ts +0 -78
- package/dist/hono/proxy/connection-manager.d.ts.map +0 -1
- package/dist/hono/proxy/connection-manager.js +0 -251
- package/dist/hono/proxy/connection-manager.js.map +0 -1
- package/dist/hono/proxy/index.d.ts +0 -8
- package/dist/hono/proxy/index.d.ts.map +0 -1
- package/dist/hono/proxy/index.js +0 -8
- package/dist/hono/proxy/index.js.map +0 -1
- package/dist/hono/proxy/routes.d.ts +0 -86
- package/dist/hono/proxy/routes.d.ts.map +0 -1
- package/dist/hono/proxy/routes.js +0 -183
- package/dist/hono/proxy/routes.js.map +0 -1
- package/dist/hono/rate-limit.d.ts +0 -101
- package/dist/hono/rate-limit.d.ts.map +0 -1
- package/dist/hono/rate-limit.js +0 -184
- package/dist/hono/rate-limit.js.map +0 -1
- package/dist/hono/realtime-sync-packs.d.ts +0 -43
- package/dist/hono/realtime-sync-packs.d.ts.map +0 -1
- package/dist/hono/realtime-sync-packs.js +0 -219
- package/dist/hono/realtime-sync-packs.js.map +0 -1
- package/dist/hono/routes/audit.d.ts +0 -12
- package/dist/hono/routes/audit.d.ts.map +0 -1
- package/dist/hono/routes/audit.js +0 -385
- package/dist/hono/routes/audit.js.map +0 -1
- package/dist/hono/routes/auth-leases.d.ts +0 -8
- package/dist/hono/routes/auth-leases.d.ts.map +0 -1
- package/dist/hono/routes/auth-leases.js +0 -88
- package/dist/hono/routes/auth-leases.js.map +0 -1
- package/dist/hono/routes/combined.d.ts +0 -8
- package/dist/hono/routes/combined.d.ts.map +0 -1
- package/dist/hono/routes/combined.js +0 -392
- package/dist/hono/routes/combined.js.map +0 -1
- package/dist/hono/routes/context.d.ts +0 -314
- package/dist/hono/routes/context.d.ts.map +0 -1
- package/dist/hono/routes/context.js +0 -1147
- package/dist/hono/routes/context.js.map +0 -1
- package/dist/hono/routes/health.d.ts +0 -8
- package/dist/hono/routes/health.d.ts.map +0 -1
- package/dist/hono/routes/health.js +0 -16
- package/dist/hono/routes/health.js.map +0 -1
- package/dist/hono/routes/realtime.d.ts +0 -9
- package/dist/hono/routes/realtime.d.ts.map +0 -1
- package/dist/hono/routes/realtime.js +0 -626
- package/dist/hono/routes/realtime.js.map +0 -1
- package/dist/hono/routes/shared.d.ts +0 -689
- package/dist/hono/routes/shared.d.ts.map +0 -1
- package/dist/hono/routes/shared.js +0 -972
- package/dist/hono/routes/shared.js.map +0 -1
- package/dist/hono/routes/snapshots.d.ts +0 -10
- package/dist/hono/routes/snapshots.d.ts.map +0 -1
- package/dist/hono/routes/snapshots.js +0 -281
- package/dist/hono/routes/snapshots.js.map +0 -1
- package/dist/hono/routes.d.ts +0 -19
- package/dist/hono/routes.d.ts.map +0 -1
- package/dist/hono/routes.js +0 -37
- package/dist/hono/routes.js.map +0 -1
- package/dist/hono/validation.d.ts +0 -5
- package/dist/hono/validation.d.ts.map +0 -1
- package/dist/hono/validation.js +0 -47
- package/dist/hono/validation.js.map +0 -1
- package/dist/hono/websocket-origin.d.ts +0 -9
- package/dist/hono/websocket-origin.d.ts.map +0 -1
- package/dist/hono/websocket-origin.js +0 -96
- package/dist/hono/websocket-origin.js.map +0 -1
- package/dist/hono/ws.d.ts +0 -341
- package/dist/hono/ws.d.ts.map +0 -1
- package/dist/hono/ws.js +0 -711
- package/dist/hono/ws.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/libsql.d.ts +0 -29
- package/dist/libsql.d.ts.map +0 -1
- package/dist/libsql.js +0 -25
- package/dist/libsql.js.map +0 -1
- package/dist/migrate.d.ts +0 -14
- package/dist/migrate.d.ts.map +0 -1
- package/dist/migrate.js +0 -13
- package/dist/migrate.js.map +0 -1
- package/dist/neon.d.ts +0 -21
- package/dist/neon.d.ts.map +0 -1
- package/dist/neon.js +0 -22
- package/dist/neon.js.map +0 -1
- package/dist/notify.d.ts +0 -77
- package/dist/notify.d.ts.map +0 -1
- package/dist/notify.js +0 -233
- package/dist/notify.js.map +0 -1
- package/dist/pglite.d.ts +0 -37
- package/dist/pglite.d.ts.map +0 -1
- package/dist/pglite.js +0 -37
- package/dist/pglite.js.map +0 -1
- package/dist/plugins/index.d.ts +0 -2
- package/dist/plugins/index.d.ts.map +0 -1
- package/dist/plugins/index.js +0 -2
- package/dist/plugins/index.js.map +0 -1
- package/dist/plugins/types.d.ts +0 -73
- package/dist/plugins/types.d.ts.map +0 -1
- package/dist/plugins/types.js +0 -30
- package/dist/plugins/types.js.map +0 -1
- package/dist/postgres/index.d.ts +0 -68
- package/dist/postgres/index.d.ts.map +0 -1
- package/dist/postgres/index.js +0 -924
- package/dist/postgres/index.js.map +0 -1
- package/dist/proxy/collection.d.ts +0 -7
- package/dist/proxy/collection.d.ts.map +0 -1
- package/dist/proxy/collection.js +0 -6
- package/dist/proxy/collection.js.map +0 -1
- package/dist/proxy/handler.d.ts +0 -42
- package/dist/proxy/handler.d.ts.map +0 -1
- package/dist/proxy/handler.js +0 -102
- package/dist/proxy/handler.js.map +0 -1
- package/dist/proxy/index.d.ts +0 -9
- package/dist/proxy/index.d.ts.map +0 -1
- package/dist/proxy/index.js +0 -14
- package/dist/proxy/index.js.map +0 -1
- package/dist/proxy/mutation-detector.d.ts +0 -35
- package/dist/proxy/mutation-detector.d.ts.map +0 -1
- package/dist/proxy/mutation-detector.js +0 -246
- package/dist/proxy/mutation-detector.js.map +0 -1
- package/dist/proxy/oplog.d.ts +0 -30
- package/dist/proxy/oplog.d.ts.map +0 -1
- package/dist/proxy/oplog.js +0 -137
- package/dist/proxy/oplog.js.map +0 -1
- package/dist/proxy/types.d.ts +0 -44
- package/dist/proxy/types.d.ts.map +0 -1
- package/dist/proxy/types.js +0 -7
- package/dist/proxy/types.js.map +0 -1
- package/dist/prune.d.ts.map +0 -1
- package/dist/prune.js.map +0 -1
- package/dist/pull.d.ts.map +0 -1
- package/dist/pull.js.map +0 -1
- package/dist/push.d.ts.map +0 -1
- package/dist/push.js.map +0 -1
- package/dist/realtime/in-memory.d.ts +0 -13
- package/dist/realtime/in-memory.d.ts.map +0 -1
- package/dist/realtime/in-memory.js +0 -28
- package/dist/realtime/in-memory.js.map +0 -1
- package/dist/realtime/index.d.ts +0 -4
- package/dist/realtime/index.d.ts.map +0 -1
- package/dist/realtime/index.js +0 -3
- package/dist/realtime/index.js.map +0 -1
- package/dist/realtime/types.d.ts +0 -62
- package/dist/realtime/types.d.ts.map +0 -1
- package/dist/realtime/types.js +0 -19
- package/dist/realtime/types.js.map +0 -1
- package/dist/relay/client-role/forward-engine.d.ts +0 -63
- package/dist/relay/client-role/forward-engine.d.ts.map +0 -1
- package/dist/relay/client-role/forward-engine.js +0 -267
- package/dist/relay/client-role/forward-engine.js.map +0 -1
- package/dist/relay/client-role/index.d.ts +0 -9
- package/dist/relay/client-role/index.d.ts.map +0 -1
- package/dist/relay/client-role/index.js +0 -9
- package/dist/relay/client-role/index.js.map +0 -1
- package/dist/relay/client-role/pull-engine.d.ts +0 -72
- package/dist/relay/client-role/pull-engine.d.ts.map +0 -1
- package/dist/relay/client-role/pull-engine.js +0 -249
- package/dist/relay/client-role/pull-engine.js.map +0 -1
- package/dist/relay/client-role/sequence-mapper.d.ts +0 -65
- package/dist/relay/client-role/sequence-mapper.d.ts.map +0 -1
- package/dist/relay/client-role/sequence-mapper.js +0 -161
- package/dist/relay/client-role/sequence-mapper.js.map +0 -1
- package/dist/relay/evaluation/relay-paths.d.ts +0 -41
- package/dist/relay/evaluation/relay-paths.d.ts.map +0 -1
- package/dist/relay/evaluation/relay-paths.js +0 -504
- package/dist/relay/evaluation/relay-paths.js.map +0 -1
- package/dist/relay/evaluation/rust-boundary.d.ts +0 -47
- package/dist/relay/evaluation/rust-boundary.d.ts.map +0 -1
- package/dist/relay/evaluation/rust-boundary.js +0 -220
- package/dist/relay/evaluation/rust-boundary.js.map +0 -1
- package/dist/relay/index.d.ts +0 -37
- package/dist/relay/index.d.ts.map +0 -1
- package/dist/relay/index.js +0 -44
- package/dist/relay/index.js.map +0 -1
- package/dist/relay/migrate.d.ts +0 -18
- package/dist/relay/migrate.d.ts.map +0 -1
- package/dist/relay/migrate.js +0 -99
- package/dist/relay/migrate.js.map +0 -1
- package/dist/relay/mode-manager.d.ts +0 -60
- package/dist/relay/mode-manager.d.ts.map +0 -1
- package/dist/relay/mode-manager.js +0 -114
- package/dist/relay/mode-manager.js.map +0 -1
- package/dist/relay/realtime.d.ts +0 -90
- package/dist/relay/realtime.d.ts.map +0 -1
- package/dist/relay/realtime.js +0 -147
- package/dist/relay/realtime.js.map +0 -1
- package/dist/relay/relay.d.ts +0 -190
- package/dist/relay/relay.d.ts.map +0 -1
- package/dist/relay/relay.js +0 -320
- package/dist/relay/relay.js.map +0 -1
- package/dist/relay/schema.d.ts +0 -158
- package/dist/relay/schema.d.ts.map +0 -1
- package/dist/relay/schema.js +0 -7
- package/dist/relay/schema.js.map +0 -1
- package/dist/relay/server-role/index.d.ts +0 -56
- package/dist/relay/server-role/index.d.ts.map +0 -1
- package/dist/relay/server-role/index.js +0 -193
- package/dist/relay/server-role/index.js.map +0 -1
- package/dist/relay/server-role/pull.d.ts +0 -27
- package/dist/relay/server-role/pull.d.ts.map +0 -1
- package/dist/relay/server-role/pull.js +0 -24
- package/dist/relay/server-role/pull.js.map +0 -1
- package/dist/relay/server-role/push.d.ts +0 -29
- package/dist/relay/server-role/push.d.ts.map +0 -1
- package/dist/relay/server-role/push.js +0 -122
- package/dist/relay/server-role/push.js.map +0 -1
- package/dist/s3/index.d.ts +0 -83
- package/dist/s3/index.d.ts.map +0 -1
- package/dist/s3/index.js +0 -223
- package/dist/s3/index.js.map +0 -1
- package/dist/schema.d.ts.map +0 -1
- package/dist/schema.js.map +0 -1
- package/dist/service-worker/index.d.ts +0 -114
- package/dist/service-worker/index.d.ts.map +0 -1
- package/dist/service-worker/index.js +0 -487
- package/dist/service-worker/index.js.map +0 -1
- package/dist/snapshot-artifacts/sqlite-bun.d.ts +0 -15
- package/dist/snapshot-artifacts/sqlite-bun.d.ts.map +0 -1
- package/dist/snapshot-artifacts/sqlite-bun.js +0 -128
- package/dist/snapshot-artifacts/sqlite-bun.js.map +0 -1
- package/dist/snapshot-artifacts.d.ts +0 -206
- package/dist/snapshot-artifacts.d.ts.map +0 -1
- package/dist/snapshot-artifacts.js +0 -545
- package/dist/snapshot-artifacts.js.map +0 -1
- package/dist/snapshot-chunks/db-metadata.d.ts +0 -56
- package/dist/snapshot-chunks/db-metadata.d.ts.map +0 -1
- package/dist/snapshot-chunks/db-metadata.js +0 -360
- package/dist/snapshot-chunks/db-metadata.js.map +0 -1
- package/dist/snapshot-chunks/index.d.ts +0 -8
- package/dist/snapshot-chunks/index.d.ts.map +0 -1
- package/dist/snapshot-chunks/index.js +0 -8
- package/dist/snapshot-chunks/index.js.map +0 -1
- package/dist/snapshot-chunks/types.d.ts +0 -80
- package/dist/snapshot-chunks/types.d.ts.map +0 -1
- package/dist/snapshot-chunks/types.js +0 -8
- package/dist/snapshot-chunks/types.js.map +0 -1
- package/dist/snapshot-chunks.d.ts +0 -90
- package/dist/snapshot-chunks.d.ts.map +0 -1
- package/dist/snapshot-chunks.js +0 -304
- package/dist/snapshot-chunks.js.map +0 -1
- package/dist/sqlite/index.d.ts +0 -53
- package/dist/sqlite/index.d.ts.map +0 -1
- package/dist/sqlite/index.js +0 -795
- package/dist/sqlite/index.js.map +0 -1
- package/dist/sqlite3.d.ts +0 -22
- package/dist/sqlite3.d.ts.map +0 -1
- package/dist/sqlite3.js +0 -99
- package/dist/sqlite3.js.map +0 -1
- package/dist/stats.d.ts +0 -28
- package/dist/stats.d.ts.map +0 -1
- package/dist/stats.js +0 -93
- package/dist/stats.js.map +0 -1
- package/dist/subscriptions/cache.d.ts +0 -58
- package/dist/subscriptions/cache.d.ts.map +0 -1
- package/dist/subscriptions/cache.js +0 -250
- package/dist/subscriptions/cache.js.map +0 -1
- package/dist/subscriptions/index.d.ts +0 -3
- package/dist/subscriptions/index.d.ts.map +0 -1
- package/dist/subscriptions/index.js +0 -3
- package/dist/subscriptions/index.js.map +0 -1
- package/dist/subscriptions/resolve.d.ts +0 -40
- package/dist/subscriptions/resolve.d.ts.map +0 -1
- package/dist/subscriptions/resolve.js +0 -275
- package/dist/subscriptions/resolve.js.map +0 -1
- package/dist/sync.d.ts +0 -24
- package/dist/sync.d.ts.map +0 -1
- package/dist/sync.js +0 -27
- package/dist/sync.js.map +0 -1
- package/src/auth-leases.ts +0 -649
- package/src/better-sqlite3.ts +0 -35
- package/src/blobs/access.ts +0 -244
- package/src/blobs/adapters/database.ts +0 -397
- package/src/blobs/index.ts +0 -9
- package/src/blobs/manager.ts +0 -901
- package/src/blobs/migrate.ts +0 -158
- package/src/blobs/types.ts +0 -74
- package/src/bun-sqlite-ambient.d.ts +0 -19
- package/src/bun-sqlite.ts +0 -27
- package/src/clients.ts +0 -22
- package/src/cloudflare/durable-object.ts +0 -289
- package/src/cloudflare/index.ts +0 -22
- package/src/cloudflare/r2.ts +0 -526
- package/src/cloudflare/scope-cache.ts +0 -341
- package/src/cloudflare/sentry.ts +0 -230
- package/src/cloudflare/worker.ts +0 -77
- package/src/commit-integrity.ts +0 -371
- package/src/compaction.ts +0 -77
- package/src/crdt-yjs/index.ts +0 -931
- package/src/d1.ts +0 -16
- package/src/dialect/base.ts +0 -360
- package/src/dialect/helpers.ts +0 -92
- package/src/dialect/index.ts +0 -7
- package/src/dialect/types.ts +0 -247
- package/src/encrypted-crdt.ts +0 -786
- package/src/filesystem/index.ts +0 -262
- package/src/handlers/collection.ts +0 -121
- package/src/handlers/create-handler.ts +0 -1134
- package/src/handlers/index.ts +0 -3
- package/src/handlers/types.ts +0 -403
- package/src/helpers/conflict.ts +0 -64
- package/src/helpers/emitted-change.ts +0 -69
- package/src/helpers/index.ts +0 -12
- package/src/helpers/paginate.ts +0 -82
- package/src/helpers/scope-authorization.ts +0 -27
- package/src/helpers/scope-commit-index.ts +0 -52
- package/src/helpers/scope-strings.ts +0 -101
- package/src/hono/api-key-auth.ts +0 -177
- package/src/hono/audit-redaction.ts +0 -135
- package/src/hono/blobs.ts +0 -851
- package/src/hono/console/gateway.ts +0 -3046
- package/src/hono/console/live-auth.ts +0 -46
- package/src/hono/console/route-descriptor.ts +0 -22
- package/src/hono/console/routes/api-keys.ts +0 -721
- package/src/hono/console/routes/clients.ts +0 -447
- package/src/hono/console/routes/commits.ts +0 -1137
- package/src/hono/console/routes/context.ts +0 -956
- package/src/hono/console/routes/events.ts +0 -669
- package/src/hono/console/routes/maintenance.ts +0 -461
- package/src/hono/console/routes/shared.ts +0 -816
- package/src/hono/console/routes/stats.ts +0 -392
- package/src/hono/console/routes/storage.ts +0 -159
- package/src/hono/console/routes.ts +0 -146
- package/src/hono/console/schema-errors.ts +0 -23
- package/src/hono/console/schemas.ts +0 -914
- package/src/hono/console/types.ts +0 -223
- package/src/hono/console/ui.ts +0 -100
- package/src/hono/create-server.ts +0 -230
- package/src/hono/errors.ts +0 -50
- package/src/hono/index.ts +0 -54
- package/src/hono/openapi.ts +0 -139
- package/src/hono/proxy/connection-manager.ts +0 -340
- package/src/hono/proxy/index.ts +0 -8
- package/src/hono/proxy/routes.ts +0 -272
- package/src/hono/rate-limit.ts +0 -319
- package/src/hono/realtime-sync-packs.ts +0 -354
- package/src/hono/routes/audit.ts +0 -499
- package/src/hono/routes/auth-leases.ts +0 -119
- package/src/hono/routes/combined.ts +0 -583
- package/src/hono/routes/context.ts +0 -1629
- package/src/hono/routes/health.ts +0 -26
- package/src/hono/routes/realtime.ts +0 -808
- package/src/hono/routes/shared.ts +0 -1626
- package/src/hono/routes/snapshots.ts +0 -345
- package/src/hono/routes.ts +0 -68
- package/src/hono/validation.ts +0 -81
- package/src/hono/websocket-origin.ts +0 -131
- package/src/hono/ws.ts +0 -1134
- package/src/libsql.ts +0 -51
- package/src/migrate.ts +0 -20
- package/src/neon.ts +0 -28
- package/src/notify.ts +0 -341
- package/src/pglite.ts +0 -68
- package/src/plugins/index.ts +0 -1
- package/src/plugins/types.ts +0 -144
- package/src/postgres/index.ts +0 -1291
- package/src/proxy/collection.ts +0 -17
- package/src/proxy/handler.ts +0 -159
- package/src/proxy/index.ts +0 -21
- package/src/proxy/mutation-detector.ts +0 -281
- package/src/proxy/oplog.ts +0 -181
- package/src/proxy/types.ts +0 -46
- package/src/realtime/in-memory.ts +0 -33
- package/src/realtime/index.ts +0 -7
- package/src/realtime/types.ts +0 -90
- package/src/relay/bun-types.d.ts +0 -50
- package/src/relay/client-role/forward-engine.ts +0 -355
- package/src/relay/client-role/index.ts +0 -9
- package/src/relay/client-role/pull-engine.ts +0 -329
- package/src/relay/client-role/sequence-mapper.ts +0 -201
- package/src/relay/evaluation/relay-paths.ts +0 -699
- package/src/relay/evaluation/rust-boundary.ts +0 -464
- package/src/relay/index.ts +0 -50
- package/src/relay/migrate.ts +0 -113
- package/src/relay/mode-manager.ts +0 -142
- package/src/relay/realtime.ts +0 -207
- package/src/relay/relay.ts +0 -431
- package/src/relay/schema.ts +0 -171
- package/src/relay/server-role/index.ts +0 -338
- package/src/relay/server-role/pull.ts +0 -43
- package/src/relay/server-role/push.ts +0 -164
- package/src/s3/index.ts +0 -346
- package/src/service-worker/index.ts +0 -773
- package/src/snapshot-artifacts/sqlite-bun.ts +0 -168
- package/src/snapshot-artifacts.ts +0 -896
- package/src/snapshot-chunks/db-metadata.ts +0 -537
- package/src/snapshot-chunks/index.ts +0 -8
- package/src/snapshot-chunks/types.ts +0 -105
- package/src/snapshot-chunks.ts +0 -453
- package/src/sqlite/index.ts +0 -1064
- package/src/sqlite3.ts +0 -137
- package/src/stats.ts +0 -180
- package/src/subscriptions/cache.ts +0 -376
- package/src/subscriptions/index.ts +0 -2
- package/src/subscriptions/resolve.ts +0 -357
- package/src/sync.ts +0 -111
package/src/blobs/manager.ts
DELETED
|
@@ -1,901 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Server-side blob manager.
|
|
3
|
-
*
|
|
4
|
-
* Orchestrates blob uploads and downloads using a pluggable storage adapter.
|
|
5
|
-
* Handles metadata tracking, upload verification, and garbage collection.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
import type {
|
|
9
|
-
BlobMetadata,
|
|
10
|
-
BlobStorageAdapter,
|
|
11
|
-
BlobUploadCompleteResponse,
|
|
12
|
-
BlobUploadInitResponse,
|
|
13
|
-
} from '@syncular/core';
|
|
14
|
-
import { parseBlobHash } from '@syncular/core';
|
|
15
|
-
import { type Kysely, sql } from 'kysely';
|
|
16
|
-
import type { SyncBlobUploadsDb } from './types';
|
|
17
|
-
|
|
18
|
-
// ============================================================================
|
|
19
|
-
// Blob Manager
|
|
20
|
-
// ============================================================================
|
|
21
|
-
|
|
22
|
-
export interface BlobManagerOptions<
|
|
23
|
-
DB extends SyncBlobUploadsDb = SyncBlobUploadsDb,
|
|
24
|
-
> {
|
|
25
|
-
/** Database instance for tracking uploads */
|
|
26
|
-
db: Kysely<DB>;
|
|
27
|
-
/** Storage adapter (S3, R2, database, etc.) */
|
|
28
|
-
adapter: BlobStorageAdapter;
|
|
29
|
-
/** Default presigned URL expiration in seconds. Default: 3600 (1 hour) */
|
|
30
|
-
defaultExpiresIn?: number;
|
|
31
|
-
/** How long incomplete uploads are kept before cleanup. Default: 86400 (24 hours) */
|
|
32
|
-
uploadTtlSeconds?: number;
|
|
33
|
-
/** Optional cleanup throughput tuning knobs. */
|
|
34
|
-
cleanupTuning?: BlobCleanupTuning;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
export interface BlobCleanupTuning {
|
|
38
|
-
/** Cleanup select/delete batch size. Default: 250 */
|
|
39
|
-
batchSize?: number;
|
|
40
|
-
/** Max concurrent storage deletes during cleanup. Default: 8 */
|
|
41
|
-
storageDeleteConcurrency?: number;
|
|
42
|
-
/** Max concurrent reference checks for completed uploads. Default: 16 */
|
|
43
|
-
referenceCheckConcurrency?: number;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
export const BLOB_CLEANUP_TUNING_PRESETS = {
|
|
47
|
-
server: {
|
|
48
|
-
batchSize: 500,
|
|
49
|
-
storageDeleteConcurrency: 16,
|
|
50
|
-
referenceCheckConcurrency: 24,
|
|
51
|
-
},
|
|
52
|
-
edge: {
|
|
53
|
-
batchSize: 100,
|
|
54
|
-
storageDeleteConcurrency: 4,
|
|
55
|
-
referenceCheckConcurrency: 8,
|
|
56
|
-
},
|
|
57
|
-
} as const satisfies Record<'server' | 'edge', Required<BlobCleanupTuning>>;
|
|
58
|
-
|
|
59
|
-
export interface InitiateUploadOptions {
|
|
60
|
-
hash: string;
|
|
61
|
-
size: number;
|
|
62
|
-
mimeType: string;
|
|
63
|
-
actorId: string;
|
|
64
|
-
partitionId?: string;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
export interface GetDownloadUrlOptions {
|
|
68
|
-
hash: string;
|
|
69
|
-
/** Optional: verify actor has access to this blob via a scope check */
|
|
70
|
-
actorId?: string;
|
|
71
|
-
partitionId?: string;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
export interface CompleteUploadOptions {
|
|
75
|
-
/**
|
|
76
|
-
* Optional actor identity to authorize upload completion.
|
|
77
|
-
* When provided, only the initiating actor may mark an upload complete.
|
|
78
|
-
*/
|
|
79
|
-
actorId?: string;
|
|
80
|
-
partitionId?: string;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
export interface BlobUploadRecord {
|
|
84
|
-
partitionId: string;
|
|
85
|
-
hash: string;
|
|
86
|
-
size: number;
|
|
87
|
-
mimeType: string;
|
|
88
|
-
status: 'pending' | 'complete';
|
|
89
|
-
actorId: string;
|
|
90
|
-
createdAt: string;
|
|
91
|
-
expiresAt: string;
|
|
92
|
-
completedAt: string | null;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
/**
|
|
96
|
-
* Create a blob manager for handling server-side blob operations.
|
|
97
|
-
*/
|
|
98
|
-
export function createBlobManager<DB extends SyncBlobUploadsDb>(
|
|
99
|
-
options: BlobManagerOptions<DB>
|
|
100
|
-
) {
|
|
101
|
-
const {
|
|
102
|
-
db,
|
|
103
|
-
adapter,
|
|
104
|
-
defaultExpiresIn = 3600,
|
|
105
|
-
uploadTtlSeconds = 86400,
|
|
106
|
-
cleanupTuning,
|
|
107
|
-
} = options;
|
|
108
|
-
|
|
109
|
-
function positiveIntOrDefault(value: number | undefined, fallback: number) {
|
|
110
|
-
if (value === undefined) return fallback;
|
|
111
|
-
if (!Number.isFinite(value)) return fallback;
|
|
112
|
-
const normalized = Math.trunc(value);
|
|
113
|
-
return normalized > 0 ? normalized : fallback;
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
function resolvePartitionId(partitionId?: string): string {
|
|
117
|
-
return partitionId ?? 'default';
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
function normalizeBlobSizeValue(value: number | string): number {
|
|
121
|
-
return typeof value === 'number' ? value : Number(value);
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
function toStoragePartitionOptions(partitionId: string): {
|
|
125
|
-
partitionId?: string;
|
|
126
|
-
} {
|
|
127
|
-
if (partitionId === 'default') return {};
|
|
128
|
-
return { partitionId };
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
const CLEANUP_BATCH_SIZE = positiveIntOrDefault(
|
|
132
|
-
cleanupTuning?.batchSize,
|
|
133
|
-
250
|
|
134
|
-
);
|
|
135
|
-
const STORAGE_DELETE_CONCURRENCY = positiveIntOrDefault(
|
|
136
|
-
cleanupTuning?.storageDeleteConcurrency,
|
|
137
|
-
8
|
|
138
|
-
);
|
|
139
|
-
const REFERENCE_CHECK_CONCURRENCY = positiveIntOrDefault(
|
|
140
|
-
cleanupTuning?.referenceCheckConcurrency,
|
|
141
|
-
16
|
|
142
|
-
);
|
|
143
|
-
|
|
144
|
-
async function runWithConcurrency<T>(
|
|
145
|
-
items: readonly T[],
|
|
146
|
-
concurrency: number,
|
|
147
|
-
worker: (item: T) => Promise<void>
|
|
148
|
-
): Promise<void> {
|
|
149
|
-
if (items.length === 0) return;
|
|
150
|
-
|
|
151
|
-
const workerCount = Math.max(1, Math.min(concurrency, items.length));
|
|
152
|
-
let nextIndex = 0;
|
|
153
|
-
|
|
154
|
-
async function runWorker(): Promise<void> {
|
|
155
|
-
while (nextIndex < items.length) {
|
|
156
|
-
const index = nextIndex;
|
|
157
|
-
nextIndex += 1;
|
|
158
|
-
const item = items[index];
|
|
159
|
-
if (item === undefined) continue;
|
|
160
|
-
await worker(item);
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
await Promise.all(Array.from({ length: workerCount }, () => runWorker()));
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
async function deleteUploadRowsByHashes(
|
|
168
|
-
partitionId: string,
|
|
169
|
-
hashes: readonly string[]
|
|
170
|
-
): Promise<number> {
|
|
171
|
-
if (hashes.length === 0) return 0;
|
|
172
|
-
|
|
173
|
-
const deletedResult = await sql`
|
|
174
|
-
delete from ${sql.table('sync_blob_uploads')}
|
|
175
|
-
where partition_id = ${partitionId}
|
|
176
|
-
and hash in (${sql.join(hashes)})
|
|
177
|
-
`.execute(db);
|
|
178
|
-
|
|
179
|
-
return Number(deletedResult.numAffectedRows ?? 0);
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
return {
|
|
183
|
-
/**
|
|
184
|
-
* Initiate a blob upload.
|
|
185
|
-
*
|
|
186
|
-
* Checks for deduplication and returns a presigned URL for uploading.
|
|
187
|
-
*/
|
|
188
|
-
async initiateUpload(
|
|
189
|
-
opts: InitiateUploadOptions
|
|
190
|
-
): Promise<BlobUploadInitResponse> {
|
|
191
|
-
const { hash, size, mimeType, actorId } = opts;
|
|
192
|
-
const partitionId = resolvePartitionId(opts.partitionId);
|
|
193
|
-
const storagePartitionOptions = toStoragePartitionOptions(partitionId);
|
|
194
|
-
|
|
195
|
-
// Validate hash format
|
|
196
|
-
if (!parseBlobHash(hash)) {
|
|
197
|
-
throw new BlobValidationError('Invalid blob hash format');
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
// Check if blob already exists (deduplication)
|
|
201
|
-
const exists = await adapter.exists(hash, storagePartitionOptions);
|
|
202
|
-
if (exists) {
|
|
203
|
-
// Also check if we have a complete upload record
|
|
204
|
-
const existingResult = await sql<{ status: 'pending' | 'complete' }>`
|
|
205
|
-
select status
|
|
206
|
-
from ${sql.table('sync_blob_uploads')}
|
|
207
|
-
where partition_id = ${partitionId}
|
|
208
|
-
and hash = ${hash}
|
|
209
|
-
and status = 'complete'
|
|
210
|
-
limit 1
|
|
211
|
-
`.execute(db);
|
|
212
|
-
const existing = existingResult.rows[0];
|
|
213
|
-
|
|
214
|
-
if (existing) {
|
|
215
|
-
return { exists: true };
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
// Blob exists in storage but we don't have a record - create one
|
|
219
|
-
const existsExpiresAt = new Date(
|
|
220
|
-
Date.now() + uploadTtlSeconds * 1000
|
|
221
|
-
).toISOString();
|
|
222
|
-
const existsCompletedAt = new Date().toISOString();
|
|
223
|
-
|
|
224
|
-
await sql`
|
|
225
|
-
insert into ${sql.table('sync_blob_uploads')} (
|
|
226
|
-
partition_id,
|
|
227
|
-
hash,
|
|
228
|
-
size,
|
|
229
|
-
mime_type,
|
|
230
|
-
status,
|
|
231
|
-
actor_id,
|
|
232
|
-
expires_at,
|
|
233
|
-
completed_at
|
|
234
|
-
)
|
|
235
|
-
values (
|
|
236
|
-
${partitionId},
|
|
237
|
-
${hash},
|
|
238
|
-
${size},
|
|
239
|
-
${mimeType},
|
|
240
|
-
'complete',
|
|
241
|
-
${actorId},
|
|
242
|
-
${existsExpiresAt},
|
|
243
|
-
${existsCompletedAt}
|
|
244
|
-
)
|
|
245
|
-
on conflict (partition_id, hash) do nothing
|
|
246
|
-
`.execute(db);
|
|
247
|
-
|
|
248
|
-
return { exists: true };
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
// Create pending upload record
|
|
252
|
-
const expiresAt = new Date(
|
|
253
|
-
Date.now() + uploadTtlSeconds * 1000
|
|
254
|
-
).toISOString();
|
|
255
|
-
|
|
256
|
-
await sql`
|
|
257
|
-
insert into ${sql.table('sync_blob_uploads')} (
|
|
258
|
-
partition_id,
|
|
259
|
-
hash,
|
|
260
|
-
size,
|
|
261
|
-
mime_type,
|
|
262
|
-
status,
|
|
263
|
-
actor_id,
|
|
264
|
-
expires_at,
|
|
265
|
-
completed_at
|
|
266
|
-
)
|
|
267
|
-
values (
|
|
268
|
-
${partitionId},
|
|
269
|
-
${hash},
|
|
270
|
-
${size},
|
|
271
|
-
${mimeType},
|
|
272
|
-
'pending',
|
|
273
|
-
${actorId},
|
|
274
|
-
${expiresAt},
|
|
275
|
-
${null}
|
|
276
|
-
)
|
|
277
|
-
on conflict (partition_id, hash)
|
|
278
|
-
do update set
|
|
279
|
-
size = ${size},
|
|
280
|
-
mime_type = ${mimeType},
|
|
281
|
-
status = 'pending',
|
|
282
|
-
actor_id = ${actorId},
|
|
283
|
-
expires_at = ${expiresAt},
|
|
284
|
-
completed_at = ${null}
|
|
285
|
-
`.execute(db);
|
|
286
|
-
|
|
287
|
-
// Generate presigned upload URL
|
|
288
|
-
const signed = await adapter.signUpload({
|
|
289
|
-
hash,
|
|
290
|
-
partitionId: storagePartitionOptions.partitionId,
|
|
291
|
-
size,
|
|
292
|
-
mimeType,
|
|
293
|
-
expiresIn: defaultExpiresIn,
|
|
294
|
-
});
|
|
295
|
-
|
|
296
|
-
return {
|
|
297
|
-
exists: false,
|
|
298
|
-
uploadId: hash, // Use hash as upload ID
|
|
299
|
-
uploadUrl: signed.url,
|
|
300
|
-
uploadMethod: signed.method,
|
|
301
|
-
uploadHeaders: signed.headers,
|
|
302
|
-
};
|
|
303
|
-
},
|
|
304
|
-
|
|
305
|
-
/**
|
|
306
|
-
* Complete a blob upload.
|
|
307
|
-
*
|
|
308
|
-
* Verifies the blob exists in storage and marks the upload as complete.
|
|
309
|
-
*/
|
|
310
|
-
async completeUpload(
|
|
311
|
-
hash: string,
|
|
312
|
-
options?: CompleteUploadOptions
|
|
313
|
-
): Promise<BlobUploadCompleteResponse> {
|
|
314
|
-
const partitionId = resolvePartitionId(options?.partitionId);
|
|
315
|
-
const storagePartitionOptions = toStoragePartitionOptions(partitionId);
|
|
316
|
-
|
|
317
|
-
// Validate hash format
|
|
318
|
-
if (!parseBlobHash(hash)) {
|
|
319
|
-
return {
|
|
320
|
-
ok: false,
|
|
321
|
-
error: 'Invalid blob hash format',
|
|
322
|
-
code: 'blob.invalid_request',
|
|
323
|
-
};
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
// Check upload record exists
|
|
327
|
-
const uploadResult = await sql<{
|
|
328
|
-
hash: string;
|
|
329
|
-
size: number | string;
|
|
330
|
-
mime_type: string;
|
|
331
|
-
status: 'pending' | 'complete';
|
|
332
|
-
actor_id: string;
|
|
333
|
-
created_at: string;
|
|
334
|
-
}>`
|
|
335
|
-
select hash, size, mime_type, status, actor_id, created_at
|
|
336
|
-
from ${sql.table('sync_blob_uploads')}
|
|
337
|
-
where partition_id = ${partitionId} and hash = ${hash}
|
|
338
|
-
limit 1
|
|
339
|
-
`.execute(db);
|
|
340
|
-
const upload = uploadResult.rows[0];
|
|
341
|
-
|
|
342
|
-
if (!upload) {
|
|
343
|
-
return { ok: false, error: 'Upload not found', code: 'blob.not_found' };
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
if (
|
|
347
|
-
options?.actorId !== undefined &&
|
|
348
|
-
upload.actor_id !== options.actorId
|
|
349
|
-
) {
|
|
350
|
-
return { ok: false, error: 'Forbidden', code: 'blob.forbidden' };
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
if (upload.status === 'complete') {
|
|
354
|
-
const uploadSize = normalizeBlobSizeValue(upload.size);
|
|
355
|
-
// Already complete - return metadata
|
|
356
|
-
return {
|
|
357
|
-
ok: true,
|
|
358
|
-
metadata: {
|
|
359
|
-
hash: upload.hash,
|
|
360
|
-
size: uploadSize,
|
|
361
|
-
mimeType: upload.mime_type,
|
|
362
|
-
createdAt: upload.created_at,
|
|
363
|
-
uploadComplete: true,
|
|
364
|
-
},
|
|
365
|
-
};
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
// Verify blob exists in storage
|
|
369
|
-
const exists = await adapter.exists(hash, storagePartitionOptions);
|
|
370
|
-
if (!exists) {
|
|
371
|
-
return {
|
|
372
|
-
ok: false,
|
|
373
|
-
error: 'Blob not found in storage',
|
|
374
|
-
code: 'blob.not_found',
|
|
375
|
-
};
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
// Optionally verify size matches
|
|
379
|
-
const uploadSize = normalizeBlobSizeValue(upload.size);
|
|
380
|
-
if (adapter.getMetadata) {
|
|
381
|
-
const meta = await adapter.getMetadata(hash, storagePartitionOptions);
|
|
382
|
-
if (meta && meta.size !== uploadSize) {
|
|
383
|
-
return {
|
|
384
|
-
ok: false,
|
|
385
|
-
error: `Size mismatch: expected ${uploadSize}, got ${meta.size}`,
|
|
386
|
-
code: 'blob.size_mismatch',
|
|
387
|
-
};
|
|
388
|
-
}
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
// Mark upload as complete
|
|
392
|
-
const completedAt = new Date().toISOString();
|
|
393
|
-
await sql`
|
|
394
|
-
update ${sql.table('sync_blob_uploads')}
|
|
395
|
-
set status = 'complete', completed_at = ${completedAt}
|
|
396
|
-
where partition_id = ${partitionId} and hash = ${hash}
|
|
397
|
-
`.execute(db);
|
|
398
|
-
|
|
399
|
-
return {
|
|
400
|
-
ok: true,
|
|
401
|
-
metadata: {
|
|
402
|
-
hash: upload.hash,
|
|
403
|
-
size: uploadSize,
|
|
404
|
-
mimeType: upload.mime_type,
|
|
405
|
-
createdAt: upload.created_at,
|
|
406
|
-
uploadComplete: true,
|
|
407
|
-
},
|
|
408
|
-
};
|
|
409
|
-
},
|
|
410
|
-
|
|
411
|
-
/**
|
|
412
|
-
* Get a presigned download URL for a blob.
|
|
413
|
-
*/
|
|
414
|
-
async getDownloadUrl(
|
|
415
|
-
opts: GetDownloadUrlOptions
|
|
416
|
-
): Promise<{ url: string; expiresAt: string; metadata: BlobMetadata }> {
|
|
417
|
-
const { hash } = opts;
|
|
418
|
-
const partitionId = resolvePartitionId(opts.partitionId);
|
|
419
|
-
const storagePartitionOptions = toStoragePartitionOptions(partitionId);
|
|
420
|
-
|
|
421
|
-
// Validate hash format
|
|
422
|
-
if (!parseBlobHash(hash)) {
|
|
423
|
-
throw new BlobNotFoundError('Invalid blob hash format');
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
// Get upload record (must be complete)
|
|
427
|
-
const uploadResult = await sql<{
|
|
428
|
-
hash: string;
|
|
429
|
-
size: number | string;
|
|
430
|
-
mime_type: string;
|
|
431
|
-
status: 'pending' | 'complete';
|
|
432
|
-
created_at: string;
|
|
433
|
-
}>`
|
|
434
|
-
select hash, size, mime_type, status, created_at
|
|
435
|
-
from ${sql.table('sync_blob_uploads')}
|
|
436
|
-
where partition_id = ${partitionId}
|
|
437
|
-
and hash = ${hash}
|
|
438
|
-
and status = 'complete'
|
|
439
|
-
limit 1
|
|
440
|
-
`.execute(db);
|
|
441
|
-
const upload = uploadResult.rows[0];
|
|
442
|
-
|
|
443
|
-
if (!upload) {
|
|
444
|
-
throw new BlobNotFoundError('Blob not found');
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
// Generate presigned download URL
|
|
448
|
-
const url = await adapter.signDownload({
|
|
449
|
-
hash,
|
|
450
|
-
partitionId: storagePartitionOptions.partitionId,
|
|
451
|
-
expiresIn: defaultExpiresIn,
|
|
452
|
-
});
|
|
453
|
-
|
|
454
|
-
const expiresAt = new Date(
|
|
455
|
-
Date.now() + defaultExpiresIn * 1000
|
|
456
|
-
).toISOString();
|
|
457
|
-
|
|
458
|
-
return {
|
|
459
|
-
url,
|
|
460
|
-
expiresAt,
|
|
461
|
-
metadata: {
|
|
462
|
-
hash: upload.hash,
|
|
463
|
-
size: normalizeBlobSizeValue(upload.size),
|
|
464
|
-
mimeType: upload.mime_type,
|
|
465
|
-
createdAt: upload.created_at,
|
|
466
|
-
uploadComplete: true,
|
|
467
|
-
},
|
|
468
|
-
};
|
|
469
|
-
},
|
|
470
|
-
|
|
471
|
-
/**
|
|
472
|
-
* Get upload record for a blob hash, including pending uploads.
|
|
473
|
-
*/
|
|
474
|
-
async getUploadRecord(
|
|
475
|
-
hash: string,
|
|
476
|
-
options?: { partitionId?: string }
|
|
477
|
-
): Promise<BlobUploadRecord | null> {
|
|
478
|
-
const partitionId = resolvePartitionId(options?.partitionId);
|
|
479
|
-
if (!parseBlobHash(hash)) {
|
|
480
|
-
return null;
|
|
481
|
-
}
|
|
482
|
-
|
|
483
|
-
const uploadResult = await sql<{
|
|
484
|
-
partition_id: string;
|
|
485
|
-
hash: string;
|
|
486
|
-
size: number | string;
|
|
487
|
-
mime_type: string;
|
|
488
|
-
status: 'pending' | 'complete';
|
|
489
|
-
actor_id: string;
|
|
490
|
-
created_at: string;
|
|
491
|
-
expires_at: string;
|
|
492
|
-
completed_at: string | null;
|
|
493
|
-
}>`
|
|
494
|
-
select
|
|
495
|
-
partition_id,
|
|
496
|
-
hash,
|
|
497
|
-
size,
|
|
498
|
-
mime_type,
|
|
499
|
-
status,
|
|
500
|
-
actor_id,
|
|
501
|
-
created_at,
|
|
502
|
-
expires_at,
|
|
503
|
-
completed_at
|
|
504
|
-
from ${sql.table('sync_blob_uploads')}
|
|
505
|
-
where partition_id = ${partitionId} and hash = ${hash}
|
|
506
|
-
limit 1
|
|
507
|
-
`.execute(db);
|
|
508
|
-
const upload = uploadResult.rows[0];
|
|
509
|
-
|
|
510
|
-
if (!upload) {
|
|
511
|
-
return null;
|
|
512
|
-
}
|
|
513
|
-
|
|
514
|
-
return {
|
|
515
|
-
partitionId: upload.partition_id,
|
|
516
|
-
hash: upload.hash,
|
|
517
|
-
size: normalizeBlobSizeValue(upload.size),
|
|
518
|
-
mimeType: upload.mime_type,
|
|
519
|
-
status: upload.status,
|
|
520
|
-
actorId: upload.actor_id,
|
|
521
|
-
createdAt: upload.created_at,
|
|
522
|
-
expiresAt: upload.expires_at,
|
|
523
|
-
completedAt: upload.completed_at,
|
|
524
|
-
};
|
|
525
|
-
},
|
|
526
|
-
|
|
527
|
-
/**
|
|
528
|
-
* Get blob metadata without generating a download URL.
|
|
529
|
-
*/
|
|
530
|
-
async getMetadata(
|
|
531
|
-
hash: string,
|
|
532
|
-
options?: { partitionId?: string }
|
|
533
|
-
): Promise<BlobMetadata | null> {
|
|
534
|
-
const partitionId = resolvePartitionId(options?.partitionId);
|
|
535
|
-
// Validate hash format
|
|
536
|
-
if (!parseBlobHash(hash)) {
|
|
537
|
-
return null;
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
const uploadResult = await sql<{
|
|
541
|
-
hash: string;
|
|
542
|
-
size: number | string;
|
|
543
|
-
mime_type: string;
|
|
544
|
-
status: 'pending' | 'complete';
|
|
545
|
-
created_at: string;
|
|
546
|
-
}>`
|
|
547
|
-
select hash, size, mime_type, status, created_at
|
|
548
|
-
from ${sql.table('sync_blob_uploads')}
|
|
549
|
-
where partition_id = ${partitionId}
|
|
550
|
-
and hash = ${hash}
|
|
551
|
-
and status = 'complete'
|
|
552
|
-
limit 1
|
|
553
|
-
`.execute(db);
|
|
554
|
-
const upload = uploadResult.rows[0];
|
|
555
|
-
|
|
556
|
-
if (!upload) {
|
|
557
|
-
return null;
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
return {
|
|
561
|
-
hash: upload.hash,
|
|
562
|
-
size: normalizeBlobSizeValue(upload.size),
|
|
563
|
-
mimeType: upload.mime_type,
|
|
564
|
-
createdAt: upload.created_at,
|
|
565
|
-
uploadComplete: true,
|
|
566
|
-
};
|
|
567
|
-
},
|
|
568
|
-
|
|
569
|
-
/**
|
|
570
|
-
* Check if a blob exists and is complete.
|
|
571
|
-
*/
|
|
572
|
-
async exists(
|
|
573
|
-
hash: string,
|
|
574
|
-
options?: { partitionId?: string }
|
|
575
|
-
): Promise<boolean> {
|
|
576
|
-
const partitionId = resolvePartitionId(options?.partitionId);
|
|
577
|
-
if (!parseBlobHash(hash)) return false;
|
|
578
|
-
|
|
579
|
-
const rowResult = await sql<{ hash: string }>`
|
|
580
|
-
select hash
|
|
581
|
-
from ${sql.table('sync_blob_uploads')}
|
|
582
|
-
where partition_id = ${partitionId}
|
|
583
|
-
and hash = ${hash}
|
|
584
|
-
and status = 'complete'
|
|
585
|
-
limit 1
|
|
586
|
-
`.execute(db);
|
|
587
|
-
|
|
588
|
-
return rowResult.rows.length > 0;
|
|
589
|
-
},
|
|
590
|
-
|
|
591
|
-
/**
|
|
592
|
-
* Clean up expired/orphaned uploads.
|
|
593
|
-
*
|
|
594
|
-
* Deletes upload records (and optionally storage) for:
|
|
595
|
-
* - Pending uploads that have expired
|
|
596
|
-
* - Completed uploads with no references (if refCheck provided)
|
|
597
|
-
*/
|
|
598
|
-
async cleanup(options?: {
|
|
599
|
-
/** Check if a blob hash is referenced by any row */
|
|
600
|
-
isReferenced?: (hash: string) => Promise<boolean>;
|
|
601
|
-
/** Delete from storage too (not just tracking table) */
|
|
602
|
-
deleteFromStorage?: boolean;
|
|
603
|
-
/** Optional partition filter for cleanup */
|
|
604
|
-
partitionId?: string;
|
|
605
|
-
}): Promise<{ deleted: number }> {
|
|
606
|
-
const now = new Date().toISOString();
|
|
607
|
-
const partitionId = resolvePartitionId(options?.partitionId);
|
|
608
|
-
const storagePartitionOptions = toStoragePartitionOptions(partitionId);
|
|
609
|
-
|
|
610
|
-
let deleted = 0;
|
|
611
|
-
|
|
612
|
-
// Find and delete expired pending uploads in batches.
|
|
613
|
-
while (true) {
|
|
614
|
-
const expiredResult = await sql<{ hash: string }>`
|
|
615
|
-
select hash
|
|
616
|
-
from ${sql.table('sync_blob_uploads')}
|
|
617
|
-
where partition_id = ${partitionId}
|
|
618
|
-
and status = 'pending'
|
|
619
|
-
and expires_at < ${now}
|
|
620
|
-
order by expires_at asc
|
|
621
|
-
limit ${CLEANUP_BATCH_SIZE}
|
|
622
|
-
`.execute(db);
|
|
623
|
-
const expiredHashes = expiredResult.rows
|
|
624
|
-
.map((row) => row.hash)
|
|
625
|
-
.filter((hash): hash is string => typeof hash === 'string');
|
|
626
|
-
|
|
627
|
-
if (expiredHashes.length === 0) {
|
|
628
|
-
break;
|
|
629
|
-
}
|
|
630
|
-
|
|
631
|
-
if (options?.deleteFromStorage) {
|
|
632
|
-
await runWithConcurrency(
|
|
633
|
-
expiredHashes,
|
|
634
|
-
STORAGE_DELETE_CONCURRENCY,
|
|
635
|
-
async (hash) => {
|
|
636
|
-
try {
|
|
637
|
-
await adapter.delete(hash, storagePartitionOptions);
|
|
638
|
-
} catch {
|
|
639
|
-
// Ignore storage errors during cleanup
|
|
640
|
-
}
|
|
641
|
-
}
|
|
642
|
-
);
|
|
643
|
-
}
|
|
644
|
-
|
|
645
|
-
deleted += await deleteUploadRowsByHashes(partitionId, expiredHashes);
|
|
646
|
-
|
|
647
|
-
if (expiredHashes.length < CLEANUP_BATCH_SIZE) {
|
|
648
|
-
break;
|
|
649
|
-
}
|
|
650
|
-
}
|
|
651
|
-
|
|
652
|
-
// If reference check provided, also clean up unreferenced complete uploads
|
|
653
|
-
// in batches with bounded parallelism.
|
|
654
|
-
if (options?.isReferenced) {
|
|
655
|
-
const isReferenced = options.isReferenced;
|
|
656
|
-
let afterHash: string | null = null;
|
|
657
|
-
|
|
658
|
-
while (true) {
|
|
659
|
-
let completeRows: Array<{ hash: string }>;
|
|
660
|
-
if (afterHash === null) {
|
|
661
|
-
const completeResult = await sql<{ hash: string }>`
|
|
662
|
-
select hash
|
|
663
|
-
from ${sql.table('sync_blob_uploads')}
|
|
664
|
-
where partition_id = ${partitionId}
|
|
665
|
-
and status = 'complete'
|
|
666
|
-
order by hash asc
|
|
667
|
-
limit ${CLEANUP_BATCH_SIZE}
|
|
668
|
-
`.execute(db);
|
|
669
|
-
completeRows = completeResult.rows;
|
|
670
|
-
} else {
|
|
671
|
-
const completeResult = await sql<{ hash: string }>`
|
|
672
|
-
select hash
|
|
673
|
-
from ${sql.table('sync_blob_uploads')}
|
|
674
|
-
where partition_id = ${partitionId}
|
|
675
|
-
and status = 'complete'
|
|
676
|
-
and hash > ${afterHash}
|
|
677
|
-
order by hash asc
|
|
678
|
-
limit ${CLEANUP_BATCH_SIZE}
|
|
679
|
-
`.execute(db);
|
|
680
|
-
completeRows = completeResult.rows;
|
|
681
|
-
}
|
|
682
|
-
|
|
683
|
-
const completeHashes = completeRows
|
|
684
|
-
.map((row) => row.hash)
|
|
685
|
-
.filter((hash): hash is string => typeof hash === 'string');
|
|
686
|
-
|
|
687
|
-
if (completeHashes.length === 0) {
|
|
688
|
-
break;
|
|
689
|
-
}
|
|
690
|
-
|
|
691
|
-
afterHash = completeHashes[completeHashes.length - 1] ?? afterHash;
|
|
692
|
-
|
|
693
|
-
const unreferencedHashes: string[] = [];
|
|
694
|
-
await runWithConcurrency(
|
|
695
|
-
completeHashes,
|
|
696
|
-
REFERENCE_CHECK_CONCURRENCY,
|
|
697
|
-
async (hash) => {
|
|
698
|
-
const referenced = await isReferenced(hash);
|
|
699
|
-
if (!referenced) {
|
|
700
|
-
unreferencedHashes.push(hash);
|
|
701
|
-
}
|
|
702
|
-
}
|
|
703
|
-
);
|
|
704
|
-
|
|
705
|
-
if (unreferencedHashes.length > 0) {
|
|
706
|
-
if (options?.deleteFromStorage) {
|
|
707
|
-
await runWithConcurrency(
|
|
708
|
-
unreferencedHashes,
|
|
709
|
-
STORAGE_DELETE_CONCURRENCY,
|
|
710
|
-
async (hash) => {
|
|
711
|
-
try {
|
|
712
|
-
await adapter.delete(hash, storagePartitionOptions);
|
|
713
|
-
} catch {
|
|
714
|
-
// Ignore storage errors during cleanup
|
|
715
|
-
}
|
|
716
|
-
}
|
|
717
|
-
);
|
|
718
|
-
}
|
|
719
|
-
|
|
720
|
-
deleted += await deleteUploadRowsByHashes(
|
|
721
|
-
partitionId,
|
|
722
|
-
unreferencedHashes
|
|
723
|
-
);
|
|
724
|
-
}
|
|
725
|
-
|
|
726
|
-
if (completeHashes.length < CLEANUP_BATCH_SIZE) {
|
|
727
|
-
break;
|
|
728
|
-
}
|
|
729
|
-
}
|
|
730
|
-
}
|
|
731
|
-
|
|
732
|
-
return { deleted };
|
|
733
|
-
},
|
|
734
|
-
|
|
735
|
-
/** The underlying storage adapter */
|
|
736
|
-
adapter,
|
|
737
|
-
};
|
|
738
|
-
}
|
|
739
|
-
|
|
740
|
-
export type BlobManager = ReturnType<typeof createBlobManager>;
|
|
741
|
-
|
|
742
|
-
// ============================================================================
|
|
743
|
-
// Garbage Collection Scheduler
|
|
744
|
-
// ============================================================================
|
|
745
|
-
|
|
746
|
-
export interface BlobCleanupSchedulerOptions {
|
|
747
|
-
/** Blob manager instance */
|
|
748
|
-
blobManager: BlobManager;
|
|
749
|
-
/** Interval between cleanup runs in milliseconds. Default: 3600000 (1 hour) */
|
|
750
|
-
intervalMs?: number;
|
|
751
|
-
/** Delete from storage too (not just tracking table). Default: true */
|
|
752
|
-
deleteFromStorage?: boolean;
|
|
753
|
-
/** Optional: Check if a blob hash is referenced by any row */
|
|
754
|
-
isReferenced?: (hash: string) => Promise<boolean>;
|
|
755
|
-
/** Optional partition to scope cleanup. Defaults to "default". */
|
|
756
|
-
partitionId?: string;
|
|
757
|
-
/** Optional: Called after each cleanup run */
|
|
758
|
-
onCleanup?: (result: { deleted: number; error?: Error }) => void;
|
|
759
|
-
}
|
|
760
|
-
|
|
761
|
-
/**
|
|
762
|
-
* Create a garbage collection scheduler for blob storage.
|
|
763
|
-
*
|
|
764
|
-
* Periodically runs cleanup to remove:
|
|
765
|
-
* - Expired pending uploads
|
|
766
|
-
* - Unreferenced blobs (if isReferenced callback provided)
|
|
767
|
-
*
|
|
768
|
-
* @example
|
|
769
|
-
* ```typescript
|
|
770
|
-
* const scheduler = createBlobCleanupScheduler({
|
|
771
|
-
* blobManager,
|
|
772
|
-
* intervalMs: 60 * 60 * 1000, // 1 hour
|
|
773
|
-
* deleteFromStorage: true,
|
|
774
|
-
* isReferenced: async (hash) => {
|
|
775
|
-
* const row = await db.selectFrom('my_table')
|
|
776
|
-
* .select('id')
|
|
777
|
-
* .where('blob_hash', '=', hash)
|
|
778
|
-
* .executeTakeFirst();
|
|
779
|
-
* return !!row;
|
|
780
|
-
* },
|
|
781
|
-
* onCleanup: (result) => {
|
|
782
|
-
* console.log(`Cleanup complete: ${result.deleted} blobs removed`);
|
|
783
|
-
* },
|
|
784
|
-
* });
|
|
785
|
-
*
|
|
786
|
-
* // Start the scheduler
|
|
787
|
-
* scheduler.start();
|
|
788
|
-
*
|
|
789
|
-
* // Stop when shutting down
|
|
790
|
-
* scheduler.stop();
|
|
791
|
-
* ```
|
|
792
|
-
*/
|
|
793
|
-
export function createBlobCleanupScheduler(
|
|
794
|
-
options: BlobCleanupSchedulerOptions
|
|
795
|
-
) {
|
|
796
|
-
const {
|
|
797
|
-
blobManager,
|
|
798
|
-
intervalMs = 3600000, // 1 hour
|
|
799
|
-
deleteFromStorage = true,
|
|
800
|
-
isReferenced,
|
|
801
|
-
partitionId,
|
|
802
|
-
onCleanup,
|
|
803
|
-
} = options;
|
|
804
|
-
|
|
805
|
-
let intervalId: ReturnType<typeof setInterval> | null = null;
|
|
806
|
-
let isRunning = false;
|
|
807
|
-
|
|
808
|
-
const runCleanup = async (): Promise<{ deleted: number; error?: Error }> => {
|
|
809
|
-
if (isRunning) {
|
|
810
|
-
return { deleted: 0 };
|
|
811
|
-
}
|
|
812
|
-
|
|
813
|
-
isRunning = true;
|
|
814
|
-
|
|
815
|
-
try {
|
|
816
|
-
const result = await blobManager.cleanup({
|
|
817
|
-
deleteFromStorage,
|
|
818
|
-
isReferenced,
|
|
819
|
-
partitionId,
|
|
820
|
-
});
|
|
821
|
-
|
|
822
|
-
onCleanup?.({ deleted: result.deleted });
|
|
823
|
-
return { deleted: result.deleted };
|
|
824
|
-
} catch (err) {
|
|
825
|
-
const error = err instanceof Error ? err : new Error(String(err));
|
|
826
|
-
onCleanup?.({ deleted: 0, error });
|
|
827
|
-
return { deleted: 0, error };
|
|
828
|
-
} finally {
|
|
829
|
-
isRunning = false;
|
|
830
|
-
}
|
|
831
|
-
};
|
|
832
|
-
|
|
833
|
-
return {
|
|
834
|
-
/**
|
|
835
|
-
* Start the cleanup scheduler.
|
|
836
|
-
* Optionally runs an immediate cleanup before starting the interval.
|
|
837
|
-
*/
|
|
838
|
-
start(options?: { immediate?: boolean }): void {
|
|
839
|
-
if (intervalId) {
|
|
840
|
-
return; // Already running
|
|
841
|
-
}
|
|
842
|
-
|
|
843
|
-
if (options?.immediate) {
|
|
844
|
-
void runCleanup();
|
|
845
|
-
}
|
|
846
|
-
|
|
847
|
-
intervalId = setInterval(() => {
|
|
848
|
-
void runCleanup();
|
|
849
|
-
}, intervalMs);
|
|
850
|
-
},
|
|
851
|
-
|
|
852
|
-
/**
|
|
853
|
-
* Stop the cleanup scheduler.
|
|
854
|
-
*/
|
|
855
|
-
stop(): void {
|
|
856
|
-
if (intervalId) {
|
|
857
|
-
clearInterval(intervalId);
|
|
858
|
-
intervalId = null;
|
|
859
|
-
}
|
|
860
|
-
},
|
|
861
|
-
|
|
862
|
-
/**
|
|
863
|
-
* Run a single cleanup manually.
|
|
864
|
-
*/
|
|
865
|
-
async runOnce(): Promise<{ deleted: number; error?: Error }> {
|
|
866
|
-
return runCleanup();
|
|
867
|
-
},
|
|
868
|
-
|
|
869
|
-
/**
|
|
870
|
-
* Check if the scheduler is currently active.
|
|
871
|
-
*/
|
|
872
|
-
get active(): boolean {
|
|
873
|
-
return intervalId !== null;
|
|
874
|
-
},
|
|
875
|
-
|
|
876
|
-
/**
|
|
877
|
-
* Check if a cleanup is currently in progress.
|
|
878
|
-
*/
|
|
879
|
-
get running(): boolean {
|
|
880
|
-
return isRunning;
|
|
881
|
-
},
|
|
882
|
-
};
|
|
883
|
-
}
|
|
884
|
-
|
|
885
|
-
// ============================================================================
|
|
886
|
-
// Errors
|
|
887
|
-
// ============================================================================
|
|
888
|
-
|
|
889
|
-
export class BlobValidationError extends Error {
|
|
890
|
-
constructor(message: string) {
|
|
891
|
-
super(message);
|
|
892
|
-
this.name = 'BlobValidationError';
|
|
893
|
-
}
|
|
894
|
-
}
|
|
895
|
-
|
|
896
|
-
export class BlobNotFoundError extends Error {
|
|
897
|
-
constructor(message: string) {
|
|
898
|
-
super(message);
|
|
899
|
-
this.name = 'BlobNotFoundError';
|
|
900
|
-
}
|
|
901
|
-
}
|