@syncular/server 0.57.0 → 0.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +31 -22
  2. package/dist/admin.js +8 -4
  3. package/dist/blob-handlers.d.ts +25 -31
  4. package/dist/blob-handlers.js +63 -41
  5. package/dist/blob-store.d.ts +42 -24
  6. package/dist/blob-store.js +0 -0
  7. package/dist/bucket-blob-store.d.ts +12 -8
  8. package/dist/bucket-blob-store.js +54 -45
  9. package/dist/bucket-segment-store.js +18 -16
  10. package/dist/content-encoding.d.ts +5 -2
  11. package/dist/content-encoding.js +42 -13
  12. package/dist/context.d.ts +16 -0
  13. package/dist/context.js +21 -1
  14. package/dist/d1-storage.d.ts +14 -98
  15. package/dist/d1-storage.js +223 -1111
  16. package/dist/errors.js +8 -0
  17. package/dist/events.d.ts +1 -1
  18. package/dist/handler.js +135 -103
  19. package/dist/local-authority.d.ts +2 -0
  20. package/dist/local-authority.js +17 -3
  21. package/dist/object-bucket.d.ts +2 -0
  22. package/dist/operations-realtime.js +69 -50
  23. package/dist/pg-executor-pglite.d.ts +5 -6
  24. package/dist/pg-executor-pglite.js +22 -14
  25. package/dist/pg-executor.d.ts +4 -1
  26. package/dist/postgres-storage.d.ts +1 -1
  27. package/dist/postgres-storage.js +182 -105
  28. package/dist/pull.d.ts +28 -20
  29. package/dist/pull.js +85 -87
  30. package/dist/push.js +272 -147
  31. package/dist/realtime.d.ts +68 -28
  32. package/dist/realtime.js +297 -136
  33. package/dist/relational-rows.d.ts +27 -15
  34. package/dist/relational-rows.js +38 -22
  35. package/dist/s3-blob-store.d.ts +11 -12
  36. package/dist/s3-blob-store.js +20 -13
  37. package/dist/s3-object-bucket.d.ts +8 -1
  38. package/dist/s3-object-bucket.js +16 -3
  39. package/dist/schema.d.ts +14 -0
  40. package/dist/schema.js +23 -0
  41. package/dist/scopes.d.ts +6 -2
  42. package/dist/scopes.js +22 -4
  43. package/dist/seed.js +1 -2
  44. package/dist/segment-download.js +18 -4
  45. package/dist/signed-url.d.ts +2 -2
  46. package/dist/signed-url.js +9 -2
  47. package/dist/sigv4.d.ts +8 -2
  48. package/dist/sigv4.js +8 -4
  49. package/dist/sqlite-blob-store.d.ts +1 -1
  50. package/dist/sqlite-blob-store.js +23 -22
  51. package/dist/sqlite-dialect.d.ts +150 -47
  52. package/dist/sqlite-dialect.js +987 -88
  53. package/dist/sqlite-lease-store.js +13 -18
  54. package/dist/sqlite-storage.d.ts +9 -62
  55. package/dist/sqlite-storage.js +293 -1117
  56. package/dist/storage.d.ts +32 -12
  57. package/package.json +2 -2
  58. package/src/admin.ts +16 -2
  59. package/src/blob-handlers.ts +91 -72
  60. package/src/blob-store.ts +0 -0
  61. package/src/bucket-blob-store.ts +75 -48
  62. package/src/bucket-segment-store.ts +20 -15
  63. package/src/content-encoding.ts +48 -16
  64. package/src/context.ts +32 -1
  65. package/src/d1-storage.ts +326 -1692
  66. package/src/errors.ts +8 -0
  67. package/src/events.ts +1 -1
  68. package/src/handler.ts +178 -127
  69. package/src/local-authority.ts +17 -3
  70. package/src/object-bucket.ts +2 -0
  71. package/src/operations-realtime.ts +85 -61
  72. package/src/pg-executor-pglite.ts +27 -23
  73. package/src/pg-executor.ts +5 -1
  74. package/src/postgres-storage.ts +231 -138
  75. package/src/pull.ts +133 -147
  76. package/src/push.ts +356 -205
  77. package/src/realtime.ts +380 -179
  78. package/src/relational-rows.ts +54 -27
  79. package/src/s3-blob-store.ts +23 -13
  80. package/src/s3-object-bucket.ts +20 -2
  81. package/src/schema.ts +31 -0
  82. package/src/scopes.ts +23 -3
  83. package/src/seed.ts +1 -2
  84. package/src/segment-download.ts +25 -4
  85. package/src/signed-url.ts +10 -4
  86. package/src/sigv4.ts +17 -4
  87. package/src/sqlite-blob-store.ts +27 -26
  88. package/src/sqlite-dialect.ts +1607 -114
  89. package/src/sqlite-lease-store.ts +12 -23
  90. package/src/sqlite-storage.ts +360 -1811
  91. package/src/storage.ts +33 -12
@@ -58,7 +58,6 @@ import {
58
58
  mergeSegmentRecord,
59
59
  publicationRecord,
60
60
  publicationsForRecord,
61
- segmentBytesEqual,
62
61
  segmentIdFor,
63
62
  selectSegmentPublication,
64
63
  } from './segment-store';
@@ -440,23 +439,29 @@ export class BucketSegmentStore implements SegmentStore {
440
439
  }
441
440
 
442
441
  /**
443
- * Write the immutable bytes object. Create-only, so a second publisher of
444
- * identical bytes is a no-op and a second publisher of DIFFERENT bytes under
445
- * the same content address is the hash collision §5.1 requires to fail
446
- * loudly instead of overwriting.
442
+ * Write the immutable bytes object unless it is stored already. A HEAD
443
+ * comes first, so republishing a segment (every bootstrap pull of an
444
+ * unchanged window) moves no bytes. The write is create-only, so two first
445
+ * publishers cannot overwrite each other. A stored object under the same
446
+ * content address with a different size is the hash collision §5.1
447
+ * requires to fail loudly; equal size under one SHA-256 address is the same
448
+ * content.
447
449
  */
448
450
  async #putBytes(segmentId: string, bytes: Uint8Array): Promise<void> {
449
451
  const key = this.objectKeyFor(segmentId);
450
- const written = await this.#bucket.put(key, {
451
- body: bytes,
452
- contentType: 'application/octet-stream',
453
- createOnly: true,
454
- });
455
- if (written) return;
456
- const current = await this.#bucket.get(key);
457
- if (current === undefined) return;
458
- const stored = new Uint8Array(await current.arrayBuffer());
459
- if (!segmentBytesEqual(stored, bytes)) {
452
+ const head = await this.#bucket.head(key);
453
+ if (
454
+ head === undefined &&
455
+ (await this.#bucket.put(key, {
456
+ body: bytes,
457
+ contentType: 'application/octet-stream',
458
+ createOnly: true,
459
+ }))
460
+ ) {
461
+ return;
462
+ }
463
+ const stored = head ?? (await this.#bucket.head(key));
464
+ if (stored !== undefined && stored.size !== bytes.length) {
460
465
  throw new Error(
461
466
  `${this.#label}: content-address collision at ${segmentId} (§5.1)`,
462
467
  );
@@ -1,7 +1,7 @@
1
1
  /**
2
- * §1.3 transport compression for every buffered HTTP body the adapters
3
- * serve — `<mount>/sync`, `<mount>/operations`, blob GETs and segment
4
- * downloads: negotiate `Content-Encoding` from the request's
2
+ * §1.3 transport compression for the HTTP bodies the adapters serve —
3
+ * `<mount>/sync`, `<mount>/operations` and segment downloads (blob downloads
4
+ * stream identity, §5.9.5): negotiate `Content-Encoding` from the request's
5
5
  * `Accept-Encoding` (zstd preferred, gzip fallback, identity when the client
6
6
  * offers neither), feature-detecting the runtime's codecs, zero
7
7
  * dependencies. Bun supplies zstd + gzip natively; every other runtime
@@ -167,31 +167,63 @@ export function encodeStream(
167
167
  * `Content-Encoding: gzip` (§1.3: clients MAY compress a request body and
168
168
  * servers MUST accept it). Returns `undefined` for any other declared
169
169
  * encoding — the adapter answers 415 rather than parse bytes it cannot read.
170
- * A gzip body that does not decompress is a `sync.invalid_request`.
170
+ * A gzip body that does not decompress is a `sync.invalid_request`. One read
171
+ * loop counts the decoded bytes of identity and gzip bodies alike and stops
172
+ * with `sync.limit_exceeded` past `maxBytes` (§1.7 `maxRequestBytes`), so a
173
+ * small compressed body never inflates past the cap.
171
174
  */
172
175
  export async function decodeRequestBody(
173
176
  request: Request,
177
+ maxBytes: number,
174
178
  ): Promise<Uint8Array | undefined> {
175
179
  const encoding = request.headers
176
180
  .get('content-encoding')
177
181
  ?.trim()
178
182
  .toLowerCase();
179
- if (encoding === undefined || encoding === '' || encoding === 'identity') {
180
- return new Uint8Array(await request.arrayBuffer());
181
- }
182
- if (encoding !== 'gzip' || typeof DecompressionStream !== 'function') {
183
+ const identity =
184
+ encoding === undefined || encoding === '' || encoding === 'identity';
185
+ if (
186
+ !identity &&
187
+ (encoding !== 'gzip' || typeof DecompressionStream !== 'function')
188
+ ) {
183
189
  return undefined;
184
190
  }
185
191
  if (request.body === null) {
192
+ if (identity) return new Uint8Array(0);
186
193
  throw syncError('sync.invalid_request', 'malformed gzip request body');
187
194
  }
188
- try {
189
- return new Uint8Array(
190
- await new Response(
191
- request.body.pipeThrough(new DecompressionStream('gzip')),
192
- ).arrayBuffer(),
193
- );
194
- } catch {
195
- throw syncError('sync.invalid_request', 'malformed gzip request body');
195
+ const reader = (
196
+ identity
197
+ ? request.body
198
+ : request.body.pipeThrough(new DecompressionStream('gzip'))
199
+ ).getReader();
200
+ const chunks: Uint8Array[] = [];
201
+ let total = 0;
202
+ for (;;) {
203
+ let step: Awaited<ReturnType<typeof reader.read>>;
204
+ try {
205
+ step = await reader.read();
206
+ } catch (error) {
207
+ if (identity) throw error;
208
+ throw syncError('sync.invalid_request', 'malformed gzip request body');
209
+ }
210
+ if (step.done) break;
211
+ total += step.value.length;
212
+ if (total > maxBytes) {
213
+ await reader.cancel().catch(() => {});
214
+ throw syncError(
215
+ 'sync.limit_exceeded',
216
+ 'request body exceeds the server limit (§1.7)',
217
+ JSON.stringify({ limit: 'maxRequestBytes', max: maxBytes }),
218
+ );
219
+ }
220
+ chunks.push(step.value);
221
+ }
222
+ const bytes = new Uint8Array(total);
223
+ let offset = 0;
224
+ for (const chunk of chunks) {
225
+ bytes.set(chunk, offset);
226
+ offset += chunk.length;
196
227
  }
228
+ return bytes;
197
229
  }
package/src/context.ts CHANGED
@@ -82,12 +82,31 @@ export interface ServerLimits {
82
82
  readonly inlineSegmentMaxBytes: number;
83
83
  /** §6.11 declared-reference cascade cap per commit (reference default). */
84
84
  readonly maxCascadeOperationsPerCommit: number;
85
+ /**
86
+ * §1.7 cap on a request body's decoded bytes: an HTTP body after gzip
87
+ * decoding, and a socket round's assembled request (§8.7).
88
+ */
89
+ readonly maxRequestBytes: number;
90
+ /** §1.7 cap on `SUBSCRIPTION` frames per request. */
91
+ readonly maxSubscriptionsPerRequest: number;
92
+ /** §1.7 cap on requested scope values, summed over a subscription's keys. */
93
+ readonly maxScopeValuesPerSubscription: number;
94
+ /** Cap on concurrent registered-query watches per watch session. */
95
+ readonly maxWatchesPerSession: number;
85
96
  }
86
97
 
87
98
  export const DEFAULT_LIMITS: ServerLimits = {
88
99
  maxOperationsPerRequest: 500,
89
100
  inlineSegmentMaxBytes: 256 * 1024,
90
101
  maxCascadeOperationsPerCommit: 1_000,
102
+ // The size above which adapters already refuse to buffer a segment body.
103
+ maxRequestBytes: 16 * 1024 * 1024,
104
+ // One subscription per pushable operation: the existing per-request cap.
105
+ maxSubscriptionsPerRequest: 500,
106
+ // One default snapshot page (§4.2): a longer list wants a coarser scope.
107
+ maxScopeValuesPerSubscription: 1_000,
108
+ // Every commit reruns a session's affected watches one after another.
109
+ maxWatchesPerSession: 100,
91
110
  };
92
111
 
93
112
  /** Receives applied commits for realtime fanout (§8.2). */
@@ -235,8 +254,20 @@ export function clockOf(ctx: SyncServerConfig): () => number {
235
254
  return ctx.clock ?? Date.now;
236
255
  }
237
256
 
257
+ /**
258
+ * The configured limits over the defaults. A cap that is not a positive safe
259
+ * integer (`NaN`, `0`, `Infinity`) would disable or jam it, so it throws;
260
+ * `inlineSegmentMaxBytes` is a threshold and may be `0` (inline nothing).
261
+ */
238
262
  export function limitsOf(ctx: SyncServerConfig): ServerLimits {
239
- return { ...DEFAULT_LIMITS, ...ctx.limits };
263
+ const limits: ServerLimits = { ...DEFAULT_LIMITS, ...ctx.limits };
264
+ for (const [name, value] of Object.entries(limits)) {
265
+ const min = name === 'inlineSegmentMaxBytes' ? 0 : 1;
266
+ if (!Number.isSafeInteger(value) || value < min) {
267
+ throw new Error(`ServerLimits.${name} is out of range`);
268
+ }
269
+ }
270
+ return limits;
240
271
  }
241
272
 
242
273
  /** Refresh the registry after host authentication and return log continuity. */