@syncular/server 0.58.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -22
- package/dist/admin.js +8 -4
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +16 -0
- package/dist/context.js +21 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +223 -1111
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +1 -1
- package/dist/handler.js +135 -103
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +17 -3
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +182 -105
- package/dist/pull.d.ts +28 -20
- package/dist/pull.js +85 -87
- package/dist/push.js +272 -147
- package/dist/realtime.d.ts +68 -28
- package/dist/realtime.js +297 -136
- package/dist/relational-rows.d.ts +27 -15
- package/dist/relational-rows.js +38 -22
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +1 -2
- package/dist/segment-download.js +18 -4
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +150 -47
- package/dist/sqlite-dialect.js +987 -88
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +293 -1117
- package/dist/storage.d.ts +32 -12
- package/package.json +2 -2
- package/src/admin.ts +16 -2
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +32 -1
- package/src/d1-storage.ts +326 -1692
- package/src/errors.ts +8 -0
- package/src/events.ts +1 -1
- package/src/handler.ts +178 -127
- package/src/local-authority.ts +17 -3
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +231 -138
- package/src/pull.ts +133 -147
- package/src/push.ts +356 -205
- package/src/realtime.ts +380 -179
- package/src/relational-rows.ts +54 -27
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +1 -2
- package/src/segment-download.ts +25 -4
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1607 -114
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +360 -1811
- package/src/storage.ts +33 -12
|
@@ -58,7 +58,6 @@ import {
|
|
|
58
58
|
mergeSegmentRecord,
|
|
59
59
|
publicationRecord,
|
|
60
60
|
publicationsForRecord,
|
|
61
|
-
segmentBytesEqual,
|
|
62
61
|
segmentIdFor,
|
|
63
62
|
selectSegmentPublication,
|
|
64
63
|
} from './segment-store';
|
|
@@ -440,23 +439,29 @@ export class BucketSegmentStore implements SegmentStore {
|
|
|
440
439
|
}
|
|
441
440
|
|
|
442
441
|
/**
|
|
443
|
-
* Write the immutable bytes object
|
|
444
|
-
*
|
|
445
|
-
*
|
|
446
|
-
*
|
|
442
|
+
* Write the immutable bytes object unless it is stored already. A HEAD
|
|
443
|
+
* comes first, so republishing a segment (every bootstrap pull of an
|
|
444
|
+
* unchanged window) moves no bytes. The write is create-only, so two first
|
|
445
|
+
* publishers cannot overwrite each other. A stored object under the same
|
|
446
|
+
* content address with a different size is the hash collision §5.1
|
|
447
|
+
* requires to fail loudly; equal size under one SHA-256 address is the same
|
|
448
|
+
* content.
|
|
447
449
|
*/
|
|
448
450
|
async #putBytes(segmentId: string, bytes: Uint8Array): Promise<void> {
|
|
449
451
|
const key = this.objectKeyFor(segmentId);
|
|
450
|
-
const
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
452
|
+
const head = await this.#bucket.head(key);
|
|
453
|
+
if (
|
|
454
|
+
head === undefined &&
|
|
455
|
+
(await this.#bucket.put(key, {
|
|
456
|
+
body: bytes,
|
|
457
|
+
contentType: 'application/octet-stream',
|
|
458
|
+
createOnly: true,
|
|
459
|
+
}))
|
|
460
|
+
) {
|
|
461
|
+
return;
|
|
462
|
+
}
|
|
463
|
+
const stored = head ?? (await this.#bucket.head(key));
|
|
464
|
+
if (stored !== undefined && stored.size !== bytes.length) {
|
|
460
465
|
throw new Error(
|
|
461
466
|
`${this.#label}: content-address collision at ${segmentId} (§5.1)`,
|
|
462
467
|
);
|
package/src/content-encoding.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* §1.3 transport compression for
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* §1.3 transport compression for the HTTP bodies the adapters serve —
|
|
3
|
+
* `<mount>/sync`, `<mount>/operations` and segment downloads (blob downloads
|
|
4
|
+
* stream identity, §5.9.5): negotiate `Content-Encoding` from the request's
|
|
5
5
|
* `Accept-Encoding` (zstd preferred, gzip fallback, identity when the client
|
|
6
6
|
* offers neither), feature-detecting the runtime's codecs, zero
|
|
7
7
|
* dependencies. Bun supplies zstd + gzip natively; every other runtime
|
|
@@ -167,31 +167,63 @@ export function encodeStream(
|
|
|
167
167
|
* `Content-Encoding: gzip` (§1.3: clients MAY compress a request body and
|
|
168
168
|
* servers MUST accept it). Returns `undefined` for any other declared
|
|
169
169
|
* encoding — the adapter answers 415 rather than parse bytes it cannot read.
|
|
170
|
-
* A gzip body that does not decompress is a `sync.invalid_request`.
|
|
170
|
+
* A gzip body that does not decompress is a `sync.invalid_request`. One read
|
|
171
|
+
* loop counts the decoded bytes of identity and gzip bodies alike and stops
|
|
172
|
+
* with `sync.limit_exceeded` past `maxBytes` (§1.7 `maxRequestBytes`), so a
|
|
173
|
+
* small compressed body never inflates past the cap.
|
|
171
174
|
*/
|
|
172
175
|
export async function decodeRequestBody(
|
|
173
176
|
request: Request,
|
|
177
|
+
maxBytes: number,
|
|
174
178
|
): Promise<Uint8Array | undefined> {
|
|
175
179
|
const encoding = request.headers
|
|
176
180
|
.get('content-encoding')
|
|
177
181
|
?.trim()
|
|
178
182
|
.toLowerCase();
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
+
const identity =
|
|
184
|
+
encoding === undefined || encoding === '' || encoding === 'identity';
|
|
185
|
+
if (
|
|
186
|
+
!identity &&
|
|
187
|
+
(encoding !== 'gzip' || typeof DecompressionStream !== 'function')
|
|
188
|
+
) {
|
|
183
189
|
return undefined;
|
|
184
190
|
}
|
|
185
191
|
if (request.body === null) {
|
|
192
|
+
if (identity) return new Uint8Array(0);
|
|
186
193
|
throw syncError('sync.invalid_request', 'malformed gzip request body');
|
|
187
194
|
}
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
195
|
+
const reader = (
|
|
196
|
+
identity
|
|
197
|
+
? request.body
|
|
198
|
+
: request.body.pipeThrough(new DecompressionStream('gzip'))
|
|
199
|
+
).getReader();
|
|
200
|
+
const chunks: Uint8Array[] = [];
|
|
201
|
+
let total = 0;
|
|
202
|
+
for (;;) {
|
|
203
|
+
let step: Awaited<ReturnType<typeof reader.read>>;
|
|
204
|
+
try {
|
|
205
|
+
step = await reader.read();
|
|
206
|
+
} catch (error) {
|
|
207
|
+
if (identity) throw error;
|
|
208
|
+
throw syncError('sync.invalid_request', 'malformed gzip request body');
|
|
209
|
+
}
|
|
210
|
+
if (step.done) break;
|
|
211
|
+
total += step.value.length;
|
|
212
|
+
if (total > maxBytes) {
|
|
213
|
+
await reader.cancel().catch(() => {});
|
|
214
|
+
throw syncError(
|
|
215
|
+
'sync.limit_exceeded',
|
|
216
|
+
'request body exceeds the server limit (§1.7)',
|
|
217
|
+
JSON.stringify({ limit: 'maxRequestBytes', max: maxBytes }),
|
|
218
|
+
);
|
|
219
|
+
}
|
|
220
|
+
chunks.push(step.value);
|
|
221
|
+
}
|
|
222
|
+
const bytes = new Uint8Array(total);
|
|
223
|
+
let offset = 0;
|
|
224
|
+
for (const chunk of chunks) {
|
|
225
|
+
bytes.set(chunk, offset);
|
|
226
|
+
offset += chunk.length;
|
|
196
227
|
}
|
|
228
|
+
return bytes;
|
|
197
229
|
}
|
package/src/context.ts
CHANGED
|
@@ -82,12 +82,31 @@ export interface ServerLimits {
|
|
|
82
82
|
readonly inlineSegmentMaxBytes: number;
|
|
83
83
|
/** §6.11 declared-reference cascade cap per commit (reference default). */
|
|
84
84
|
readonly maxCascadeOperationsPerCommit: number;
|
|
85
|
+
/**
|
|
86
|
+
* §1.7 cap on a request body's decoded bytes: an HTTP body after gzip
|
|
87
|
+
* decoding, and a socket round's assembled request (§8.7).
|
|
88
|
+
*/
|
|
89
|
+
readonly maxRequestBytes: number;
|
|
90
|
+
/** §1.7 cap on `SUBSCRIPTION` frames per request. */
|
|
91
|
+
readonly maxSubscriptionsPerRequest: number;
|
|
92
|
+
/** §1.7 cap on requested scope values, summed over a subscription's keys. */
|
|
93
|
+
readonly maxScopeValuesPerSubscription: number;
|
|
94
|
+
/** Cap on concurrent registered-query watches per watch session. */
|
|
95
|
+
readonly maxWatchesPerSession: number;
|
|
85
96
|
}
|
|
86
97
|
|
|
87
98
|
export const DEFAULT_LIMITS: ServerLimits = {
|
|
88
99
|
maxOperationsPerRequest: 500,
|
|
89
100
|
inlineSegmentMaxBytes: 256 * 1024,
|
|
90
101
|
maxCascadeOperationsPerCommit: 1_000,
|
|
102
|
+
// The size above which adapters already refuse to buffer a segment body.
|
|
103
|
+
maxRequestBytes: 16 * 1024 * 1024,
|
|
104
|
+
// One subscription per pushable operation: the existing per-request cap.
|
|
105
|
+
maxSubscriptionsPerRequest: 500,
|
|
106
|
+
// One default snapshot page (§4.2): a longer list wants a coarser scope.
|
|
107
|
+
maxScopeValuesPerSubscription: 1_000,
|
|
108
|
+
// Every commit reruns a session's affected watches one after another.
|
|
109
|
+
maxWatchesPerSession: 100,
|
|
91
110
|
};
|
|
92
111
|
|
|
93
112
|
/** Receives applied commits for realtime fanout (§8.2). */
|
|
@@ -235,8 +254,20 @@ export function clockOf(ctx: SyncServerConfig): () => number {
|
|
|
235
254
|
return ctx.clock ?? Date.now;
|
|
236
255
|
}
|
|
237
256
|
|
|
257
|
+
/**
|
|
258
|
+
* The configured limits over the defaults. A cap that is not a positive safe
|
|
259
|
+
* integer (`NaN`, `0`, `Infinity`) would disable or jam it, so it throws;
|
|
260
|
+
* `inlineSegmentMaxBytes` is a threshold and may be `0` (inline nothing).
|
|
261
|
+
*/
|
|
238
262
|
export function limitsOf(ctx: SyncServerConfig): ServerLimits {
|
|
239
|
-
|
|
263
|
+
const limits: ServerLimits = { ...DEFAULT_LIMITS, ...ctx.limits };
|
|
264
|
+
for (const [name, value] of Object.entries(limits)) {
|
|
265
|
+
const min = name === 'inlineSegmentMaxBytes' ? 0 : 1;
|
|
266
|
+
if (!Number.isSafeInteger(value) || value < min) {
|
|
267
|
+
throw new Error(`ServerLimits.${name} is out of range`);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
return limits;
|
|
240
271
|
}
|
|
241
272
|
|
|
242
273
|
/** Refresh the registry after host authentication and return log continuity. */
|