@syncular/server 0.38.0 → 0.39.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/content-encoding.d.ts +24 -23
- package/dist/content-encoding.js +77 -22
- package/dist/d1-storage.js +1 -1
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +207 -64
- package/dist/pull.d.ts +7 -3
- package/dist/pull.js +21 -12
- package/dist/relational-rows.d.ts +16 -10
- package/dist/relational-rows.js +48 -42
- package/dist/sqlite-storage.js +16 -3
- package/package.json +2 -2
- package/src/content-encoding.ts +99 -31
- package/src/d1-storage.ts +1 -1
- package/src/postgres-storage.ts +277 -93
- package/src/pull.ts +31 -18
- package/src/relational-rows.ts +53 -47
- package/src/sqlite-storage.ts +20 -3
|
@@ -1,44 +1,45 @@
|
|
|
1
|
+
export type ContentEncoding = 'zstd' | 'gzip';
|
|
2
|
+
/** Does the `Accept-Encoding` header value accept `encoding` (q > 0)? */
|
|
3
|
+
export declare function acceptsEncoding(header: string, encoding: ContentEncoding): boolean;
|
|
1
4
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* neither), feature-detecting the runtime's codecs — zero dependencies.
|
|
6
|
-
*
|
|
7
|
-
* Segment bytes are stored and content-addressed UNCOMPRESSED (§5.1);
|
|
8
|
-
* this is a per-response serving concern only. It is never applied to
|
|
9
|
-
* stored objects or the signed-URL path (§5.8: presigned GETs serve the
|
|
10
|
-
* addressed bytes verbatim; edge compression is a deployment concern).
|
|
11
|
-
*
|
|
12
|
-
* Decision data (2026-07-03, 100k-row bench table, Bun 1.3): rows
|
|
13
|
-
* segments 6.2× in 8 ms (zstd) / 7.7× in 42 ms (gzip); sqlite images
|
|
14
|
-
* 3.4× in 14 ms (zstd) / 51 ms (gzip); client decompression ≤ 8 ms.
|
|
5
|
+
* Add `Accept-Encoding` to a response's `Vary` header. Every response whose
|
|
6
|
+
* body was negotiated carries it, so a cache never serves one encoding to a
|
|
7
|
+
* client that asked for another.
|
|
15
8
|
*/
|
|
16
|
-
export
|
|
17
|
-
export interface
|
|
9
|
+
export declare function varyOnAcceptEncoding(headers: Headers): void;
|
|
10
|
+
export interface EncodedBody {
|
|
18
11
|
readonly bytes: Uint8Array;
|
|
19
12
|
/** Absent ⇒ identity (no `Content-Encoding` header). */
|
|
20
|
-
readonly contentEncoding?:
|
|
13
|
+
readonly contentEncoding?: ContentEncoding;
|
|
21
14
|
}
|
|
22
15
|
/**
|
|
23
|
-
* Compress a buffered
|
|
16
|
+
* Compress a buffered response body per §1.3. Returns the original bytes
|
|
24
17
|
* (identity) when the client offered no supported encoding, the body is
|
|
25
18
|
* tiny, or the runtime has no codec.
|
|
26
19
|
*/
|
|
27
|
-
export declare function
|
|
20
|
+
export declare function encodeBody(bytes: Uint8Array, acceptEncoding: string | undefined): Promise<EncodedBody>;
|
|
28
21
|
/**
|
|
29
22
|
* Segments above this size are relayed as a stream instead of being
|
|
30
23
|
* buffered and compressed in memory: a sqlite image can exceed the memory of
|
|
31
24
|
* a Workers isolate (128 MB).
|
|
32
25
|
*/
|
|
33
26
|
export declare const SEGMENT_STREAM_THRESHOLD_BYTES: number;
|
|
34
|
-
export interface
|
|
27
|
+
export interface EncodedStreamBody {
|
|
35
28
|
readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
|
|
36
29
|
/** Absent ⇒ identity (no `Content-Encoding` header). */
|
|
37
30
|
readonly contentEncoding?: 'gzip';
|
|
38
31
|
}
|
|
39
32
|
/**
|
|
40
|
-
* Compress a streamed
|
|
41
|
-
*
|
|
42
|
-
*
|
|
33
|
+
* Compress a streamed body per §1.3 with the runtime's `CompressionStream`:
|
|
34
|
+
* gzip when the client accepts it, identity otherwise (zstd has no standard
|
|
35
|
+
* streaming codec).
|
|
43
36
|
*/
|
|
44
|
-
export declare function
|
|
37
|
+
export declare function encodeStream(body: ReadableStream<Uint8Array<ArrayBuffer>>, acceptEncoding: string | undefined): EncodedStreamBody;
|
|
38
|
+
/**
|
|
39
|
+
* The bytes of a request body, decompressed when the client sent
|
|
40
|
+
* `Content-Encoding: gzip` (§1.3: clients MAY compress a request body and
|
|
41
|
+
* servers MUST accept it). Returns `undefined` for any other declared
|
|
42
|
+
* encoding — the adapter answers 415 rather than parse bytes it cannot read.
|
|
43
|
+
* A gzip body that does not decompress is a `sync.invalid_request`.
|
|
44
|
+
*/
|
|
45
|
+
export declare function decodeRequestBody(request: Request): Promise<Uint8Array | undefined>;
|
package/dist/content-encoding.js
CHANGED
|
@@ -1,20 +1,33 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* §
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
2
|
+
* §1.3 transport compression for every buffered HTTP body the adapters
|
|
3
|
+
* serve — `<mount>/sync`, `<mount>/operations`, blob GETs and segment
|
|
4
|
+
* downloads: negotiate `Content-Encoding` from the request's
|
|
5
|
+
* `Accept-Encoding` (zstd preferred, gzip fallback, identity when the client
|
|
6
|
+
* offers neither), feature-detecting the runtime's codecs, zero
|
|
7
|
+
* dependencies. Bun supplies zstd + gzip natively; every other runtime
|
|
8
|
+
* (workerd, Node, Deno) gets gzip through the standard `CompressionStream`.
|
|
6
9
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
10
|
+
* Body bytes are stored and content-addressed UNCOMPRESSED (§5.1); this is a
|
|
11
|
+
* per-response serving concern only. It is never applied to stored objects or
|
|
12
|
+
* the signed-URL path (§5.8: presigned GETs serve the addressed bytes
|
|
13
|
+
* verbatim; edge compression is a deployment concern). Clients MAY gzip a
|
|
14
|
+
* request body (§1.3) — `decodeRequestBody` is the accepting half.
|
|
11
15
|
*
|
|
12
16
|
* Decision data (2026-07-03, 100k-row bench table, Bun 1.3): rows
|
|
13
17
|
* segments 6.2× in 8 ms (zstd) / 7.7× in 42 ms (gzip); sqlite images
|
|
14
18
|
* 3.4× in 14 ms (zstd) / 51 ms (gzip); client decompression ≤ 8 ms.
|
|
15
19
|
*/
|
|
16
|
-
|
|
17
|
-
|
|
20
|
+
import { syncError } from './errors.js';
|
|
21
|
+
/** Bodies below this size are served identity (headers outweigh gains). */
|
|
22
|
+
const MIN_COMPRESS_BYTES = 1024;
|
|
23
|
+
async function gzipWithCompressionStream(bytes) {
|
|
24
|
+
const compressed = await new Response(new Blob([bytes.slice().buffer])
|
|
25
|
+
.stream()
|
|
26
|
+
.pipeThrough(new CompressionStream('gzip'))).arrayBuffer();
|
|
27
|
+
return new Uint8Array(compressed);
|
|
28
|
+
}
|
|
29
|
+
/** Preference order is zstd, gzip (§1.3). A runtime with neither codec
|
|
30
|
+
* serves identity: negotiation, never a hard requirement here. */
|
|
18
31
|
function detectCodecs() {
|
|
19
32
|
const bun = globalThis.Bun;
|
|
20
33
|
const codecs = [];
|
|
@@ -26,13 +39,14 @@ function detectCodecs() {
|
|
|
26
39
|
if (typeof gzip === 'function') {
|
|
27
40
|
codecs.push({ encoding: 'gzip', compress: (bytes) => gzip(bytes) });
|
|
28
41
|
}
|
|
42
|
+
else if (typeof CompressionStream === 'function') {
|
|
43
|
+
codecs.push({ encoding: 'gzip', compress: gzipWithCompressionStream });
|
|
44
|
+
}
|
|
29
45
|
return codecs;
|
|
30
46
|
}
|
|
31
47
|
const CODECS = detectCodecs();
|
|
32
|
-
/** Bodies below this size are served identity (headers outweigh gains). */
|
|
33
|
-
const MIN_COMPRESS_BYTES = 1024;
|
|
34
48
|
/** Does the `Accept-Encoding` header value accept `encoding` (q > 0)? */
|
|
35
|
-
function
|
|
49
|
+
export function acceptsEncoding(header, encoding) {
|
|
36
50
|
for (const entry of header.split(',')) {
|
|
37
51
|
const [rawName, ...params] = entry.trim().split(';');
|
|
38
52
|
const name = rawName?.trim().toLowerCase();
|
|
@@ -49,18 +63,31 @@ function accepts(header, encoding) {
|
|
|
49
63
|
return false;
|
|
50
64
|
}
|
|
51
65
|
/**
|
|
52
|
-
*
|
|
66
|
+
* Add `Accept-Encoding` to a response's `Vary` header. Every response whose
|
|
67
|
+
* body was negotiated carries it, so a cache never serves one encoding to a
|
|
68
|
+
* client that asked for another.
|
|
69
|
+
*/
|
|
70
|
+
export function varyOnAcceptEncoding(headers) {
|
|
71
|
+
const vary = headers.get('vary');
|
|
72
|
+
if (vary === null)
|
|
73
|
+
headers.set('vary', 'Accept-Encoding');
|
|
74
|
+
else if (!/\baccept-encoding\b/i.test(vary)) {
|
|
75
|
+
headers.set('vary', `${vary}, Accept-Encoding`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Compress a buffered response body per §1.3. Returns the original bytes
|
|
53
80
|
* (identity) when the client offered no supported encoding, the body is
|
|
54
81
|
* tiny, or the runtime has no codec.
|
|
55
82
|
*/
|
|
56
|
-
export function
|
|
83
|
+
export async function encodeBody(bytes, acceptEncoding) {
|
|
57
84
|
if (acceptEncoding === undefined || bytes.length < MIN_COMPRESS_BYTES) {
|
|
58
85
|
return { bytes };
|
|
59
86
|
}
|
|
60
87
|
for (const codec of CODECS) {
|
|
61
|
-
if (
|
|
88
|
+
if (acceptsEncoding(acceptEncoding, codec.encoding)) {
|
|
62
89
|
return {
|
|
63
|
-
bytes: codec.compress(bytes),
|
|
90
|
+
bytes: await codec.compress(bytes),
|
|
64
91
|
contentEncoding: codec.encoding,
|
|
65
92
|
};
|
|
66
93
|
}
|
|
@@ -74,13 +101,13 @@ export function encodeSegmentBody(bytes, acceptEncoding) {
|
|
|
74
101
|
*/
|
|
75
102
|
export const SEGMENT_STREAM_THRESHOLD_BYTES = 16 * 1024 * 1024;
|
|
76
103
|
/**
|
|
77
|
-
* Compress a streamed
|
|
78
|
-
*
|
|
79
|
-
*
|
|
104
|
+
* Compress a streamed body per §1.3 with the runtime's `CompressionStream`:
|
|
105
|
+
* gzip when the client accepts it, identity otherwise (zstd has no standard
|
|
106
|
+
* streaming codec).
|
|
80
107
|
*/
|
|
81
|
-
export function
|
|
108
|
+
export function encodeStream(body, acceptEncoding) {
|
|
82
109
|
if (acceptEncoding === undefined ||
|
|
83
|
-
!
|
|
110
|
+
!acceptsEncoding(acceptEncoding, 'gzip') ||
|
|
84
111
|
typeof CompressionStream !== 'function') {
|
|
85
112
|
return { body };
|
|
86
113
|
}
|
|
@@ -89,3 +116,31 @@ export function encodeSegmentStream(body, acceptEncoding) {
|
|
|
89
116
|
contentEncoding: 'gzip',
|
|
90
117
|
};
|
|
91
118
|
}
|
|
119
|
+
/**
|
|
120
|
+
* The bytes of a request body, decompressed when the client sent
|
|
121
|
+
* `Content-Encoding: gzip` (§1.3: clients MAY compress a request body and
|
|
122
|
+
* servers MUST accept it). Returns `undefined` for any other declared
|
|
123
|
+
* encoding — the adapter answers 415 rather than parse bytes it cannot read.
|
|
124
|
+
* A gzip body that does not decompress is a `sync.invalid_request`.
|
|
125
|
+
*/
|
|
126
|
+
export async function decodeRequestBody(request) {
|
|
127
|
+
const encoding = request.headers
|
|
128
|
+
.get('content-encoding')
|
|
129
|
+
?.trim()
|
|
130
|
+
.toLowerCase();
|
|
131
|
+
if (encoding === undefined || encoding === '' || encoding === 'identity') {
|
|
132
|
+
return new Uint8Array(await request.arrayBuffer());
|
|
133
|
+
}
|
|
134
|
+
if (encoding !== 'gzip' || typeof DecompressionStream !== 'function') {
|
|
135
|
+
return undefined;
|
|
136
|
+
}
|
|
137
|
+
if (request.body === null) {
|
|
138
|
+
throw syncError('sync.invalid_request', 'malformed gzip request body');
|
|
139
|
+
}
|
|
140
|
+
try {
|
|
141
|
+
return new Uint8Array(await new Response(request.body.pipeThrough(new DecompressionStream('gzip'))).arrayBuffer());
|
|
142
|
+
}
|
|
143
|
+
catch {
|
|
144
|
+
throw syncError('sync.invalid_request', 'malformed gzip request body');
|
|
145
|
+
}
|
|
146
|
+
}
|
package/dist/d1-storage.js
CHANGED
|
@@ -943,7 +943,7 @@ export class D1ServerStorage {
|
|
|
943
943
|
? migratePayload(plan.oldLayout, table, bytes)
|
|
944
944
|
: bytes;
|
|
945
945
|
return this.#db
|
|
946
|
-
.prepare(rewriteRowSql(table
|
|
946
|
+
.prepare(rewriteRowSql(table))
|
|
947
947
|
.bind(...rewriteValues(table, row.partition, row.row_id, payload, 'sqlite'));
|
|
948
948
|
});
|
|
949
949
|
const last = results.at(-1);
|
|
@@ -53,7 +53,7 @@ import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDec
|
|
|
53
53
|
* their table.
|
|
54
54
|
*
|
|
55
55
|
* `sync_row_scopes_by_row (partition, tbl, row_id)` is the one secondary
|
|
56
|
-
* scope index: `
|
|
56
|
+
* scope index: `writeRowsOn` and `deleteRow` replace a row's scope entries
|
|
57
57
|
* with a `(partition, tbl, row_id)` predicate, and `row_id` is the LAST
|
|
58
58
|
* column of the inverted PRIMARY KEY, so without this index that predicate
|
|
59
59
|
* scans the table's whole scope range on every row written (a bulk import
|