@syncular/server 0.0.1-97 → 0.0.1-98
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pull.d.ts.map +1 -1
- package/dist/pull.js +74 -32
- package/dist/pull.js.map +1 -1
- package/dist/schema.d.ts +1 -1
- package/dist/schema.d.ts.map +1 -1
- package/dist/snapshot-chunks/adapters/s3.d.ts +4 -4
- package/dist/snapshot-chunks/db-metadata.d.ts +2 -2
- package/dist/snapshot-chunks/db-metadata.d.ts.map +1 -1
- package/dist/snapshot-chunks/db-metadata.js +63 -33
- package/dist/snapshot-chunks/db-metadata.js.map +1 -1
- package/dist/snapshot-chunks/types.d.ts +6 -6
- package/dist/snapshot-chunks/types.d.ts.map +1 -1
- package/dist/snapshot-chunks.d.ts +7 -7
- package/dist/snapshot-chunks.d.ts.map +1 -1
- package/dist/snapshot-chunks.js +5 -4
- package/dist/snapshot-chunks.js.map +1 -1
- package/package.json +1 -1
- package/src/pull.ts +103 -32
- package/src/schema.ts +1 -1
- package/src/snapshot-chunks/db-metadata.test.ts +2 -2
- package/src/snapshot-chunks/db-metadata.ts +86 -40
- package/src/snapshot-chunks/types.ts +10 -6
- package/src/snapshot-chunks.ts +17 -11
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Used for efficiently serving large bootstrap snapshots (e.g. catalogs)
|
|
5
5
|
* without embedding huge JSON payloads into pull responses.
|
|
6
6
|
*/
|
|
7
|
-
import type
|
|
7
|
+
import { type SyncSnapshotChunkCompression, type SyncSnapshotChunkEncoding, type SyncSnapshotChunkRef } from '@syncular/core';
|
|
8
8
|
import { type Kysely } from 'kysely';
|
|
9
9
|
import type { SyncCoreDb } from './schema';
|
|
10
10
|
export interface SnapshotChunkPageKey {
|
|
@@ -14,8 +14,8 @@ export interface SnapshotChunkPageKey {
|
|
|
14
14
|
asOfCommitSeq: number;
|
|
15
15
|
rowCursor: string | null;
|
|
16
16
|
rowLimit: number;
|
|
17
|
-
encoding:
|
|
18
|
-
compression:
|
|
17
|
+
encoding: SyncSnapshotChunkEncoding;
|
|
18
|
+
compression: SyncSnapshotChunkCompression;
|
|
19
19
|
}
|
|
20
20
|
export interface SnapshotChunkRow {
|
|
21
21
|
chunkId: string;
|
|
@@ -25,8 +25,8 @@ export interface SnapshotChunkRow {
|
|
|
25
25
|
asOfCommitSeq: number;
|
|
26
26
|
rowCursor: string;
|
|
27
27
|
rowLimit: number;
|
|
28
|
-
encoding:
|
|
29
|
-
compression:
|
|
28
|
+
encoding: SyncSnapshotChunkEncoding;
|
|
29
|
+
compression: SyncSnapshotChunkCompression;
|
|
30
30
|
sha256: string;
|
|
31
31
|
byteLength: number;
|
|
32
32
|
body: Uint8Array | ReadableStream<Uint8Array>;
|
|
@@ -43,8 +43,8 @@ export declare function insertSnapshotChunk<DB extends SyncCoreDb>(db: Kysely<DB
|
|
|
43
43
|
asOfCommitSeq: number;
|
|
44
44
|
rowCursor: string | null;
|
|
45
45
|
rowLimit: number;
|
|
46
|
-
encoding:
|
|
47
|
-
compression:
|
|
46
|
+
encoding: SyncSnapshotChunkEncoding;
|
|
47
|
+
compression: SyncSnapshotChunkCompression;
|
|
48
48
|
sha256: string;
|
|
49
49
|
body: Uint8Array;
|
|
50
50
|
expiresAt: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot-chunks.d.ts","sourceRoot":"","sources":["../src/snapshot-chunks.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"snapshot-chunks.d.ts","sourceRoot":"","sources":["../src/snapshot-chunks.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAGL,KAAK,4BAA4B,EACjC,KAAK,yBAAyB,EAC9B,KAAK,oBAAoB,EAC1B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,KAAK,MAAM,EAAO,MAAM,QAAQ,CAAC;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAE3C,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,yBAAyB,CAAC;IACpC,WAAW,EAAE,4BAA4B,CAAC;CAC3C;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,yBAAyB,CAAC;IACpC,WAAW,EAAE,4BAA4B,CAAC;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB;AAqBD,wBAAsB,6BAA6B,CAAC,EAAE,SAAS,UAAU,EACvE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,IAAI,EAAE,oBAAoB,GAAG;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/C,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CA+CtC;AAED,wBAAsB,mBAAmB,CAAC,EAAE,SAAS,UAAU,EAC7D,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,IAAI,EAAE;IACJ,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,yBAAyB,CAAC;IACpC,WAAW,EAAE,4BAA4B,CAAC;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,UAAU,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB,GACA,OAAO,CAAC,oBAAoB,CAAC,CAyE/B;AAED,wBAAsB,iBAAiB,CAAC,EAAE,SAAS,UAAU,EAC3D,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE;IACR,wDAAwD;IACxD,YAAY,CAAC,EAAE;QACb,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;QACvD,eAAe,CAAC,CACd,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;KAC/C,CAAC;CACH,GACA,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAkGlC;AAED,wBAAsB,2BAA2B,CAAC,EAAE,SAAS,UAAU,EACrE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,MAAM,SAA2B,GAChC,OAAO,CAAC,MAAM,CAAC,CAOjB"}
|
package/dist/snapshot-chunks.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* Used for efficiently serving large bootstrap snapshots (e.g. catalogs)
|
|
5
5
|
* without embedding huge JSON payloads into pull responses.
|
|
6
6
|
*/
|
|
7
|
+
import { SYNC_SNAPSHOT_CHUNK_COMPRESSION, SYNC_SNAPSHOT_CHUNK_ENCODING, } from '@syncular/core';
|
|
7
8
|
import { sql } from 'kysely';
|
|
8
9
|
function coerceChunkRow(value) {
|
|
9
10
|
// pg returns Buffer (subclass of Uint8Array); sqlite returns Uint8Array
|
|
@@ -46,10 +47,10 @@ export async function readSnapshotChunkRefByPageKey(db, args) {
|
|
|
46
47
|
const row = rowResult.rows[0];
|
|
47
48
|
if (!row)
|
|
48
49
|
return null;
|
|
49
|
-
if (row.encoding !==
|
|
50
|
+
if (row.encoding !== SYNC_SNAPSHOT_CHUNK_ENCODING) {
|
|
50
51
|
throw new Error(`Unexpected snapshot chunk encoding: ${String(row.encoding)}`);
|
|
51
52
|
}
|
|
52
|
-
if (row.compression !==
|
|
53
|
+
if (row.compression !== SYNC_SNAPSHOT_CHUNK_COMPRESSION) {
|
|
53
54
|
throw new Error(`Unexpected snapshot chunk compression: ${String(row.compression)}`);
|
|
54
55
|
}
|
|
55
56
|
return {
|
|
@@ -153,10 +154,10 @@ export async function readSnapshotChunk(db, chunkId, options) {
|
|
|
153
154
|
const row = rowResult.rows[0];
|
|
154
155
|
if (!row)
|
|
155
156
|
return null;
|
|
156
|
-
if (row.encoding !==
|
|
157
|
+
if (row.encoding !== SYNC_SNAPSHOT_CHUNK_ENCODING) {
|
|
157
158
|
throw new Error(`Unexpected snapshot chunk encoding: ${String(row.encoding)}`);
|
|
158
159
|
}
|
|
159
|
-
if (row.compression !==
|
|
160
|
+
if (row.compression !== SYNC_SNAPSHOT_CHUNK_COMPRESSION) {
|
|
160
161
|
throw new Error(`Unexpected snapshot chunk compression: ${String(row.compression)}`);
|
|
161
162
|
}
|
|
162
163
|
// Read body from external storage if available, otherwise use inline body
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot-chunks.js","sourceRoot":"","sources":["../src/snapshot-chunks.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;
|
|
1
|
+
{"version":3,"file":"snapshot-chunks.js","sourceRoot":"","sources":["../src/snapshot-chunks.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,+BAA+B,EAC/B,4BAA4B,GAI7B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAe,GAAG,EAAE,MAAM,QAAQ,CAAC;AA8B1C,SAAS,cAAc,CAAC,KAAc,EAAc;IAClD,wEAAwE;IACxE,IAAI,KAAK,YAAY,UAAU;QAAE,OAAO,KAAK,CAAC;IAC9C,IAAI,OAAO,MAAM,KAAK,WAAW,IAAI,KAAK,YAAY,MAAM;QAAE,OAAO,KAAK,CAAC;IAC3E,IAAI,KAAK,YAAY,WAAW;QAAE,OAAO,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IAC/D,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,EAAE,CAAC;QACtE,OAAO,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,IAAI,KAAK,CACb,wCAAwC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAChF,CAAC;AAAA,CACH;AAED,SAAS,eAAe,CAAC,KAAc,EAAU;IAC/C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,KAAK,YAAY,IAAI;QAAE,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IACtD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AAAA,CACtB;AAED,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,EAAc,EACd,IAAgD,EACV;IACtC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACvD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IAE1C,MAAM,SAAS,GAAG,MAAM,GAAG,CAMzB;;WAEO,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC;;uBAErB,IAAI,CAAC,WAAW;wBACf,IAAI,CAAC,QAAQ;oBACjB,IAAI,CAAC,KAAK;+BACC,IAAI,CAAC,aAAa;yBACxB,YAAY;wBACb,IAAI,CAAC,QAAQ;uBACd,IAAI,CAAC,QAAQ;0BACV,IAAI,CAAC,WAAW;yBACjB,MAAM;;GAE5B,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACd,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE9B,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,IAAI,GAAG,CAAC,QAAQ,KAAK,4BAA4B,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CACb,uCAAuC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAC9D,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,CAAC,WAAW,KAAK,+BAA+B,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CACb,0CAA0C,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CACpE,CAAC;IACJ,CAAC;IAED,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,QAAQ;QAChB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC;QACxC,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,WAAW,EAAE,GAAG,CAAC,WAAW;KAC7B,CAAC;AAAA,CACH;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,EAAc,EACd,IAaC,EAC8B;IAC/B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IAE1C,yDAAyD;IACzD,MAAM,QAAQ,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC;IAEzC,MAAM,GAAG,CAAA;kBACO,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC;;;;;;;;;;;;;;;;;;QAkB3C,IAAI,CAAC,OAAO;QACZ,IAAI,CAAC,WAAW;QAChB,IAAI,CAAC,QAAQ;QACb,IAAI,CAAC,KAAK;QACV,IAAI,CAAC,aAAa;QAClB,YAAY;QACZ,IAAI,CAAC,QAAQ;QACb,IAAI,CAAC,QAAQ;QACb,IAAI,CAAC,WAAW;QAChB,IAAI,CAAC,MAAM;QACX,IAAI,CAAC,IAAI,CAAC,MAAM;QAChB,QAAQ;QACR,IAAI,CAAC,IAAI;QACT,GAAG;QACH,IAAI,CAAC,SAAS;;;;;;;;;;;;;qBAaD,IAAI,CAAC,SAAS;oBACf,QAAQ;GACzB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAEd,MAAM,GAAG,GAAG,MAAM,6BAA6B,CAAC,EAAE,EAAE;QAClD,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAC,CAAC;IAEH,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IAED,OAAO,GAAG,CAAC;AAAA,CACZ;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,EAAc,EACd,OAAe,EACf,OAQC,EACiC;IAClC,MAAM,SAAS,GAAG,MAAM,GAAG,CAezB;;;;;;;;;;;;;;;;WAgBO,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC;uBACrB,OAAO;;GAE3B,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACd,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE9B,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,IAAI,GAAG,CAAC,QAAQ,KAAK,4BAA4B,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CACb,uCAAuC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAC9D,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,CAAC,WAAW,KAAK,+BAA+B,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CACb,0CAA0C,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CACpE,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,IAAI,IAA6C,CAAC;IAClD,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QAC1B,IAAI,OAAO,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC;YACzC,MAAM,kBAAkB,GACtB,MAAM,OAAO,CAAC,YAAY,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YACtD,IAAI,kBAAkB,EAAE,CAAC;gBACvB,IAAI,GAAG,kBAAkB,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBACnE,IAAI,YAAY,EAAE,CAAC;oBACjB,IAAI,GAAG,YAAY,CAAC;gBACtB,CAAC;qBAAM,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;oBACpB,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAClC,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,EAAE,CAAC,CAAC;gBACtE,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACnE,IAAI,YAAY,EAAE,CAAC;gBACjB,IAAI,GAAG,YAAY,CAAC;YACtB,CAAC;iBAAM,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACpB,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,EAAE,CAAC,CAAC;YACtE,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,QAAQ;QACrB,WAAW,EAAE,GAAG,CAAC,YAAY;QAC7B,QAAQ,EAAE,GAAG,CAAC,SAAS;QACvB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,CAAC;QAChD,SAAS,EAAE,GAAG,CAAC,UAAU;QACzB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,CAAC;QACpC,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC;QACxC,IAAI;QACJ,SAAS,EAAE,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC;KAC3C,CAAC;AAAA,CACH;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,EAAc,EACd,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAChB;IACjB,MAAM,GAAG,GAAG,MAAM,GAAG,CAAA;kBACL,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC;0BACzB,MAAM;GAC7B,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAEd,OAAO,MAAM,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC;AAAA,CACzC"}
|
package/package.json
CHANGED
package/src/pull.ts
CHANGED
|
@@ -5,7 +5,11 @@ import {
|
|
|
5
5
|
captureSyncException,
|
|
6
6
|
countSyncMetric,
|
|
7
7
|
distributionSyncMetric,
|
|
8
|
+
encodeSnapshotRowFrames,
|
|
9
|
+
encodeSnapshotRows,
|
|
8
10
|
type ScopeValues,
|
|
11
|
+
SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
12
|
+
SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
9
13
|
type SyncBootstrapState,
|
|
10
14
|
type SyncChange,
|
|
11
15
|
type SyncCommit,
|
|
@@ -29,14 +33,81 @@ import { resolveEffectiveScopesForSubscriptions } from './subscriptions/resolve'
|
|
|
29
33
|
const gzipAsync = promisify(gzip);
|
|
30
34
|
const ASYNC_GZIP_MIN_BYTES = 64 * 1024;
|
|
31
35
|
|
|
32
|
-
|
|
33
|
-
if (
|
|
34
|
-
return new Uint8Array(
|
|
36
|
+
function concatByteChunks(chunks: readonly Uint8Array[]): Uint8Array {
|
|
37
|
+
if (chunks.length === 1) {
|
|
38
|
+
return chunks[0] ?? new Uint8Array();
|
|
35
39
|
}
|
|
36
|
-
|
|
40
|
+
|
|
41
|
+
let total = 0;
|
|
42
|
+
for (const chunk of chunks) {
|
|
43
|
+
total += chunk.length;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const merged = new Uint8Array(total);
|
|
47
|
+
let offset = 0;
|
|
48
|
+
for (const chunk of chunks) {
|
|
49
|
+
merged.set(chunk, offset);
|
|
50
|
+
offset += chunk.length;
|
|
51
|
+
}
|
|
52
|
+
return merged;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function bytesToReadableStream(bytes: Uint8Array): ReadableStream<Uint8Array> {
|
|
56
|
+
return new ReadableStream<Uint8Array>({
|
|
57
|
+
start(controller) {
|
|
58
|
+
controller.enqueue(bytes);
|
|
59
|
+
controller.close();
|
|
60
|
+
},
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function chunksToReadableStream(
|
|
65
|
+
chunks: readonly Uint8Array[]
|
|
66
|
+
): ReadableStream<Uint8Array> {
|
|
67
|
+
return new ReadableStream<Uint8Array>({
|
|
68
|
+
start(controller) {
|
|
69
|
+
for (const chunk of chunks) {
|
|
70
|
+
controller.enqueue(chunk);
|
|
71
|
+
}
|
|
72
|
+
controller.close();
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function compressSnapshotPayload(
|
|
78
|
+
payload: Uint8Array
|
|
79
|
+
): Promise<Uint8Array> {
|
|
80
|
+
if (payload.byteLength < ASYNC_GZIP_MIN_BYTES) {
|
|
81
|
+
return new Uint8Array(gzipSync(payload));
|
|
82
|
+
}
|
|
83
|
+
const compressed = await gzipAsync(payload);
|
|
37
84
|
return new Uint8Array(compressed);
|
|
38
85
|
}
|
|
39
86
|
|
|
87
|
+
async function compressSnapshotPayloadStream(
|
|
88
|
+
chunks: readonly Uint8Array[]
|
|
89
|
+
): Promise<{
|
|
90
|
+
stream: ReadableStream<Uint8Array>;
|
|
91
|
+
byteLength?: number;
|
|
92
|
+
}> {
|
|
93
|
+
if (typeof CompressionStream !== 'undefined') {
|
|
94
|
+
const source = chunksToReadableStream(chunks);
|
|
95
|
+
const gzipStream = new CompressionStream(
|
|
96
|
+
'gzip'
|
|
97
|
+
) as unknown as TransformStream<Uint8Array, Uint8Array>;
|
|
98
|
+
return {
|
|
99
|
+
stream: source.pipeThrough(gzipStream),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const payload = concatByteChunks(chunks);
|
|
104
|
+
const compressed = await compressSnapshotPayload(payload);
|
|
105
|
+
return {
|
|
106
|
+
stream: bytesToReadableStream(compressed),
|
|
107
|
+
byteLength: compressed.length,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
40
111
|
export interface PullResult {
|
|
41
112
|
response: SyncPullResponse;
|
|
42
113
|
/**
|
|
@@ -356,7 +427,7 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
356
427
|
pageCount: number;
|
|
357
428
|
ttlMs: number;
|
|
358
429
|
hash: ReturnType<typeof createHash>;
|
|
359
|
-
|
|
430
|
+
rowFrameParts: Uint8Array[];
|
|
360
431
|
}
|
|
361
432
|
|
|
362
433
|
const flushSnapshotBundle = async (
|
|
@@ -375,23 +446,24 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
375
446
|
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
376
447
|
rowCursor: bundle.startCursor,
|
|
377
448
|
rowLimit: bundleRowLimit,
|
|
378
|
-
encoding:
|
|
379
|
-
compression:
|
|
449
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
450
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
380
451
|
nowIso,
|
|
381
452
|
});
|
|
382
453
|
|
|
383
454
|
let chunkRef = cached;
|
|
384
455
|
if (!chunkRef) {
|
|
385
|
-
const ndjson = bundle.ndjsonParts.join('');
|
|
386
|
-
const compressedBody = await compressSnapshotNdjson(ndjson);
|
|
387
456
|
const sha256 = bundle.hash.digest('hex');
|
|
388
457
|
const expiresAt = new Date(
|
|
389
458
|
Date.now() + Math.max(1000, bundle.ttlMs)
|
|
390
459
|
).toISOString();
|
|
391
|
-
const byteLength = compressedBody.length;
|
|
392
460
|
|
|
393
461
|
if (args.chunkStorage) {
|
|
394
462
|
if (args.chunkStorage.storeChunkStream) {
|
|
463
|
+
const { stream: bodyStream, byteLength } =
|
|
464
|
+
await compressSnapshotPayloadStream(
|
|
465
|
+
bundle.rowFrameParts
|
|
466
|
+
);
|
|
395
467
|
chunkRef = await args.chunkStorage.storeChunkStream({
|
|
396
468
|
partitionId,
|
|
397
469
|
scopeKey: cacheKey,
|
|
@@ -399,19 +471,17 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
399
471
|
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
400
472
|
rowCursor: bundle.startCursor,
|
|
401
473
|
rowLimit: bundleRowLimit,
|
|
402
|
-
encoding:
|
|
403
|
-
compression:
|
|
474
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
475
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
404
476
|
sha256,
|
|
405
477
|
byteLength,
|
|
406
|
-
bodyStream
|
|
407
|
-
start(controller) {
|
|
408
|
-
controller.enqueue(compressedBody);
|
|
409
|
-
controller.close();
|
|
410
|
-
},
|
|
411
|
-
}),
|
|
478
|
+
bodyStream,
|
|
412
479
|
expiresAt,
|
|
413
480
|
});
|
|
414
481
|
} else {
|
|
482
|
+
const compressedBody = await compressSnapshotPayload(
|
|
483
|
+
concatByteChunks(bundle.rowFrameParts)
|
|
484
|
+
);
|
|
415
485
|
chunkRef = await args.chunkStorage.storeChunk({
|
|
416
486
|
partitionId,
|
|
417
487
|
scopeKey: cacheKey,
|
|
@@ -419,14 +489,17 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
419
489
|
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
420
490
|
rowCursor: bundle.startCursor,
|
|
421
491
|
rowLimit: bundleRowLimit,
|
|
422
|
-
encoding:
|
|
423
|
-
compression:
|
|
492
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
493
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
424
494
|
sha256,
|
|
425
495
|
body: compressedBody,
|
|
426
496
|
expiresAt,
|
|
427
497
|
});
|
|
428
498
|
}
|
|
429
499
|
} else {
|
|
500
|
+
const compressedBody = await compressSnapshotPayload(
|
|
501
|
+
concatByteChunks(bundle.rowFrameParts)
|
|
502
|
+
);
|
|
430
503
|
const chunkId = randomUUID();
|
|
431
504
|
chunkRef = await insertSnapshotChunk(trx, {
|
|
432
505
|
chunkId,
|
|
@@ -436,8 +509,8 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
436
509
|
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
437
510
|
rowCursor: bundle.startCursor,
|
|
438
511
|
rowLimit: bundleRowLimit,
|
|
439
|
-
encoding:
|
|
440
|
-
compression:
|
|
512
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
513
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
441
514
|
sha256,
|
|
442
515
|
body: compressedBody,
|
|
443
516
|
expiresAt,
|
|
@@ -479,6 +552,9 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
479
552
|
if (activeBundle) {
|
|
480
553
|
await flushSnapshotBundle(activeBundle);
|
|
481
554
|
}
|
|
555
|
+
const bundleHash = createHash('sha256');
|
|
556
|
+
const bundleHeader = encodeSnapshotRows([]);
|
|
557
|
+
bundleHash.update(bundleHeader);
|
|
482
558
|
activeBundle = {
|
|
483
559
|
table: nextTableName,
|
|
484
560
|
startCursor: nextState.rowCursor,
|
|
@@ -487,8 +563,8 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
487
563
|
pageCount: 0,
|
|
488
564
|
ttlMs:
|
|
489
565
|
tableHandler.snapshotChunkTtlMs ?? 24 * 60 * 60 * 1000,
|
|
490
|
-
hash:
|
|
491
|
-
|
|
566
|
+
hash: bundleHash,
|
|
567
|
+
rowFrameParts: [bundleHeader],
|
|
492
568
|
};
|
|
493
569
|
}
|
|
494
570
|
|
|
@@ -503,14 +579,9 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
503
579
|
sub.params
|
|
504
580
|
);
|
|
505
581
|
|
|
506
|
-
const
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
lines.push(s === undefined ? 'null' : s);
|
|
510
|
-
}
|
|
511
|
-
const ndjson = lines.length > 0 ? `${lines.join('\n')}\n` : '';
|
|
512
|
-
activeBundle.hash.update(ndjson);
|
|
513
|
-
activeBundle.ndjsonParts.push(ndjson);
|
|
582
|
+
const rowFrames = encodeSnapshotRowFrames(page.rows ?? []);
|
|
583
|
+
activeBundle.hash.update(rowFrames);
|
|
584
|
+
activeBundle.rowFrameParts.push(rowFrames);
|
|
514
585
|
activeBundle.pageCount += 1;
|
|
515
586
|
|
|
516
587
|
if (page.nextCursor != null) {
|
package/src/schema.ts
CHANGED
|
@@ -105,7 +105,7 @@ export interface SyncSnapshotChunksTable {
|
|
|
105
105
|
row_cursor: string;
|
|
106
106
|
/** Snapshot row limit used to produce this chunk */
|
|
107
107
|
row_limit: number;
|
|
108
|
-
/** Row encoding (e.g. '
|
|
108
|
+
/** Row encoding (e.g. 'json-row-frame-v1') */
|
|
109
109
|
encoding: string;
|
|
110
110
|
/** Compression algorithm (e.g. 'gzip') */
|
|
111
111
|
compression: string;
|
|
@@ -64,7 +64,7 @@ describe('createDbMetadataChunkStorage', () => {
|
|
|
64
64
|
asOfCommitSeq: 1,
|
|
65
65
|
rowCursor: null,
|
|
66
66
|
rowLimit: 100,
|
|
67
|
-
encoding: '
|
|
67
|
+
encoding: 'json-row-frame-v1',
|
|
68
68
|
compression: 'gzip',
|
|
69
69
|
sha256: 'chunk-sha',
|
|
70
70
|
expiresAt,
|
|
@@ -78,7 +78,7 @@ describe('createDbMetadataChunkStorage', () => {
|
|
|
78
78
|
asOfCommitSeq: 1,
|
|
79
79
|
rowCursor: null,
|
|
80
80
|
rowLimit: 100,
|
|
81
|
-
encoding: '
|
|
81
|
+
encoding: 'json-row-frame-v1',
|
|
82
82
|
compression: 'gzip',
|
|
83
83
|
sha256: 'chunk-sha',
|
|
84
84
|
expiresAt: new Date(Date.now() + 120_000).toISOString(),
|
|
@@ -6,7 +6,14 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import { createHash } from 'node:crypto';
|
|
9
|
-
import
|
|
9
|
+
import {
|
|
10
|
+
type BlobStorageAdapter,
|
|
11
|
+
SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
12
|
+
SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
13
|
+
type SyncSnapshotChunkCompression,
|
|
14
|
+
type SyncSnapshotChunkEncoding,
|
|
15
|
+
type SyncSnapshotChunkRef,
|
|
16
|
+
} from '@syncular/core';
|
|
10
17
|
import type { Kysely } from 'kysely';
|
|
11
18
|
import type { SyncCoreDb } from '../schema';
|
|
12
19
|
import type { SnapshotChunkMetadata, SnapshotChunkPageKey } from './types';
|
|
@@ -43,7 +50,7 @@ export function createDbMetadataChunkStorage(
|
|
|
43
50
|
'chunkId' | 'byteLength' | 'blobHash'
|
|
44
51
|
> & {
|
|
45
52
|
bodyStream: ReadableStream<Uint8Array>;
|
|
46
|
-
byteLength
|
|
53
|
+
byteLength?: number;
|
|
47
54
|
}
|
|
48
55
|
) => Promise<SyncSnapshotChunkRef>;
|
|
49
56
|
readChunk: (chunkId: string) => Promise<Uint8Array | null>;
|
|
@@ -57,9 +64,16 @@ export function createDbMetadataChunkStorage(
|
|
|
57
64
|
} {
|
|
58
65
|
const { db, blobAdapter, chunkIdPrefix = 'chunk_' } = options;
|
|
59
66
|
|
|
60
|
-
// Generate deterministic blob hash from
|
|
61
|
-
function computeBlobHash(
|
|
62
|
-
|
|
67
|
+
// Generate deterministic blob hash from chunk identity metadata.
|
|
68
|
+
function computeBlobHash(metadata: {
|
|
69
|
+
encoding: SyncSnapshotChunkEncoding;
|
|
70
|
+
compression: SyncSnapshotChunkCompression;
|
|
71
|
+
sha256: string;
|
|
72
|
+
}): string {
|
|
73
|
+
const digest = createHash('sha256')
|
|
74
|
+
.update(`${metadata.encoding}:${metadata.compression}:${metadata.sha256}`)
|
|
75
|
+
.digest('hex');
|
|
76
|
+
return `sha256:${digest}`;
|
|
63
77
|
}
|
|
64
78
|
|
|
65
79
|
function bytesToStream(bytes: Uint8Array): ReadableStream<Uint8Array> {
|
|
@@ -99,6 +113,24 @@ export function createDbMetadataChunkStorage(
|
|
|
99
113
|
}
|
|
100
114
|
}
|
|
101
115
|
|
|
116
|
+
async function streamByteLength(
|
|
117
|
+
stream: ReadableStream<Uint8Array>
|
|
118
|
+
): Promise<number> {
|
|
119
|
+
const reader = stream.getReader();
|
|
120
|
+
try {
|
|
121
|
+
let total = 0;
|
|
122
|
+
while (true) {
|
|
123
|
+
const { done, value } = await reader.read();
|
|
124
|
+
if (done) break;
|
|
125
|
+
if (!value) continue;
|
|
126
|
+
total += value.length;
|
|
127
|
+
}
|
|
128
|
+
return total;
|
|
129
|
+
} finally {
|
|
130
|
+
reader.releaseLock();
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
102
134
|
// Generate unique chunk ID
|
|
103
135
|
function generateChunkId(): string {
|
|
104
136
|
return `${chunkIdPrefix}${Date.now()}_${Math.random().toString(36).slice(2, 11)}`;
|
|
@@ -111,8 +143,8 @@ export function createDbMetadataChunkStorage(
|
|
|
111
143
|
asOfCommitSeq: number;
|
|
112
144
|
rowCursor: string | null;
|
|
113
145
|
rowLimit: number;
|
|
114
|
-
encoding:
|
|
115
|
-
compression:
|
|
146
|
+
encoding: SyncSnapshotChunkEncoding;
|
|
147
|
+
compression: SyncSnapshotChunkCompression;
|
|
116
148
|
nowIso?: string;
|
|
117
149
|
includeExpired?: boolean;
|
|
118
150
|
}): Promise<SyncSnapshotChunkRef | null> {
|
|
@@ -136,12 +168,12 @@ export function createDbMetadataChunkStorage(
|
|
|
136
168
|
|
|
137
169
|
if (!row) return null;
|
|
138
170
|
|
|
139
|
-
if (row.encoding !==
|
|
171
|
+
if (row.encoding !== SYNC_SNAPSHOT_CHUNK_ENCODING) {
|
|
140
172
|
throw new Error(
|
|
141
173
|
`Unexpected snapshot chunk encoding: ${String(row.encoding)}`
|
|
142
174
|
);
|
|
143
175
|
}
|
|
144
|
-
if (row.compression !==
|
|
176
|
+
if (row.compression !== SYNC_SNAPSHOT_CHUNK_COMPRESSION) {
|
|
145
177
|
throw new Error(
|
|
146
178
|
`Unexpected snapshot chunk compression: ${String(row.compression)}`
|
|
147
179
|
);
|
|
@@ -248,7 +280,7 @@ export function createDbMetadataChunkStorage(
|
|
|
248
280
|
}
|
|
249
281
|
): Promise<SyncSnapshotChunkRef> {
|
|
250
282
|
const { body, ...metaWithoutBody } = metadata;
|
|
251
|
-
const blobHash = computeBlobHash(
|
|
283
|
+
const blobHash = computeBlobHash(metaWithoutBody);
|
|
252
284
|
|
|
253
285
|
// Check if blob already exists (content-addressed dedup)
|
|
254
286
|
const blobExists = await blobAdapter.exists(blobHash);
|
|
@@ -296,51 +328,65 @@ export function createDbMetadataChunkStorage(
|
|
|
296
328
|
'chunkId' | 'byteLength' | 'blobHash'
|
|
297
329
|
> & {
|
|
298
330
|
bodyStream: ReadableStream<Uint8Array>;
|
|
299
|
-
byteLength
|
|
331
|
+
byteLength?: number;
|
|
300
332
|
}
|
|
301
333
|
): Promise<SyncSnapshotChunkRef> {
|
|
302
334
|
const { bodyStream, byteLength, ...metaWithoutBody } = metadata;
|
|
303
|
-
const
|
|
304
|
-
|
|
305
|
-
const hasher = createHash('sha256');
|
|
306
|
-
let observedByteLength = 0;
|
|
307
|
-
const hashReader = hashStream.getReader();
|
|
308
|
-
try {
|
|
309
|
-
while (true) {
|
|
310
|
-
const { done, value } = await hashReader.read();
|
|
311
|
-
if (done) break;
|
|
312
|
-
if (!value) continue;
|
|
313
|
-
hasher.update(value);
|
|
314
|
-
observedByteLength += value.length;
|
|
315
|
-
}
|
|
316
|
-
} finally {
|
|
317
|
-
hashReader.releaseLock();
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
if (observedByteLength !== byteLength) {
|
|
321
|
-
throw new Error(
|
|
322
|
-
`Snapshot chunk byte length mismatch: expected ${byteLength}, got ${observedByteLength}`
|
|
323
|
-
);
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
const blobHash = `sha256:${hasher.digest('hex')}`;
|
|
335
|
+
const blobHash = computeBlobHash(metaWithoutBody);
|
|
327
336
|
|
|
328
337
|
const blobExists = await blobAdapter.exists(blobHash);
|
|
338
|
+
let observedByteLength: number;
|
|
339
|
+
|
|
329
340
|
if (!blobExists) {
|
|
330
341
|
if (blobAdapter.putStream) {
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
342
|
+
const [uploadStream, countStream] = bodyStream.tee();
|
|
343
|
+
const uploadPromise =
|
|
344
|
+
typeof byteLength === 'number'
|
|
345
|
+
? blobAdapter.putStream(blobHash, uploadStream, {
|
|
346
|
+
byteLength,
|
|
347
|
+
contentLength: byteLength,
|
|
348
|
+
})
|
|
349
|
+
: blobAdapter.putStream(blobHash, uploadStream);
|
|
350
|
+
const countPromise = streamByteLength(countStream);
|
|
351
|
+
|
|
352
|
+
const [, countedByteLength] = await Promise.all([
|
|
353
|
+
uploadPromise,
|
|
354
|
+
countPromise,
|
|
355
|
+
]);
|
|
356
|
+
observedByteLength = countedByteLength;
|
|
334
357
|
} else if (blobAdapter.put) {
|
|
335
|
-
const body = await streamToBytes(
|
|
358
|
+
const body = await streamToBytes(bodyStream);
|
|
336
359
|
await blobAdapter.put(blobHash, body);
|
|
360
|
+
observedByteLength = body.length;
|
|
337
361
|
} else {
|
|
338
362
|
throw new Error(
|
|
339
363
|
`Blob adapter ${blobAdapter.name} does not support direct put() for snapshot chunks`
|
|
340
364
|
);
|
|
341
365
|
}
|
|
366
|
+
} else if (typeof byteLength === 'number') {
|
|
367
|
+
observedByteLength = byteLength;
|
|
368
|
+
await bodyStream.cancel();
|
|
369
|
+
} else if (blobAdapter.getMetadata) {
|
|
370
|
+
const metadata = await blobAdapter.getMetadata(blobHash);
|
|
371
|
+
if (!metadata) {
|
|
372
|
+
throw new Error(
|
|
373
|
+
`Blob metadata missing for existing chunk ${blobHash}`
|
|
374
|
+
);
|
|
375
|
+
}
|
|
376
|
+
observedByteLength = metadata.size;
|
|
377
|
+
await bodyStream.cancel();
|
|
342
378
|
} else {
|
|
343
|
-
await
|
|
379
|
+
observedByteLength = await streamByteLength(bodyStream);
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
if (
|
|
383
|
+
typeof byteLength === 'number' &&
|
|
384
|
+
Number.isFinite(byteLength) &&
|
|
385
|
+
observedByteLength !== byteLength
|
|
386
|
+
) {
|
|
387
|
+
throw new Error(
|
|
388
|
+
`Snapshot chunk byte length mismatch: expected ${byteLength}, got ${observedByteLength}`
|
|
389
|
+
);
|
|
344
390
|
}
|
|
345
391
|
|
|
346
392
|
await upsertChunkMetadata(metaWithoutBody, {
|
|
@@ -5,7 +5,11 @@
|
|
|
5
5
|
* Enables flexible storage backends (database, S3, R2, etc.)
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import type {
|
|
8
|
+
import type {
|
|
9
|
+
SyncSnapshotChunkCompression,
|
|
10
|
+
SyncSnapshotChunkEncoding,
|
|
11
|
+
SyncSnapshotChunkRef,
|
|
12
|
+
} from '@syncular/core';
|
|
9
13
|
|
|
10
14
|
/**
|
|
11
15
|
* Page key for identifying a specific chunk
|
|
@@ -17,8 +21,8 @@ export interface SnapshotChunkPageKey {
|
|
|
17
21
|
asOfCommitSeq: number;
|
|
18
22
|
rowCursor: string | null;
|
|
19
23
|
rowLimit: number;
|
|
20
|
-
encoding:
|
|
21
|
-
compression:
|
|
24
|
+
encoding: SyncSnapshotChunkEncoding;
|
|
25
|
+
compression: SyncSnapshotChunkCompression;
|
|
22
26
|
}
|
|
23
27
|
|
|
24
28
|
/**
|
|
@@ -32,8 +36,8 @@ export interface SnapshotChunkMetadata {
|
|
|
32
36
|
asOfCommitSeq: number;
|
|
33
37
|
rowCursor: string | null;
|
|
34
38
|
rowLimit: number;
|
|
35
|
-
encoding:
|
|
36
|
-
compression:
|
|
39
|
+
encoding: SyncSnapshotChunkEncoding;
|
|
40
|
+
compression: SyncSnapshotChunkCompression;
|
|
37
41
|
sha256: string;
|
|
38
42
|
byteLength: number;
|
|
39
43
|
blobHash: string; // Reference to blob storage
|
|
@@ -70,7 +74,7 @@ export interface SnapshotChunkStorage {
|
|
|
70
74
|
'chunkId' | 'byteLength' | 'blobHash'
|
|
71
75
|
> & {
|
|
72
76
|
bodyStream: ReadableStream<Uint8Array>;
|
|
73
|
-
byteLength
|
|
77
|
+
byteLength?: number;
|
|
74
78
|
}
|
|
75
79
|
): Promise<SyncSnapshotChunkRef>;
|
|
76
80
|
|