@syncular/server 0.58.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -36
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +46 -40
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +19 -5
- package/dist/context.js +19 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +249 -1119
- package/dist/errors.js +16 -0
- package/dist/events.d.ts +36 -18
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +503 -208
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +24 -12
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +420 -262
- package/dist/pull.d.ts +70 -47
- package/dist/pull.js +270 -301
- package/dist/push.d.ts +12 -3
- package/dist/push.js +361 -173
- package/dist/realtime.d.ts +85 -39
- package/dist/realtime.js +355 -219
- package/dist/relational-rows.d.ts +21 -66
- package/dist/relational-rows.js +28 -109
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +10 -16
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +22 -6
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +164 -84
- package/dist/sqlite-dialect.js +1145 -150
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +303 -1123
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +185 -63
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +99 -57
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +33 -6
- package/src/d1-storage.ts +350 -1700
- package/src/errors.ts +16 -0
- package/src/events.ts +37 -18
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +654 -261
- package/src/local-authority.ts +27 -14
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/operations.ts +44 -42
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +557 -354
- package/src/pull.ts +406 -440
- package/src/push.ts +466 -220
- package/src/realtime.ts +447 -290
- package/src/relational-rows.ts +43 -121
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +13 -20
- package/src/segment-download.ts +33 -7
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1851 -199
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +390 -1821
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +215 -67
package/src/sigv4.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Minimal AWS Signature Version 4 — exactly the subset the S3 segment
|
|
3
|
-
*
|
|
3
|
+
* and blob stores need (header-signed requests and presigned URLs), hand-rolled
|
|
4
4
|
* per the dependency-light doctrine: SigV4 is small and
|
|
5
5
|
* well-specified, so no SDK.
|
|
6
6
|
*
|
|
@@ -226,9 +226,15 @@ export interface PresignArgs {
|
|
|
226
226
|
readonly credentials: SigV4Credentials;
|
|
227
227
|
readonly nowMs: number;
|
|
228
228
|
readonly expiresSeconds: number;
|
|
229
|
+
/**
|
|
230
|
+
* Headers the request must carry with exactly these values, signed beside
|
|
231
|
+
* `host` (a presigned PUT binds `content-length` and
|
|
232
|
+
* `x-amz-checksum-sha256`, so the store rejects other bytes).
|
|
233
|
+
*/
|
|
234
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
229
235
|
}
|
|
230
236
|
|
|
231
|
-
/** Query-authenticated (presigned) URL
|
|
237
|
+
/** Query-authenticated (presigned) URL over `host` plus `headers`. */
|
|
232
238
|
export async function presignUrl(args: PresignArgs): Promise<URL> {
|
|
233
239
|
const service = args.service ?? 's3';
|
|
234
240
|
const method = args.method ?? 'GET';
|
|
@@ -242,7 +248,14 @@ export async function presignUrl(args: PresignArgs): Promise<URL> {
|
|
|
242
248
|
);
|
|
243
249
|
url.searchParams.set('X-Amz-Date', amzDate);
|
|
244
250
|
url.searchParams.set('X-Amz-Expires', String(args.expiresSeconds));
|
|
245
|
-
|
|
251
|
+
const headers = { ...args.headers, host: url.host };
|
|
252
|
+
url.searchParams.set(
|
|
253
|
+
'X-Amz-SignedHeaders',
|
|
254
|
+
Object.keys(headers)
|
|
255
|
+
.map((name) => name.toLowerCase())
|
|
256
|
+
.sort()
|
|
257
|
+
.join(';'),
|
|
258
|
+
);
|
|
246
259
|
if (args.credentials.sessionToken !== undefined) {
|
|
247
260
|
url.searchParams.set('X-Amz-Security-Token', args.credentials.sessionToken);
|
|
248
261
|
}
|
|
@@ -250,7 +263,7 @@ export async function presignUrl(args: PresignArgs): Promise<URL> {
|
|
|
250
263
|
method,
|
|
251
264
|
canonicalPath: url.pathname,
|
|
252
265
|
query: url.searchParams,
|
|
253
|
-
headers
|
|
266
|
+
headers,
|
|
254
267
|
payloadHash: UNSIGNED_PAYLOAD,
|
|
255
268
|
});
|
|
256
269
|
const signature = await sigV4Signature({
|
package/src/sqlite-blob-store.ts
CHANGED
|
@@ -32,33 +32,32 @@ export class SqliteBlobStore implements BlobStore {
|
|
|
32
32
|
nowMs: number,
|
|
33
33
|
mediaType?: string,
|
|
34
34
|
): Promise<BlobRecord> {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
35
|
+
// A re-upload refreshes the sweep grace clock and names the media type an
|
|
36
|
+
// earlier upload left out; it never replaces a recorded one.
|
|
37
|
+
const row = this.db
|
|
38
|
+
.query<
|
|
39
|
+
{ media_type: string | null; created_at_ms: number },
|
|
40
|
+
[string, string, number, string | null, number, Uint8Array]
|
|
41
|
+
>(
|
|
42
|
+
`INSERT INTO sync_blobs(
|
|
43
|
+
partition, blob_id, byte_length, media_type, created_at_ms, bytes
|
|
44
|
+
) VALUES (?,?,?,?,?,?)
|
|
45
|
+
ON CONFLICT(partition, blob_id) DO UPDATE SET
|
|
46
|
+
media_type = coalesce(media_type, excluded.media_type),
|
|
47
|
+
created_at_ms = max(created_at_ms, excluded.created_at_ms)
|
|
48
|
+
RETURNING media_type, created_at_ms`,
|
|
49
|
+
)
|
|
50
|
+
.get(partition, blobId, bytes.length, mediaType ?? null, nowMs, bytes);
|
|
51
|
+
if (row === null) {
|
|
52
|
+
throw new Error('SqliteBlobStore: blob upsert returned no row');
|
|
46
53
|
}
|
|
47
|
-
|
|
54
|
+
return {
|
|
48
55
|
blobId,
|
|
49
56
|
partition,
|
|
50
57
|
byteLength: bytes.length,
|
|
51
|
-
...(
|
|
52
|
-
createdAtMs:
|
|
58
|
+
...(row.media_type !== null ? { mediaType: row.media_type } : {}),
|
|
59
|
+
createdAtMs: row.created_at_ms,
|
|
53
60
|
};
|
|
54
|
-
this.db
|
|
55
|
-
.query(
|
|
56
|
-
`INSERT OR IGNORE INTO sync_blobs(
|
|
57
|
-
partition, blob_id, byte_length, media_type, created_at_ms, bytes
|
|
58
|
-
) VALUES (?,?,?,?,?,?)`,
|
|
59
|
-
)
|
|
60
|
-
.run(partition, blobId, bytes.length, mediaType ?? null, nowMs, bytes);
|
|
61
|
-
return record;
|
|
62
61
|
}
|
|
63
62
|
|
|
64
63
|
async delete(partition: string, blobId: string): Promise<void> {
|
|
@@ -110,7 +109,7 @@ export class SqliteBlobStore implements BlobStore {
|
|
|
110
109
|
async sweepOrphans(
|
|
111
110
|
partition: string,
|
|
112
111
|
olderThanMs: number,
|
|
113
|
-
|
|
112
|
+
isReferenced: (blobId: string) => Promise<boolean>,
|
|
114
113
|
): Promise<string[]> {
|
|
115
114
|
const candidates = this.db
|
|
116
115
|
.query<{ blob_id: string }, [string, number]>(
|
|
@@ -119,12 +118,14 @@ export class SqliteBlobStore implements BlobStore {
|
|
|
119
118
|
)
|
|
120
119
|
.all(partition, olderThanMs);
|
|
121
120
|
const swept: string[] = [];
|
|
121
|
+
// The age condition repeats in the DELETE: a re-upload during the
|
|
122
|
+
// reference lookup refreshed the grace clock and keeps the blob.
|
|
122
123
|
const del = this.db.query(
|
|
123
|
-
'DELETE FROM sync_blobs WHERE partition=? AND blob_id=?',
|
|
124
|
+
'DELETE FROM sync_blobs WHERE partition=? AND blob_id=? AND created_at_ms < ?',
|
|
124
125
|
);
|
|
125
126
|
for (const { blob_id } of candidates) {
|
|
126
|
-
if (
|
|
127
|
-
|
|
127
|
+
if (await isReferenced(blob_id)) continue;
|
|
128
|
+
if (del.run(partition, blob_id, olderThanMs).changes > 0) {
|
|
128
129
|
swept.push(blob_id);
|
|
129
130
|
}
|
|
130
131
|
}
|