@syncular/server 0.57.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -22
- package/dist/admin.js +8 -4
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +16 -0
- package/dist/context.js +21 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +223 -1111
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +1 -1
- package/dist/handler.js +135 -103
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +17 -3
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +182 -105
- package/dist/pull.d.ts +28 -20
- package/dist/pull.js +85 -87
- package/dist/push.js +272 -147
- package/dist/realtime.d.ts +68 -28
- package/dist/realtime.js +297 -136
- package/dist/relational-rows.d.ts +27 -15
- package/dist/relational-rows.js +38 -22
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +1 -2
- package/dist/segment-download.js +18 -4
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +150 -47
- package/dist/sqlite-dialect.js +987 -88
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +293 -1117
- package/dist/storage.d.ts +32 -12
- package/package.json +2 -2
- package/src/admin.ts +16 -2
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +32 -1
- package/src/d1-storage.ts +326 -1692
- package/src/errors.ts +8 -0
- package/src/events.ts +1 -1
- package/src/handler.ts +178 -127
- package/src/local-authority.ts +17 -3
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +231 -138
- package/src/pull.ts +133 -147
- package/src/push.ts +356 -205
- package/src/realtime.ts +380 -179
- package/src/relational-rows.ts +54 -27
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +1 -2
- package/src/segment-download.ts +25 -4
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1607 -114
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +360 -1811
- package/src/storage.ts +33 -12
package/src/relational-rows.ts
CHANGED
|
@@ -518,7 +518,7 @@ export function toSqlValue(
|
|
|
518
518
|
column: RowColumn,
|
|
519
519
|
value: RowValue,
|
|
520
520
|
dialect: RelationalDialect,
|
|
521
|
-
):
|
|
521
|
+
): RowValue {
|
|
522
522
|
if (value === null) return null;
|
|
523
523
|
if (column.type === 'boolean' && dialect === 'sqlite') {
|
|
524
524
|
return value === true ? 1 : 0;
|
|
@@ -545,7 +545,7 @@ export function upsertValues(
|
|
|
545
545
|
partition: string,
|
|
546
546
|
row: StoredRow,
|
|
547
547
|
dialect: RelationalDialect,
|
|
548
|
-
):
|
|
548
|
+
): RowValue[] {
|
|
549
549
|
// A non-materialized table skips the decode entirely — the upsert is a
|
|
550
550
|
// six-column meta write (the old blob-store cost).
|
|
551
551
|
const projection = table.materialize
|
|
@@ -609,17 +609,25 @@ export function selectRowSql(
|
|
|
609
609
|
|
|
610
610
|
/**
|
|
611
611
|
* SELECT many stored rows of one table by primary key in one statement.
|
|
612
|
-
*
|
|
613
|
-
*
|
|
614
|
-
* rowIds]; absent ids contribute no
|
|
612
|
+
* The ids bind as one parameter (Postgres `text[]` with `= ANY($2)`, SQLite
|
|
613
|
+
* a JSON array through `json_each`), so the statement count does not grow
|
|
614
|
+
* with the id count. Params: [partition, rowIds]; absent ids contribute no
|
|
615
|
+
* row.
|
|
615
616
|
*/
|
|
616
|
-
export function selectRowsSql(
|
|
617
|
-
|
|
617
|
+
export function selectRowsSql(
|
|
618
|
+
table: CompiledTable,
|
|
619
|
+
dialect: RelationalDialect,
|
|
620
|
+
): string {
|
|
621
|
+
const [partition, rowIds] =
|
|
622
|
+
dialect === 'sqlite'
|
|
623
|
+
? ['?', 'IN (SELECT value FROM json_each(?))']
|
|
624
|
+
: ['$1', '=ANY($2::text[])'];
|
|
625
|
+
return `SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=${partition} AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}${rowIds}`;
|
|
618
626
|
}
|
|
619
627
|
|
|
620
628
|
export interface IndexRowPageStatement {
|
|
621
629
|
readonly sql: string;
|
|
622
|
-
readonly params: readonly
|
|
630
|
+
readonly params: readonly RowValue[];
|
|
623
631
|
}
|
|
624
632
|
|
|
625
633
|
/**
|
|
@@ -636,7 +644,7 @@ export function indexRowPageStatement(
|
|
|
636
644
|
limit: number,
|
|
637
645
|
dialect: RelationalDialect,
|
|
638
646
|
): IndexRowPageStatement {
|
|
639
|
-
const params:
|
|
647
|
+
const params: RowValue[] = [partition];
|
|
640
648
|
const placeholder = (): string =>
|
|
641
649
|
dialect === 'sqlite' ? '?' : `$${params.length}`;
|
|
642
650
|
const predicates = index.columns.map((columnName, valueIndex) => {
|
|
@@ -678,8 +686,11 @@ export function indexRowPageStatement(
|
|
|
678
686
|
* Bind order:
|
|
679
687
|
* - postgres: [partition, tbl, var, ...values, afterRowId, limit]
|
|
680
688
|
* (the join reuses `$1` for the partition);
|
|
681
|
-
* - sqlite: [partition, tbl, var,
|
|
682
|
-
* partition] (positional
|
|
689
|
+
* - sqlite: [partition, tbl, var, values as JSON array text, afterRowId,
|
|
690
|
+
* limit, partition] (positional `?`; the partition binds again for the
|
|
691
|
+
* join). The values expand through `json_each`, so the statement text and
|
|
692
|
+
* parameter count do not depend on `valueCount` and D1's 100-parameter
|
|
693
|
+
* cap is unreachable through scope values.
|
|
683
694
|
*/
|
|
684
695
|
export function scanRowPageSql(
|
|
685
696
|
table: CompiledTable,
|
|
@@ -687,14 +698,16 @@ export function scanRowPageSql(
|
|
|
687
698
|
dialect: RelationalDialect,
|
|
688
699
|
): string {
|
|
689
700
|
const p = (n: number) => (dialect === 'sqlite' ? '?' : `$${n}`);
|
|
690
|
-
const values
|
|
691
|
-
|
|
701
|
+
const values =
|
|
702
|
+
dialect === 'sqlite'
|
|
703
|
+
? 'SELECT value FROM json_each(?)'
|
|
704
|
+
: Array.from({ length: valueCount }, (_, i) => p(4 + i)).join(',');
|
|
692
705
|
const after = p(4 + valueCount);
|
|
693
706
|
const limit = p(5 + valueCount);
|
|
694
707
|
const joinPartition = dialect === 'sqlite' ? '?' : '$1';
|
|
695
708
|
return `SELECT c.row_id AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
|
|
696
709
|
FROM (SELECT DISTINCT row_id FROM sync_row_scopes
|
|
697
|
-
WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values
|
|
710
|
+
WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values})
|
|
698
711
|
AND row_id>${after}
|
|
699
712
|
ORDER BY row_id LIMIT ${limit}) c
|
|
700
713
|
LEFT JOIN ${quoteIdent(table.name)} r
|
|
@@ -726,17 +739,20 @@ export function scanRowPageSql(
|
|
|
726
739
|
* Bind order:
|
|
727
740
|
* - postgres: [partition, tbl, var, ...values, afterSeq, throughSeq,
|
|
728
741
|
* limit] (the joins reuse `$1`/`$2`);
|
|
729
|
-
* - sqlite: [partition, tbl, var,
|
|
730
|
-
* limit, partition, partition, tbl] (positional
|
|
731
|
-
* again for the joins).
|
|
742
|
+
* - sqlite: [partition, tbl, var, values as JSON array text, afterSeq,
|
|
743
|
+
* throughSeq, limit, partition, partition, tbl] (positional `?`;
|
|
744
|
+
* partition/tbl bind again for the joins). The values expand through
|
|
745
|
+
* `json_each` like `scanRowPageSql`.
|
|
732
746
|
*/
|
|
733
747
|
export function commitWindowPageSql(
|
|
734
748
|
valueCount: number,
|
|
735
749
|
dialect: RelationalDialect,
|
|
736
750
|
): string {
|
|
737
751
|
const p = (n: number) => (dialect === 'sqlite' ? '?' : `$${n}`);
|
|
738
|
-
const values
|
|
739
|
-
|
|
752
|
+
const values =
|
|
753
|
+
dialect === 'sqlite'
|
|
754
|
+
? 'SELECT value FROM json_each(?)'
|
|
755
|
+
: Array.from({ length: valueCount }, (_, i) => p(4 + i)).join(',');
|
|
740
756
|
const after = p(4 + valueCount);
|
|
741
757
|
const through = p(5 + valueCount);
|
|
742
758
|
const limit = p(6 + valueCount);
|
|
@@ -745,7 +761,7 @@ export function commitWindowPageSql(
|
|
|
745
761
|
return `SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
746
762
|
ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
747
763
|
FROM (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
748
|
-
WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values
|
|
764
|
+
WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values})
|
|
749
765
|
AND commit_seq>${after} AND commit_seq<=${through}
|
|
750
766
|
ORDER BY commit_seq LIMIT ${limit}) c
|
|
751
767
|
LEFT JOIN sync_commits m
|
|
@@ -834,7 +850,7 @@ export function prefetchRowsBatchSql(
|
|
|
834
850
|
const projection = tables
|
|
835
851
|
.map((table) => {
|
|
836
852
|
const quoted = quoteIdent(table.name);
|
|
837
|
-
return `SELECT ids.tbl AS tbl, ids.id AS lookup_id, r.${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions, t.commit_seq AS tombstone_seq
|
|
853
|
+
return `SELECT ids.tbl AS tbl, ids.id AS lookup_id, r.${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions, t.commit_seq AS tombstone_seq, t.server_version AS tombstone_version, t.scopes AS tombstone_scopes
|
|
838
854
|
FROM ids
|
|
839
855
|
LEFT JOIN ${quoted} r
|
|
840
856
|
ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${params.partition} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=ids.id
|
|
@@ -877,13 +893,24 @@ export const LATEST_SCOPED_CHANGE_BATCH_SQL = `SELECT s.i AS sub,
|
|
|
877
893
|
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals jsonb)
|
|
878
894
|
ORDER BY s.i`;
|
|
879
895
|
|
|
880
|
-
/**
|
|
881
|
-
|
|
882
|
-
|
|
896
|
+
/**
|
|
897
|
+
* SELECT the rows that reference one blob with their stored scope maps, in
|
|
898
|
+
* one statement for any number of references: the by-blob index range
|
|
899
|
+
* (§5.9.4) supplies the candidates and a `CASE` on the candidate's table
|
|
900
|
+
* name looks each scope map up by primary key in that table's own relation.
|
|
901
|
+
* A reference whose table left the schema or whose row is gone has NULL
|
|
902
|
+
* `scopes`. Needs at least one table. Params: [partition, blobId].
|
|
903
|
+
*/
|
|
904
|
+
export function rowsReferencingBlobSql(
|
|
905
|
+
tables: Iterable<CompiledTable>,
|
|
883
906
|
dialect: RelationalDialect,
|
|
884
907
|
): string {
|
|
885
908
|
const p = dialect === 'sqlite' ? ['?', '?'] : ['$1', '$2'];
|
|
886
|
-
|
|
909
|
+
const branches = [...tables].map(
|
|
910
|
+
(table) =>
|
|
911
|
+
`WHEN '${table.name.replaceAll("'", "''")}' THEN (SELECT t.${quoteIdent(SYNC_SCOPES_COLUMN)} FROM ${quoteIdent(table.name)} t WHERE t.${quoteIdent(SYNC_PARTITION_COLUMN)}=r.partition AND t.${quoteIdent(SYNC_ROW_ID_COLUMN)}=r.row_id)`,
|
|
912
|
+
);
|
|
913
|
+
return `SELECT r.tbl AS tbl, r.row_id AS row_id, CASE r.tbl ${branches.join(' ')} END AS scopes FROM sync_blob_refs r WHERE r.partition=${p[0]} AND r.blob_id=${p[1]}`;
|
|
887
914
|
}
|
|
888
915
|
|
|
889
916
|
/** DELETE one stored row. Params: [partition, rowId]. */
|
|
@@ -1124,8 +1151,8 @@ export function rewriteValues(
|
|
|
1124
1151
|
rowId: string,
|
|
1125
1152
|
payload: Uint8Array,
|
|
1126
1153
|
dialect: RelationalDialect,
|
|
1127
|
-
):
|
|
1128
|
-
const out:
|
|
1154
|
+
): RowValue[] {
|
|
1155
|
+
const out: RowValue[] = [];
|
|
1129
1156
|
if (table.materialize) {
|
|
1130
1157
|
const values = decodeRow(table.columns, payload);
|
|
1131
1158
|
for (let i = 0; i < table.columns.length; i++) {
|
package/src/s3-blob-store.ts
CHANGED
|
@@ -97,19 +97,19 @@ export class S3BlobStore extends BucketBlobStore {
|
|
|
97
97
|
|
|
98
98
|
/**
|
|
99
99
|
* SigV4 presigned PUT for a blob object (§5.9.3 direct-to-storage upload).
|
|
100
|
-
* The
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
* handler after host authentication; TTL SHOULD be ≤ 15 min (default 900).
|
|
100
|
+
* The signed key embeds the `blobId`, and the signature binds
|
|
101
|
+
* `content-length` to the granted `byteLength` and `x-amz-checksum-sha256`
|
|
102
|
+
* to the `blobId` digest (the same SHA-256, re-encoded from lowercase hex to
|
|
103
|
+
* the base64 of its 32 raw bytes). The object store therefore rejects a PUT
|
|
104
|
+
* of any other size (signature mismatch) or any other bytes (`BadDigest`),
|
|
105
|
+
* so a grant can only place the content-addressed bytes at their key.
|
|
106
|
+
* Issued by the upload-grant handler after host authentication and the
|
|
107
|
+
* size-cap check; TTL SHOULD be ≤ 15 min (default 900).
|
|
109
108
|
*/
|
|
110
109
|
async presignBlobPut(
|
|
111
110
|
partition: string,
|
|
112
111
|
blobId: string,
|
|
112
|
+
byteLength: number,
|
|
113
113
|
options?: { readonly ttlSeconds?: number; readonly nowMs?: number },
|
|
114
114
|
): Promise<SegmentUrlIssue> {
|
|
115
115
|
if (!BLOB_ID_PATTERN.test(blobId)) {
|
|
@@ -117,6 +117,9 @@ export class S3BlobStore extends BucketBlobStore {
|
|
|
117
117
|
}
|
|
118
118
|
const ttlSeconds = options?.ttlSeconds ?? 900;
|
|
119
119
|
const nowMs = options?.nowMs ?? Date.now();
|
|
120
|
+
const digest = (blobId.slice('sha256:'.length).match(/../g) ?? []).map(
|
|
121
|
+
(hex) => Number.parseInt(hex, 16),
|
|
122
|
+
);
|
|
120
123
|
const url = await presignUrl({
|
|
121
124
|
method: 'PUT',
|
|
122
125
|
url: this.#bucket.urlFor(this.objectKeyFor(partition, blobId)),
|
|
@@ -124,6 +127,10 @@ export class S3BlobStore extends BucketBlobStore {
|
|
|
124
127
|
credentials: this.#credentials,
|
|
125
128
|
nowMs,
|
|
126
129
|
expiresSeconds: ttlSeconds,
|
|
130
|
+
headers: {
|
|
131
|
+
'content-length': String(byteLength),
|
|
132
|
+
'x-amz-checksum-sha256': btoa(String.fromCharCode(...digest)),
|
|
133
|
+
},
|
|
127
134
|
});
|
|
128
135
|
return {
|
|
129
136
|
url: url.toString(),
|
|
@@ -158,8 +165,8 @@ export function s3PresignedBlobUrls(
|
|
|
158
165
|
* `blobUploadUrls: s3PresignedBlobUploads(blobStore, { ttlSeconds: 900 })`.
|
|
159
166
|
* The upload-grant handler issues the presigned PUT only after host
|
|
160
167
|
* authentication + the size-cap check; the client PUTs bytes straight to the
|
|
161
|
-
* object store with no host auth
|
|
162
|
-
*
|
|
168
|
+
* object store with no host auth, and the store verifies the signed
|
|
169
|
+
* `content-length` and `x-amz-checksum-sha256` (§5.9.3).
|
|
163
170
|
*/
|
|
164
171
|
export function s3PresignedBlobUploads(
|
|
165
172
|
store: S3BlobStore,
|
|
@@ -169,7 +176,10 @@ export function s3PresignedBlobUploads(
|
|
|
169
176
|
...(options?.ttlSeconds !== undefined
|
|
170
177
|
? { ttlSeconds: options.ttlSeconds }
|
|
171
178
|
: {}),
|
|
172
|
-
presign: ({ partition, blobId, ttlSeconds, nowMs }) =>
|
|
173
|
-
store.presignBlobPut(partition, blobId,
|
|
179
|
+
presign: ({ partition, blobId, byteLength, ttlSeconds, nowMs }) =>
|
|
180
|
+
store.presignBlobPut(partition, blobId, byteLength, {
|
|
181
|
+
ttlSeconds,
|
|
182
|
+
nowMs,
|
|
183
|
+
}),
|
|
174
184
|
};
|
|
175
185
|
}
|
package/src/s3-object-bucket.ts
CHANGED
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
type SigV4Credentials,
|
|
22
22
|
sha256Hex,
|
|
23
23
|
signRequest,
|
|
24
|
+
uriEncode,
|
|
24
25
|
} from './sigv4';
|
|
25
26
|
|
|
26
27
|
export interface S3ObjectBucketConfig {
|
|
@@ -113,10 +114,23 @@ export class S3ObjectBucket implements ObjectBucket {
|
|
|
113
114
|
};
|
|
114
115
|
}
|
|
115
116
|
|
|
116
|
-
/**
|
|
117
|
+
/**
|
|
118
|
+
* The object URL the presign helpers sign (`{endpoint}/{bucket}/{key}`).
|
|
119
|
+
* Every `/`-separated key segment is URI-encoded the way SigV4
|
|
120
|
+
* canonicalizes an S3 path (RFC 3986 unreserved characters stay raw), so a
|
|
121
|
+
* partition id holding `#`, `?`, `%` or `:` stays inside its own key and the
|
|
122
|
+
* pathname is already the canonical path. A `.` or `..` segment has no URL
|
|
123
|
+
* form (the URL parser and S3 clients collapse it), so it is refused.
|
|
124
|
+
*/
|
|
117
125
|
urlFor(key: string): URL {
|
|
118
126
|
const endpoint = this.#config.endpoint.replace(/\/+$/, '');
|
|
119
|
-
|
|
127
|
+
const segments = key.split('/');
|
|
128
|
+
if (segments.some((segment) => segment === '.' || segment === '..')) {
|
|
129
|
+
throw new Error('S3ObjectBucket: object key has a dot segment');
|
|
130
|
+
}
|
|
131
|
+
return new URL(
|
|
132
|
+
`${endpoint}/${this.#config.bucket}/${segments.map((segment) => uriEncode(segment, true)).join('/')}`,
|
|
133
|
+
);
|
|
120
134
|
}
|
|
121
135
|
|
|
122
136
|
async #request(
|
|
@@ -183,9 +197,13 @@ export class S3ObjectBucket implements ObjectBucket {
|
|
|
183
197
|
const response = await this.#request('HEAD', key);
|
|
184
198
|
if (response === undefined) return undefined;
|
|
185
199
|
await response.arrayBuffer();
|
|
200
|
+
const lastModified = Date.parse(
|
|
201
|
+
response.headers.get('last-modified') ?? '',
|
|
202
|
+
);
|
|
186
203
|
return {
|
|
187
204
|
etag: response.headers.get('etag') ?? undefined,
|
|
188
205
|
size: sizeOf(response.headers),
|
|
206
|
+
lastModifiedMs: Number.isNaN(lastModified) ? undefined : lastModified,
|
|
189
207
|
metadata: metadataOf(response.headers),
|
|
190
208
|
};
|
|
191
209
|
}
|
package/src/schema.ts
CHANGED
|
@@ -161,6 +161,22 @@ export interface CompiledSchema {
|
|
|
161
161
|
|
|
162
162
|
const PATTERN_RE = /^([^{}]+):\{([^{}:]+)\}$/;
|
|
163
163
|
|
|
164
|
+
/** A scope column has a type other than string or integer (§3.1). */
|
|
165
|
+
export class ScopeColumnTypeError extends Error {
|
|
166
|
+
readonly code = 'schema.scope_column_type';
|
|
167
|
+
|
|
168
|
+
constructor(
|
|
169
|
+
readonly details: {
|
|
170
|
+
readonly table: string;
|
|
171
|
+
readonly column: string;
|
|
172
|
+
readonly type: string;
|
|
173
|
+
},
|
|
174
|
+
) {
|
|
175
|
+
super('a scope column must have type string or integer (§3.1)');
|
|
176
|
+
this.name = 'ScopeColumnTypeError';
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
164
180
|
/**
|
|
165
181
|
* Compile one `prefix:{variable}` scope pattern against the table's column
|
|
166
182
|
* list, resolving the scope column's positional index for row extraction.
|
|
@@ -186,6 +202,14 @@ function compilePattern(
|
|
|
186
202
|
`table ${table.name}: scope pattern ${JSON.stringify(pattern)} names unknown column ${JSON.stringify(column)}`,
|
|
187
203
|
);
|
|
188
204
|
}
|
|
205
|
+
const type = table.columns[index]?.type;
|
|
206
|
+
if (type !== 'string' && type !== 'integer') {
|
|
207
|
+
throw new ScopeColumnTypeError({
|
|
208
|
+
table: table.name,
|
|
209
|
+
column,
|
|
210
|
+
type: type ?? 'undefined',
|
|
211
|
+
});
|
|
212
|
+
}
|
|
189
213
|
return { variable, prefix, column, columnIndex: index };
|
|
190
214
|
}
|
|
191
215
|
|
|
@@ -409,6 +433,13 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
|
|
|
409
433
|
`table ${child.name}: reference column ${JSON.stringify(reference.column)} names unknown table ${JSON.stringify(reference.parentTable)}`,
|
|
410
434
|
);
|
|
411
435
|
}
|
|
436
|
+
if (!child.materialize) {
|
|
437
|
+
// Enforcement reaches children through `scanRowsByIndex`, which reads
|
|
438
|
+
// the materialized columns.
|
|
439
|
+
throw new Error(
|
|
440
|
+
'a declared reference requires a materialized child table (§6.11)',
|
|
441
|
+
);
|
|
442
|
+
}
|
|
412
443
|
const childColumn = child.columns[reference.columnIndex];
|
|
413
444
|
const parentKey = parent.columns[parent.primaryKeyIndex];
|
|
414
445
|
if (childColumn === undefined || parentKey === undefined) {
|
package/src/scopes.ts
CHANGED
|
@@ -22,22 +22,28 @@ export type EffectiveOutcome =
|
|
|
22
22
|
| { readonly status: 'revoked' };
|
|
23
23
|
|
|
24
24
|
/**
|
|
25
|
-
* Effective = requested ∩ allowed per §3.2 rules 4–
|
|
25
|
+
* Effective = requested ∩ allowed per §3.2 rules 4–6. Inputs are assumed
|
|
26
26
|
* key-validated (requested keys declared, no `'*'` requested values).
|
|
27
|
+
* `declared` is the table's declared variables: when a requested key passes
|
|
28
|
+
* through an allowed `'*'`, every declared variable the request omits and
|
|
29
|
+
* the actor holds a non-wildcard list for joins effective with that list
|
|
30
|
+
* (rule 4), so the wildcard never lifts the actor's other fences.
|
|
27
31
|
*/
|
|
28
32
|
export function computeEffective(
|
|
33
|
+
declared: ReadonlySet<string>,
|
|
29
34
|
requested: ScopeMap,
|
|
30
35
|
resolved: ResolvedScopes,
|
|
31
36
|
): EffectiveOutcome {
|
|
32
37
|
if (!resolved.ok) return { status: 'revoked' };
|
|
33
38
|
const effective: ScopeMap = {};
|
|
34
|
-
|
|
35
|
-
for (const key of
|
|
39
|
+
let wildcard = false;
|
|
40
|
+
for (const key of Object.keys(requested)) {
|
|
36
41
|
const requestedValues = requested[key] ?? [];
|
|
37
42
|
const allowedValues = resolved.allowed[key];
|
|
38
43
|
if (allowedValues === undefined) return { status: 'revoked' };
|
|
39
44
|
let values: string[];
|
|
40
45
|
if (allowedValues.includes('*')) {
|
|
46
|
+
wildcard = true;
|
|
41
47
|
values = [...requestedValues];
|
|
42
48
|
} else {
|
|
43
49
|
values = requestedValues.filter((v) => allowedValues.includes(v));
|
|
@@ -46,6 +52,20 @@ export function computeEffective(
|
|
|
46
52
|
effective[key] = values;
|
|
47
53
|
}
|
|
48
54
|
if (Object.keys(effective).length === 0) return { status: 'revoked' };
|
|
55
|
+
if (wildcard) {
|
|
56
|
+
for (const variable of declared) {
|
|
57
|
+
const allowedValues = resolved.allowed[variable];
|
|
58
|
+
if (
|
|
59
|
+
Object.hasOwn(requested, variable) ||
|
|
60
|
+
allowedValues === undefined ||
|
|
61
|
+
allowedValues.includes('*')
|
|
62
|
+
) {
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
if (allowedValues.length === 0) return { status: 'revoked' };
|
|
66
|
+
effective[variable] = [...allowedValues];
|
|
67
|
+
}
|
|
68
|
+
}
|
|
49
69
|
return { status: 'active', effective };
|
|
50
70
|
}
|
|
51
71
|
|
package/src/seed.ts
CHANGED
|
@@ -218,8 +218,7 @@ export async function seedMutations(
|
|
|
218
218
|
const acquisitionHeader = acquisition.frames[0];
|
|
219
219
|
if (
|
|
220
220
|
acquisition.msgKind !== 'response' ||
|
|
221
|
-
acquisitionHeader?.type !== 'RESP_HEADER'
|
|
222
|
-
acquisitionHeader.logEpoch === undefined
|
|
221
|
+
acquisitionHeader?.type !== 'RESP_HEADER'
|
|
223
222
|
) {
|
|
224
223
|
throw new SyncError(
|
|
225
224
|
'sync.invalid_request',
|
package/src/segment-download.ts
CHANGED
|
@@ -263,15 +263,36 @@ async function authorizePublication(
|
|
|
263
263
|
}
|
|
264
264
|
resolved = { ok: false };
|
|
265
265
|
}
|
|
266
|
-
|
|
267
|
-
|
|
266
|
+
// §3.2: the effective scopes, and so the digest, depend on the declared
|
|
267
|
+
// variables of the publication's table (the wildcard fence).
|
|
268
|
+
const digests = new Map<string, string | undefined>();
|
|
269
|
+
for (const publication of inContext) {
|
|
270
|
+
const key = `${publication.schemaVersion}\u0000${publication.table}`;
|
|
271
|
+
if (digests.has(key)) continue;
|
|
272
|
+
const table = schemaWindowOf(ctx.schema, ctx.schemaWindow)
|
|
273
|
+
.find((candidate) => candidate.version === publication.schemaVersion)
|
|
274
|
+
?.tables.get(publication.table);
|
|
275
|
+
const outcome = computeEffective(
|
|
276
|
+
table?.declaredVariables ?? new Set(),
|
|
277
|
+
requested,
|
|
278
|
+
resolved,
|
|
279
|
+
);
|
|
280
|
+
digests.set(
|
|
281
|
+
key,
|
|
282
|
+
outcome.status === 'active'
|
|
283
|
+
? await scopeDigest(outcome.effective)
|
|
284
|
+
: undefined,
|
|
285
|
+
);
|
|
286
|
+
}
|
|
287
|
+
if ([...digests.values()].every((digest) => digest === undefined)) {
|
|
268
288
|
throw syncError('sync.forbidden', 'segment scopes not held (§5.5)');
|
|
269
289
|
}
|
|
270
|
-
const digest = await scopeDigest(outcome.effective);
|
|
271
290
|
// The digest is the grant, but only inside the caller's partition and live
|
|
272
291
|
// epoch: identical bytes published elsewhere never authorize this caller.
|
|
273
292
|
const forDigest = inContext.filter(
|
|
274
|
-
(publication) =>
|
|
293
|
+
(publication) =>
|
|
294
|
+
digests.get(`${publication.schemaVersion}\u0000${publication.table}`) ===
|
|
295
|
+
publication.scopeDigest,
|
|
275
296
|
);
|
|
276
297
|
if (forDigest.length === 0) {
|
|
277
298
|
throw syncError('sync.forbidden', 'scope digest mismatch (§3.5, §5.5)');
|
package/src/signed-url.ts
CHANGED
|
@@ -146,8 +146,8 @@ export async function issueBlobUrl(
|
|
|
146
146
|
* twin of `BlobPresignConfig`: the provider signs a single PUT whose object
|
|
147
147
|
* key embeds the `blobId`. Issuance happens ONLY after host authentication in
|
|
148
148
|
* the upload-grant handler (uploading is host-auth-only, not scope-bearing —
|
|
149
|
-
* §5.9.3). The
|
|
150
|
-
*
|
|
149
|
+
* §5.9.3). The provider MUST bind the PUT to the granted `byteLength` and to
|
|
150
|
+
* the content address, so the object store rejects other bytes. Never a multipart or
|
|
151
151
|
* chunk protocol — single PUT only, this rung and the next (§5.9.3 non-goal).
|
|
152
152
|
* `S3BlobStore.presignBlobPut` (via `s3PresignedBlobUploads`) is the in-tree
|
|
153
153
|
* implementation.
|
|
@@ -286,8 +286,14 @@ export async function verifySegmentToken(
|
|
|
286
286
|
syncError('sync.forbidden', `segment token rejected: ${reason}`);
|
|
287
287
|
const dot = token.indexOf('.');
|
|
288
288
|
if (dot < 0) throw forbidden('malformed token');
|
|
289
|
-
|
|
290
|
-
|
|
289
|
+
let payload: Uint8Array;
|
|
290
|
+
let mac: Uint8Array;
|
|
291
|
+
try {
|
|
292
|
+
payload = fromBase64url(token.slice(0, dot));
|
|
293
|
+
mac = fromBase64url(token.slice(dot + 1));
|
|
294
|
+
} catch {
|
|
295
|
+
throw forbidden('malformed token');
|
|
296
|
+
}
|
|
291
297
|
const computed = await hmac(key, payload);
|
|
292
298
|
if (!constantTimeEqual(mac, computed)) throw forbidden('bad MAC');
|
|
293
299
|
let claims: SegmentTokenClaims;
|
package/src/sigv4.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Minimal AWS Signature Version 4 — exactly the subset the S3 segment
|
|
3
|
-
*
|
|
3
|
+
* and blob stores need (header-signed requests and presigned URLs), hand-rolled
|
|
4
4
|
* per the dependency-light doctrine: SigV4 is small and
|
|
5
5
|
* well-specified, so no SDK.
|
|
6
6
|
*
|
|
@@ -226,9 +226,15 @@ export interface PresignArgs {
|
|
|
226
226
|
readonly credentials: SigV4Credentials;
|
|
227
227
|
readonly nowMs: number;
|
|
228
228
|
readonly expiresSeconds: number;
|
|
229
|
+
/**
|
|
230
|
+
* Headers the request must carry with exactly these values, signed beside
|
|
231
|
+
* `host` (a presigned PUT binds `content-length` and
|
|
232
|
+
* `x-amz-checksum-sha256`, so the store rejects other bytes).
|
|
233
|
+
*/
|
|
234
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
229
235
|
}
|
|
230
236
|
|
|
231
|
-
/** Query-authenticated (presigned) URL
|
|
237
|
+
/** Query-authenticated (presigned) URL over `host` plus `headers`. */
|
|
232
238
|
export async function presignUrl(args: PresignArgs): Promise<URL> {
|
|
233
239
|
const service = args.service ?? 's3';
|
|
234
240
|
const method = args.method ?? 'GET';
|
|
@@ -242,7 +248,14 @@ export async function presignUrl(args: PresignArgs): Promise<URL> {
|
|
|
242
248
|
);
|
|
243
249
|
url.searchParams.set('X-Amz-Date', amzDate);
|
|
244
250
|
url.searchParams.set('X-Amz-Expires', String(args.expiresSeconds));
|
|
245
|
-
|
|
251
|
+
const headers = { ...args.headers, host: url.host };
|
|
252
|
+
url.searchParams.set(
|
|
253
|
+
'X-Amz-SignedHeaders',
|
|
254
|
+
Object.keys(headers)
|
|
255
|
+
.map((name) => name.toLowerCase())
|
|
256
|
+
.sort()
|
|
257
|
+
.join(';'),
|
|
258
|
+
);
|
|
246
259
|
if (args.credentials.sessionToken !== undefined) {
|
|
247
260
|
url.searchParams.set('X-Amz-Security-Token', args.credentials.sessionToken);
|
|
248
261
|
}
|
|
@@ -250,7 +263,7 @@ export async function presignUrl(args: PresignArgs): Promise<URL> {
|
|
|
250
263
|
method,
|
|
251
264
|
canonicalPath: url.pathname,
|
|
252
265
|
query: url.searchParams,
|
|
253
|
-
headers
|
|
266
|
+
headers,
|
|
254
267
|
payloadHash: UNSIGNED_PAYLOAD,
|
|
255
268
|
});
|
|
256
269
|
const signature = await sigV4Signature({
|
package/src/sqlite-blob-store.ts
CHANGED
|
@@ -32,33 +32,32 @@ export class SqliteBlobStore implements BlobStore {
|
|
|
32
32
|
nowMs: number,
|
|
33
33
|
mediaType?: string,
|
|
34
34
|
): Promise<BlobRecord> {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
35
|
+
// A re-upload refreshes the sweep grace clock and names the media type an
|
|
36
|
+
// earlier upload left out; it never replaces a recorded one.
|
|
37
|
+
const row = this.db
|
|
38
|
+
.query<
|
|
39
|
+
{ media_type: string | null; created_at_ms: number },
|
|
40
|
+
[string, string, number, string | null, number, Uint8Array]
|
|
41
|
+
>(
|
|
42
|
+
`INSERT INTO sync_blobs(
|
|
43
|
+
partition, blob_id, byte_length, media_type, created_at_ms, bytes
|
|
44
|
+
) VALUES (?,?,?,?,?,?)
|
|
45
|
+
ON CONFLICT(partition, blob_id) DO UPDATE SET
|
|
46
|
+
media_type = coalesce(media_type, excluded.media_type),
|
|
47
|
+
created_at_ms = max(created_at_ms, excluded.created_at_ms)
|
|
48
|
+
RETURNING media_type, created_at_ms`,
|
|
49
|
+
)
|
|
50
|
+
.get(partition, blobId, bytes.length, mediaType ?? null, nowMs, bytes);
|
|
51
|
+
if (row === null) {
|
|
52
|
+
throw new Error('SqliteBlobStore: blob upsert returned no row');
|
|
46
53
|
}
|
|
47
|
-
|
|
54
|
+
return {
|
|
48
55
|
blobId,
|
|
49
56
|
partition,
|
|
50
57
|
byteLength: bytes.length,
|
|
51
|
-
...(
|
|
52
|
-
createdAtMs:
|
|
58
|
+
...(row.media_type !== null ? { mediaType: row.media_type } : {}),
|
|
59
|
+
createdAtMs: row.created_at_ms,
|
|
53
60
|
};
|
|
54
|
-
this.db
|
|
55
|
-
.query(
|
|
56
|
-
`INSERT OR IGNORE INTO sync_blobs(
|
|
57
|
-
partition, blob_id, byte_length, media_type, created_at_ms, bytes
|
|
58
|
-
) VALUES (?,?,?,?,?,?)`,
|
|
59
|
-
)
|
|
60
|
-
.run(partition, blobId, bytes.length, mediaType ?? null, nowMs, bytes);
|
|
61
|
-
return record;
|
|
62
61
|
}
|
|
63
62
|
|
|
64
63
|
async delete(partition: string, blobId: string): Promise<void> {
|
|
@@ -110,7 +109,7 @@ export class SqliteBlobStore implements BlobStore {
|
|
|
110
109
|
async sweepOrphans(
|
|
111
110
|
partition: string,
|
|
112
111
|
olderThanMs: number,
|
|
113
|
-
|
|
112
|
+
isReferenced: (blobId: string) => Promise<boolean>,
|
|
114
113
|
): Promise<string[]> {
|
|
115
114
|
const candidates = this.db
|
|
116
115
|
.query<{ blob_id: string }, [string, number]>(
|
|
@@ -119,12 +118,14 @@ export class SqliteBlobStore implements BlobStore {
|
|
|
119
118
|
)
|
|
120
119
|
.all(partition, olderThanMs);
|
|
121
120
|
const swept: string[] = [];
|
|
121
|
+
// The age condition repeats in the DELETE: a re-upload during the
|
|
122
|
+
// reference lookup refreshed the grace clock and keeps the blob.
|
|
122
123
|
const del = this.db.query(
|
|
123
|
-
'DELETE FROM sync_blobs WHERE partition=? AND blob_id=?',
|
|
124
|
+
'DELETE FROM sync_blobs WHERE partition=? AND blob_id=? AND created_at_ms < ?',
|
|
124
125
|
);
|
|
125
126
|
for (const { blob_id } of candidates) {
|
|
126
|
-
if (
|
|
127
|
-
|
|
127
|
+
if (await isReferenced(blob_id)) continue;
|
|
128
|
+
if (del.run(partition, blob_id, olderThanMs).changes > 0) {
|
|
128
129
|
swept.push(blob_id);
|
|
129
130
|
}
|
|
130
131
|
}
|