@syncular/server 0.58.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -36
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +46 -40
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +19 -5
- package/dist/context.js +19 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +249 -1119
- package/dist/errors.js +16 -0
- package/dist/events.d.ts +36 -18
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +503 -208
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +24 -12
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +420 -262
- package/dist/pull.d.ts +70 -47
- package/dist/pull.js +270 -301
- package/dist/push.d.ts +12 -3
- package/dist/push.js +361 -173
- package/dist/realtime.d.ts +85 -39
- package/dist/realtime.js +355 -219
- package/dist/relational-rows.d.ts +21 -66
- package/dist/relational-rows.js +28 -109
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +10 -16
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +22 -6
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +164 -84
- package/dist/sqlite-dialect.js +1145 -150
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +303 -1123
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +185 -63
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +99 -57
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +33 -6
- package/src/d1-storage.ts +350 -1700
- package/src/errors.ts +16 -0
- package/src/events.ts +37 -18
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +654 -261
- package/src/local-authority.ts +27 -14
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/operations.ts +44 -42
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +557 -354
- package/src/pull.ts +406 -440
- package/src/push.ts +466 -220
- package/src/realtime.ts +447 -290
- package/src/relational-rows.ts +43 -121
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +13 -20
- package/src/segment-download.ts +33 -7
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1851 -199
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +390 -1821
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +215 -67
package/src/relational-rows.ts
CHANGED
|
@@ -518,7 +518,7 @@ export function toSqlValue(
|
|
|
518
518
|
column: RowColumn,
|
|
519
519
|
value: RowValue,
|
|
520
520
|
dialect: RelationalDialect,
|
|
521
|
-
):
|
|
521
|
+
): RowValue {
|
|
522
522
|
if (value === null) return null;
|
|
523
523
|
if (column.type === 'boolean' && dialect === 'sqlite') {
|
|
524
524
|
return value === true ? 1 : 0;
|
|
@@ -545,7 +545,7 @@ export function upsertValues(
|
|
|
545
545
|
partition: string,
|
|
546
546
|
row: StoredRow,
|
|
547
547
|
dialect: RelationalDialect,
|
|
548
|
-
):
|
|
548
|
+
): RowValue[] {
|
|
549
549
|
// A non-materialized table skips the decode entirely — the upsert is a
|
|
550
550
|
// six-column meta write (the old blob-store cost).
|
|
551
551
|
const projection = table.materialize
|
|
@@ -609,17 +609,25 @@ export function selectRowSql(
|
|
|
609
609
|
|
|
610
610
|
/**
|
|
611
611
|
* SELECT many stored rows of one table by primary key in one statement.
|
|
612
|
-
*
|
|
613
|
-
*
|
|
614
|
-
* rowIds]; absent ids contribute no
|
|
612
|
+
* The ids bind as one parameter (Postgres `text[]` with `= ANY($2)`, SQLite
|
|
613
|
+
* a JSON array through `json_each`), so the statement count does not grow
|
|
614
|
+
* with the id count. Params: [partition, rowIds]; absent ids contribute no
|
|
615
|
+
* row.
|
|
615
616
|
*/
|
|
616
|
-
export function selectRowsSql(
|
|
617
|
-
|
|
617
|
+
export function selectRowsSql(
|
|
618
|
+
table: CompiledTable,
|
|
619
|
+
dialect: RelationalDialect,
|
|
620
|
+
): string {
|
|
621
|
+
const [partition, rowIds] =
|
|
622
|
+
dialect === 'sqlite'
|
|
623
|
+
? ['?', 'IN (SELECT value FROM json_each(?))']
|
|
624
|
+
: ['$1', '=ANY($2::text[])'];
|
|
625
|
+
return `SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=${partition} AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}${rowIds}`;
|
|
618
626
|
}
|
|
619
627
|
|
|
620
628
|
export interface IndexRowPageStatement {
|
|
621
629
|
readonly sql: string;
|
|
622
|
-
readonly params: readonly
|
|
630
|
+
readonly params: readonly RowValue[];
|
|
623
631
|
}
|
|
624
632
|
|
|
625
633
|
/**
|
|
@@ -636,7 +644,7 @@ export function indexRowPageStatement(
|
|
|
636
644
|
limit: number,
|
|
637
645
|
dialect: RelationalDialect,
|
|
638
646
|
): IndexRowPageStatement {
|
|
639
|
-
const params:
|
|
647
|
+
const params: RowValue[] = [partition];
|
|
640
648
|
const placeholder = (): string =>
|
|
641
649
|
dialect === 'sqlite' ? '?' : `$${params.length}`;
|
|
642
650
|
const predicates = index.columns.map((columnName, valueIndex) => {
|
|
@@ -678,8 +686,11 @@ export function indexRowPageStatement(
|
|
|
678
686
|
* Bind order:
|
|
679
687
|
* - postgres: [partition, tbl, var, ...values, afterRowId, limit]
|
|
680
688
|
* (the join reuses `$1` for the partition);
|
|
681
|
-
* - sqlite: [partition, tbl, var,
|
|
682
|
-
* partition] (positional
|
|
689
|
+
* - sqlite: [partition, tbl, var, values as JSON array text, afterRowId,
|
|
690
|
+
* limit, partition] (positional `?`; the partition binds again for the
|
|
691
|
+
* join). The values expand through `json_each`, so the statement text and
|
|
692
|
+
* parameter count do not depend on `valueCount` and D1's 100-parameter
|
|
693
|
+
* cap is unreachable through scope values.
|
|
683
694
|
*/
|
|
684
695
|
export function scanRowPageSql(
|
|
685
696
|
table: CompiledTable,
|
|
@@ -687,14 +698,16 @@ export function scanRowPageSql(
|
|
|
687
698
|
dialect: RelationalDialect,
|
|
688
699
|
): string {
|
|
689
700
|
const p = (n: number) => (dialect === 'sqlite' ? '?' : `$${n}`);
|
|
690
|
-
const values
|
|
691
|
-
|
|
701
|
+
const values =
|
|
702
|
+
dialect === 'sqlite'
|
|
703
|
+
? 'SELECT value FROM json_each(?)'
|
|
704
|
+
: Array.from({ length: valueCount }, (_, i) => p(4 + i)).join(',');
|
|
692
705
|
const after = p(4 + valueCount);
|
|
693
706
|
const limit = p(5 + valueCount);
|
|
694
707
|
const joinPartition = dialect === 'sqlite' ? '?' : '$1';
|
|
695
708
|
return `SELECT c.row_id AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
|
|
696
709
|
FROM (SELECT DISTINCT row_id FROM sync_row_scopes
|
|
697
|
-
WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values
|
|
710
|
+
WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values})
|
|
698
711
|
AND row_id>${after}
|
|
699
712
|
ORDER BY row_id LIMIT ${limit}) c
|
|
700
713
|
LEFT JOIN ${quoteIdent(table.name)} r
|
|
@@ -702,59 +715,6 @@ export function scanRowPageSql(
|
|
|
702
715
|
ORDER BY c.row_id`;
|
|
703
716
|
}
|
|
704
717
|
|
|
705
|
-
/**
|
|
706
|
-
* One-round-trip page read for `readCommitWindow` (sync_* tables only; it
|
|
707
|
-
* lives here beside `scanRowPageSql` because it is the same join-the-index
|
|
708
|
-
* posture, dialect-parameterized the same way): candidates from the inverted
|
|
709
|
-
* change-scope index (ordered + LIMITed at the covering `sync_change_scopes`
|
|
710
|
-
* PK — exactly the old candidate query, so the index-first invariant of
|
|
711
|
-
* `CommitWindowQuery` is unchanged) LEFT JOINed to the commit metadata and
|
|
712
|
-
* to the commit's changes for the table, so a whole window page arrives in
|
|
713
|
-
* ONE statement instead of 1 + 2×candidates round trips. This is the
|
|
714
|
-
* incremental-pull hot path — it runs on every sync round; before the join a
|
|
715
|
-
* 500-candidate catch-up window on Postgres paid ~1000 network round trips.
|
|
716
|
-
*
|
|
717
|
-
* LEFT JOIN (not INNER): a candidate whose commit vanished (scope-index
|
|
718
|
-
* entry without a commit/changes row) must still reach the caller — its
|
|
719
|
-
* `commit_seq` advances the window cursor — so it comes back with NULL
|
|
720
|
-
* meta/change columns rather than disappearing from the page.
|
|
721
|
-
*
|
|
722
|
-
* Rows arrive ordered (commit_seq, idx): oldest-first commits, each commit's
|
|
723
|
-
* change rows consecutive in `idx` order — the caller groups consecutive
|
|
724
|
-
* rows and verifies the full multi-variable scope match in JS.
|
|
725
|
-
*
|
|
726
|
-
* Bind order:
|
|
727
|
-
* - postgres: [partition, tbl, var, ...values, afterSeq, throughSeq,
|
|
728
|
-
* limit] (the joins reuse `$1`/`$2`);
|
|
729
|
-
* - sqlite: [partition, tbl, var, ...values, afterSeq, throughSeq,
|
|
730
|
-
* limit, partition, partition, tbl] (positional `?` — partition/tbl bind
|
|
731
|
-
* again for the joins).
|
|
732
|
-
*/
|
|
733
|
-
export function commitWindowPageSql(
|
|
734
|
-
valueCount: number,
|
|
735
|
-
dialect: RelationalDialect,
|
|
736
|
-
): string {
|
|
737
|
-
const p = (n: number) => (dialect === 'sqlite' ? '?' : `$${n}`);
|
|
738
|
-
const values: string[] = [];
|
|
739
|
-
for (let i = 0; i < valueCount; i++) values.push(p(4 + i));
|
|
740
|
-
const after = p(4 + valueCount);
|
|
741
|
-
const through = p(5 + valueCount);
|
|
742
|
-
const limit = p(6 + valueCount);
|
|
743
|
-
const joinPartition = dialect === 'sqlite' ? '?' : '$1';
|
|
744
|
-
const joinTbl = dialect === 'sqlite' ? '?' : '$2';
|
|
745
|
-
return `SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
746
|
-
ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
747
|
-
FROM (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
748
|
-
WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values.join(',')})
|
|
749
|
-
AND commit_seq>${after} AND commit_seq<=${through}
|
|
750
|
-
ORDER BY commit_seq LIMIT ${limit}) c
|
|
751
|
-
LEFT JOIN sync_commits m
|
|
752
|
-
ON m.partition=${joinPartition} AND m.commit_seq=c.commit_seq
|
|
753
|
-
LEFT JOIN sync_changes ch
|
|
754
|
-
ON ch.partition=${joinPartition} AND ch.commit_seq=c.commit_seq AND ch.tbl=${joinTbl}
|
|
755
|
-
ORDER BY c.commit_seq, ch.idx`;
|
|
756
|
-
}
|
|
757
|
-
|
|
758
718
|
/**
|
|
759
719
|
* PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
|
|
760
720
|
* page of every subscription scanning any table in a pull. `$2` is a JSON
|
|
@@ -793,29 +753,6 @@ export function scanRowBatchSql(tables: readonly CompiledTable[]): string {
|
|
|
793
753
|
ORDER BY sub, row_id`;
|
|
794
754
|
}
|
|
795
755
|
|
|
796
|
-
/**
|
|
797
|
-
* PostgreSQL set-based form of `commitWindowPageSql`: one statement answers
|
|
798
|
-
* the window page of every subscription in a pull, across all tables. `$2`
|
|
799
|
-
* is a JSON array of `{i, tbl, var, vals, after, through, lim}` records, each
|
|
800
|
-
* running the same index-first candidate subquery through a LATERAL join.
|
|
801
|
-
* Rows arrive ordered by (i, commit_seq, idx).
|
|
802
|
-
*
|
|
803
|
-
* Params: [partition, specs (JSON text)], the specs bound as text like
|
|
804
|
-
* `scanRowBatchSql`.
|
|
805
|
-
*/
|
|
806
|
-
export const COMMIT_WINDOW_BATCH_SQL = `SELECT s.i AS sub, c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
807
|
-
ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
808
|
-
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after bigint, through bigint, lim int)
|
|
809
|
-
CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
810
|
-
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)
|
|
811
|
-
AND commit_seq>s.after AND commit_seq<=s.through
|
|
812
|
-
ORDER BY commit_seq LIMIT s.lim) c
|
|
813
|
-
LEFT JOIN sync_commits m
|
|
814
|
-
ON m.partition=$1 AND m.commit_seq=c.commit_seq
|
|
815
|
-
LEFT JOIN sync_changes ch
|
|
816
|
-
ON ch.partition=$1 AND ch.commit_seq=c.commit_seq AND ch.tbl=s.tbl
|
|
817
|
-
ORDER BY s.i, c.commit_seq, ch.idx`;
|
|
818
|
-
|
|
819
756
|
/**
|
|
820
757
|
* PostgreSQL push read-ahead across every table a commit targets, in one
|
|
821
758
|
* statement: each requested row id with its stored row (NULL columns when
|
|
@@ -834,7 +771,7 @@ export function prefetchRowsBatchSql(
|
|
|
834
771
|
const projection = tables
|
|
835
772
|
.map((table) => {
|
|
836
773
|
const quoted = quoteIdent(table.name);
|
|
837
|
-
return `SELECT ids.tbl AS tbl, ids.id AS lookup_id, r.${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions, t.commit_seq AS tombstone_seq
|
|
774
|
+
return `SELECT ids.tbl AS tbl, ids.id AS lookup_id, r.${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions, t.commit_seq AS tombstone_seq, t.server_version AS tombstone_version, t.scopes AS tombstone_scopes
|
|
838
775
|
FROM ids
|
|
839
776
|
LEFT JOIN ${quoted} r
|
|
840
777
|
ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${params.partition} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=ids.id
|
|
@@ -852,38 +789,23 @@ export function prefetchRowsBatchSql(
|
|
|
852
789
|
}
|
|
853
790
|
|
|
854
791
|
/**
|
|
855
|
-
*
|
|
856
|
-
* one
|
|
857
|
-
*
|
|
858
|
-
*
|
|
859
|
-
*
|
|
860
|
-
*
|
|
861
|
-
* Params: [partition, tbl, var, values as JSON array text].
|
|
862
|
-
*/
|
|
863
|
-
export const LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL = `SELECT max((SELECT max(commit_seq) FROM sync_change_scopes
|
|
864
|
-
WHERE partition=? AND tbl=? AND var=? AND value=v.value)) AS seq
|
|
865
|
-
FROM json_each(?) AS v`;
|
|
866
|
-
|
|
867
|
-
/**
|
|
868
|
-
* PostgreSQL set-based form of `LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL` for the
|
|
869
|
-
* subscriptions of a pull, across all tables. `$2` is a JSON array of
|
|
870
|
-
* `{i, tbl, var, vals}` records. Params: [partition, specs (JSON text)].
|
|
792
|
+
* SELECT the rows that reference one blob with their stored scope maps, in
|
|
793
|
+
* one statement for any number of references: the by-blob index range
|
|
794
|
+
* (§5.9.4) supplies the candidates and a `CASE` on the candidate's table
|
|
795
|
+
* name looks each scope map up by primary key in that table's own relation.
|
|
796
|
+
* A reference whose table left the schema or whose row is gone has NULL
|
|
797
|
+
* `scopes`. Needs at least one table. Params: [partition, blobId].
|
|
871
798
|
*/
|
|
872
|
-
export
|
|
873
|
-
|
|
874
|
-
SELECT (SELECT max(commit_seq) FROM sync_change_scopes
|
|
875
|
-
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=v.value) AS seq
|
|
876
|
-
FROM jsonb_array_elements_text(s.vals) AS v(value)) AS latest) AS seq
|
|
877
|
-
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals jsonb)
|
|
878
|
-
ORDER BY s.i`;
|
|
879
|
-
|
|
880
|
-
/** SELECT a row's version + scope map (admin/blob authz). Params: [partition, rowId]. */
|
|
881
|
-
export function selectRowScopesSql(
|
|
882
|
-
table: CompiledTable,
|
|
799
|
+
export function rowsReferencingBlobSql(
|
|
800
|
+
tables: Iterable<CompiledTable>,
|
|
883
801
|
dialect: RelationalDialect,
|
|
884
802
|
): string {
|
|
885
803
|
const p = dialect === 'sqlite' ? ['?', '?'] : ['$1', '$2'];
|
|
886
|
-
|
|
804
|
+
const branches = [...tables].map(
|
|
805
|
+
(table) =>
|
|
806
|
+
`WHEN '${table.name.replaceAll("'", "''")}' THEN (SELECT t.${quoteIdent(SYNC_SCOPES_COLUMN)} FROM ${quoteIdent(table.name)} t WHERE t.${quoteIdent(SYNC_PARTITION_COLUMN)}=r.partition AND t.${quoteIdent(SYNC_ROW_ID_COLUMN)}=r.row_id)`,
|
|
807
|
+
);
|
|
808
|
+
return `SELECT r.tbl AS tbl, r.row_id AS row_id, CASE r.tbl ${branches.join(' ')} END AS scopes FROM sync_blob_refs r WHERE r.partition=${p[0]} AND r.blob_id=${p[1]}`;
|
|
887
809
|
}
|
|
888
810
|
|
|
889
811
|
/** DELETE one stored row. Params: [partition, rowId]. */
|
|
@@ -1124,8 +1046,8 @@ export function rewriteValues(
|
|
|
1124
1046
|
rowId: string,
|
|
1125
1047
|
payload: Uint8Array,
|
|
1126
1048
|
dialect: RelationalDialect,
|
|
1127
|
-
):
|
|
1128
|
-
const out:
|
|
1049
|
+
): RowValue[] {
|
|
1050
|
+
const out: RowValue[] = [];
|
|
1129
1051
|
if (table.materialize) {
|
|
1130
1052
|
const values = decodeRow(table.columns, payload);
|
|
1131
1053
|
for (let i = 0; i < table.columns.length; i++) {
|
package/src/s3-blob-store.ts
CHANGED
|
@@ -97,19 +97,19 @@ export class S3BlobStore extends BucketBlobStore {
|
|
|
97
97
|
|
|
98
98
|
/**
|
|
99
99
|
* SigV4 presigned PUT for a blob object (§5.9.3 direct-to-storage upload).
|
|
100
|
-
* The
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
* handler after host authentication; TTL SHOULD be ≤ 15 min (default 900).
|
|
100
|
+
* The signed key embeds the `blobId`, and the signature binds
|
|
101
|
+
* `content-length` to the granted `byteLength` and `x-amz-checksum-sha256`
|
|
102
|
+
* to the `blobId` digest (the same SHA-256, re-encoded from lowercase hex to
|
|
103
|
+
* the base64 of its 32 raw bytes). The object store therefore rejects a PUT
|
|
104
|
+
* of any other size (signature mismatch) or any other bytes (`BadDigest`),
|
|
105
|
+
* so a grant can only place the content-addressed bytes at their key.
|
|
106
|
+
* Issued by the upload-grant handler after host authentication and the
|
|
107
|
+
* size-cap check; TTL SHOULD be ≤ 15 min (default 900).
|
|
109
108
|
*/
|
|
110
109
|
async presignBlobPut(
|
|
111
110
|
partition: string,
|
|
112
111
|
blobId: string,
|
|
112
|
+
byteLength: number,
|
|
113
113
|
options?: { readonly ttlSeconds?: number; readonly nowMs?: number },
|
|
114
114
|
): Promise<SegmentUrlIssue> {
|
|
115
115
|
if (!BLOB_ID_PATTERN.test(blobId)) {
|
|
@@ -117,6 +117,9 @@ export class S3BlobStore extends BucketBlobStore {
|
|
|
117
117
|
}
|
|
118
118
|
const ttlSeconds = options?.ttlSeconds ?? 900;
|
|
119
119
|
const nowMs = options?.nowMs ?? Date.now();
|
|
120
|
+
const digest = (blobId.slice('sha256:'.length).match(/../g) ?? []).map(
|
|
121
|
+
(hex) => Number.parseInt(hex, 16),
|
|
122
|
+
);
|
|
120
123
|
const url = await presignUrl({
|
|
121
124
|
method: 'PUT',
|
|
122
125
|
url: this.#bucket.urlFor(this.objectKeyFor(partition, blobId)),
|
|
@@ -124,6 +127,10 @@ export class S3BlobStore extends BucketBlobStore {
|
|
|
124
127
|
credentials: this.#credentials,
|
|
125
128
|
nowMs,
|
|
126
129
|
expiresSeconds: ttlSeconds,
|
|
130
|
+
headers: {
|
|
131
|
+
'content-length': String(byteLength),
|
|
132
|
+
'x-amz-checksum-sha256': btoa(String.fromCharCode(...digest)),
|
|
133
|
+
},
|
|
127
134
|
});
|
|
128
135
|
return {
|
|
129
136
|
url: url.toString(),
|
|
@@ -158,8 +165,8 @@ export function s3PresignedBlobUrls(
|
|
|
158
165
|
* `blobUploadUrls: s3PresignedBlobUploads(blobStore, { ttlSeconds: 900 })`.
|
|
159
166
|
* The upload-grant handler issues the presigned PUT only after host
|
|
160
167
|
* authentication + the size-cap check; the client PUTs bytes straight to the
|
|
161
|
-
* object store with no host auth
|
|
162
|
-
*
|
|
168
|
+
* object store with no host auth, and the store verifies the signed
|
|
169
|
+
* `content-length` and `x-amz-checksum-sha256` (§5.9.3).
|
|
163
170
|
*/
|
|
164
171
|
export function s3PresignedBlobUploads(
|
|
165
172
|
store: S3BlobStore,
|
|
@@ -169,7 +176,10 @@ export function s3PresignedBlobUploads(
|
|
|
169
176
|
...(options?.ttlSeconds !== undefined
|
|
170
177
|
? { ttlSeconds: options.ttlSeconds }
|
|
171
178
|
: {}),
|
|
172
|
-
presign: ({ partition, blobId, ttlSeconds, nowMs }) =>
|
|
173
|
-
store.presignBlobPut(partition, blobId,
|
|
179
|
+
presign: ({ partition, blobId, byteLength, ttlSeconds, nowMs }) =>
|
|
180
|
+
store.presignBlobPut(partition, blobId, byteLength, {
|
|
181
|
+
ttlSeconds,
|
|
182
|
+
nowMs,
|
|
183
|
+
}),
|
|
174
184
|
};
|
|
175
185
|
}
|
package/src/s3-object-bucket.ts
CHANGED
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
type SigV4Credentials,
|
|
22
22
|
sha256Hex,
|
|
23
23
|
signRequest,
|
|
24
|
+
uriEncode,
|
|
24
25
|
} from './sigv4';
|
|
25
26
|
|
|
26
27
|
export interface S3ObjectBucketConfig {
|
|
@@ -113,10 +114,23 @@ export class S3ObjectBucket implements ObjectBucket {
|
|
|
113
114
|
};
|
|
114
115
|
}
|
|
115
116
|
|
|
116
|
-
/**
|
|
117
|
+
/**
|
|
118
|
+
* The object URL the presign helpers sign (`{endpoint}/{bucket}/{key}`).
|
|
119
|
+
* Every `/`-separated key segment is URI-encoded the way SigV4
|
|
120
|
+
* canonicalizes an S3 path (RFC 3986 unreserved characters stay raw), so a
|
|
121
|
+
* partition id holding `#`, `?`, `%` or `:` stays inside its own key and the
|
|
122
|
+
* pathname is already the canonical path. A `.` or `..` segment has no URL
|
|
123
|
+
* form (the URL parser and S3 clients collapse it), so it is refused.
|
|
124
|
+
*/
|
|
117
125
|
urlFor(key: string): URL {
|
|
118
126
|
const endpoint = this.#config.endpoint.replace(/\/+$/, '');
|
|
119
|
-
|
|
127
|
+
const segments = key.split('/');
|
|
128
|
+
if (segments.some((segment) => segment === '.' || segment === '..')) {
|
|
129
|
+
throw new Error('S3ObjectBucket: object key has a dot segment');
|
|
130
|
+
}
|
|
131
|
+
return new URL(
|
|
132
|
+
`${endpoint}/${this.#config.bucket}/${segments.map((segment) => uriEncode(segment, true)).join('/')}`,
|
|
133
|
+
);
|
|
120
134
|
}
|
|
121
135
|
|
|
122
136
|
async #request(
|
|
@@ -183,9 +197,13 @@ export class S3ObjectBucket implements ObjectBucket {
|
|
|
183
197
|
const response = await this.#request('HEAD', key);
|
|
184
198
|
if (response === undefined) return undefined;
|
|
185
199
|
await response.arrayBuffer();
|
|
200
|
+
const lastModified = Date.parse(
|
|
201
|
+
response.headers.get('last-modified') ?? '',
|
|
202
|
+
);
|
|
186
203
|
return {
|
|
187
204
|
etag: response.headers.get('etag') ?? undefined,
|
|
188
205
|
size: sizeOf(response.headers),
|
|
206
|
+
lastModifiedMs: Number.isNaN(lastModified) ? undefined : lastModified,
|
|
189
207
|
metadata: metadataOf(response.headers),
|
|
190
208
|
};
|
|
191
209
|
}
|
package/src/schema.ts
CHANGED
|
@@ -161,6 +161,22 @@ export interface CompiledSchema {
|
|
|
161
161
|
|
|
162
162
|
const PATTERN_RE = /^([^{}]+):\{([^{}:]+)\}$/;
|
|
163
163
|
|
|
164
|
+
/** A scope column has a type other than string or integer (§3.1). */
|
|
165
|
+
export class ScopeColumnTypeError extends Error {
|
|
166
|
+
readonly code = 'schema.scope_column_type';
|
|
167
|
+
|
|
168
|
+
constructor(
|
|
169
|
+
readonly details: {
|
|
170
|
+
readonly table: string;
|
|
171
|
+
readonly column: string;
|
|
172
|
+
readonly type: string;
|
|
173
|
+
},
|
|
174
|
+
) {
|
|
175
|
+
super('a scope column must have type string or integer (§3.1)');
|
|
176
|
+
this.name = 'ScopeColumnTypeError';
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
164
180
|
/**
|
|
165
181
|
* Compile one `prefix:{variable}` scope pattern against the table's column
|
|
166
182
|
* list, resolving the scope column's positional index for row extraction.
|
|
@@ -186,6 +202,14 @@ function compilePattern(
|
|
|
186
202
|
`table ${table.name}: scope pattern ${JSON.stringify(pattern)} names unknown column ${JSON.stringify(column)}`,
|
|
187
203
|
);
|
|
188
204
|
}
|
|
205
|
+
const type = table.columns[index]?.type;
|
|
206
|
+
if (type !== 'string' && type !== 'integer') {
|
|
207
|
+
throw new ScopeColumnTypeError({
|
|
208
|
+
table: table.name,
|
|
209
|
+
column,
|
|
210
|
+
type: type ?? 'undefined',
|
|
211
|
+
});
|
|
212
|
+
}
|
|
189
213
|
return { variable, prefix, column, columnIndex: index };
|
|
190
214
|
}
|
|
191
215
|
|
|
@@ -409,6 +433,13 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
|
|
|
409
433
|
`table ${child.name}: reference column ${JSON.stringify(reference.column)} names unknown table ${JSON.stringify(reference.parentTable)}`,
|
|
410
434
|
);
|
|
411
435
|
}
|
|
436
|
+
if (!child.materialize) {
|
|
437
|
+
// Enforcement reaches children through `scanRowsByIndex`, which reads
|
|
438
|
+
// the materialized columns.
|
|
439
|
+
throw new Error(
|
|
440
|
+
'a declared reference requires a materialized child table (§6.11)',
|
|
441
|
+
);
|
|
442
|
+
}
|
|
412
443
|
const childColumn = child.columns[reference.columnIndex];
|
|
413
444
|
const parentKey = parent.columns[parent.primaryKeyIndex];
|
|
414
445
|
if (childColumn === undefined || parentKey === undefined) {
|
package/src/scopes.ts
CHANGED
|
@@ -22,22 +22,28 @@ export type EffectiveOutcome =
|
|
|
22
22
|
| { readonly status: 'revoked' };
|
|
23
23
|
|
|
24
24
|
/**
|
|
25
|
-
* Effective = requested ∩ allowed per §3.2 rules 4–
|
|
25
|
+
* Effective = requested ∩ allowed per §3.2 rules 4–6. Inputs are assumed
|
|
26
26
|
* key-validated (requested keys declared, no `'*'` requested values).
|
|
27
|
+
* `declared` is the table's declared variables: when a requested key passes
|
|
28
|
+
* through an allowed `'*'`, every declared variable the request omits and
|
|
29
|
+
* the actor holds a non-wildcard list for joins effective with that list
|
|
30
|
+
* (rule 4), so the wildcard never lifts the actor's other fences.
|
|
27
31
|
*/
|
|
28
32
|
export function computeEffective(
|
|
33
|
+
declared: ReadonlySet<string>,
|
|
29
34
|
requested: ScopeMap,
|
|
30
35
|
resolved: ResolvedScopes,
|
|
31
36
|
): EffectiveOutcome {
|
|
32
37
|
if (!resolved.ok) return { status: 'revoked' };
|
|
33
38
|
const effective: ScopeMap = {};
|
|
34
|
-
|
|
35
|
-
for (const key of
|
|
39
|
+
let wildcard = false;
|
|
40
|
+
for (const key of Object.keys(requested)) {
|
|
36
41
|
const requestedValues = requested[key] ?? [];
|
|
37
42
|
const allowedValues = resolved.allowed[key];
|
|
38
43
|
if (allowedValues === undefined) return { status: 'revoked' };
|
|
39
44
|
let values: string[];
|
|
40
45
|
if (allowedValues.includes('*')) {
|
|
46
|
+
wildcard = true;
|
|
41
47
|
values = [...requestedValues];
|
|
42
48
|
} else {
|
|
43
49
|
values = requestedValues.filter((v) => allowedValues.includes(v));
|
|
@@ -46,6 +52,20 @@ export function computeEffective(
|
|
|
46
52
|
effective[key] = values;
|
|
47
53
|
}
|
|
48
54
|
if (Object.keys(effective).length === 0) return { status: 'revoked' };
|
|
55
|
+
if (wildcard) {
|
|
56
|
+
for (const variable of declared) {
|
|
57
|
+
const allowedValues = resolved.allowed[variable];
|
|
58
|
+
if (
|
|
59
|
+
Object.hasOwn(requested, variable) ||
|
|
60
|
+
allowedValues === undefined ||
|
|
61
|
+
allowedValues.includes('*')
|
|
62
|
+
) {
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
if (allowedValues.length === 0) return { status: 'revoked' };
|
|
66
|
+
effective[variable] = [...allowedValues];
|
|
67
|
+
}
|
|
68
|
+
}
|
|
49
69
|
return { status: 'active', effective };
|
|
50
70
|
}
|
|
51
71
|
|
package/src/seed.ts
CHANGED
|
@@ -10,10 +10,11 @@
|
|
|
10
10
|
* the seed replays the cached push result and writes nothing twice.
|
|
11
11
|
*/
|
|
12
12
|
import {
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
decodeResponse,
|
|
14
|
+
encodeRequest,
|
|
15
15
|
encodeSparseRow,
|
|
16
16
|
PROTOCOL_WIRE_VERSION,
|
|
17
|
+
registrationDigest,
|
|
17
18
|
type PushOperation,
|
|
18
19
|
type PushResultFrame,
|
|
19
20
|
type RequestFrame,
|
|
@@ -192,9 +193,9 @@ export async function seedMutations(
|
|
|
192
193
|
partition: target.partition,
|
|
193
194
|
actorId: target.actorId,
|
|
194
195
|
};
|
|
195
|
-
const acquisition =
|
|
196
|
+
const acquisition = decodeResponse(
|
|
196
197
|
await handleSyncRequest(
|
|
197
|
-
|
|
198
|
+
encodeRequest({
|
|
198
199
|
wireVersion: PROTOCOL_WIRE_VERSION,
|
|
199
200
|
msgKind: 'request',
|
|
200
201
|
frames: [
|
|
@@ -209,6 +210,9 @@ export async function seedMutations(
|
|
|
209
210
|
limitSnapshotRows: 0,
|
|
210
211
|
maxSnapshotPages: 0,
|
|
211
212
|
accept: 0b0011,
|
|
213
|
+
cursor: -1,
|
|
214
|
+
registrationDigest: registrationDigest([]),
|
|
215
|
+
echoDigest: new Uint8Array(),
|
|
212
216
|
},
|
|
213
217
|
],
|
|
214
218
|
}),
|
|
@@ -216,11 +220,7 @@ export async function seedMutations(
|
|
|
216
220
|
),
|
|
217
221
|
);
|
|
218
222
|
const acquisitionHeader = acquisition.frames[0];
|
|
219
|
-
if (
|
|
220
|
-
acquisition.msgKind !== 'response' ||
|
|
221
|
-
acquisitionHeader?.type !== 'RESP_HEADER' ||
|
|
222
|
-
acquisitionHeader.logEpoch === undefined
|
|
223
|
-
) {
|
|
223
|
+
if (acquisitionHeader?.type !== 'RESP_HEADER') {
|
|
224
224
|
throw new SyncError(
|
|
225
225
|
'sync.invalid_request',
|
|
226
226
|
'seedMutations: epoch acquisition returned an invalid response',
|
|
@@ -233,15 +233,8 @@ export async function seedMutations(
|
|
|
233
233
|
schemaVersion: config.schema.version,
|
|
234
234
|
logEpoch: acquisitionHeader.logEpoch,
|
|
235
235
|
},
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
{
|
|
239
|
-
type: 'PULL_HEADER',
|
|
240
|
-
limitCommits: 0,
|
|
241
|
-
limitSnapshotRows: 0,
|
|
242
|
-
maxSnapshotPages: 0,
|
|
243
|
-
accept: 0b0011,
|
|
244
|
-
},
|
|
236
|
+
// §6.10: a database-less producer sends no outboxId and clientSeq 0.
|
|
237
|
+
{ type: 'PUSH_COMMIT', clientCommitId, clientSeq: 0, operations },
|
|
245
238
|
];
|
|
246
239
|
type SeedTerminalEvent = Extract<
|
|
247
240
|
SyncularServerEvent,
|
|
@@ -260,7 +253,7 @@ export async function seedMutations(
|
|
|
260
253
|
},
|
|
261
254
|
};
|
|
262
255
|
const response = await handleSyncRequest(
|
|
263
|
-
|
|
256
|
+
encodeRequest({
|
|
264
257
|
wireVersion: PROTOCOL_WIRE_VERSION,
|
|
265
258
|
msgKind: 'request',
|
|
266
259
|
frames,
|
|
@@ -275,7 +268,7 @@ export async function seedMutations(
|
|
|
275
268
|
);
|
|
276
269
|
|
|
277
270
|
// Fail loud: surface the first rejected/failed operation.
|
|
278
|
-
const message =
|
|
271
|
+
const message = decodeResponse(response);
|
|
279
272
|
const result = message.frames.find(
|
|
280
273
|
(frame): frame is PushResultFrame =>
|
|
281
274
|
frame.type === 'PUSH_RESULT' && frame.clientCommitId === clientCommitId,
|
package/src/segment-download.ts
CHANGED
|
@@ -28,7 +28,10 @@ import {
|
|
|
28
28
|
|
|
29
29
|
export interface SegmentDownloadRequest {
|
|
30
30
|
readonly segmentId: string;
|
|
31
|
-
/**
|
|
31
|
+
/**
|
|
32
|
+
* The `X-Syncular-Scopes` header: canonical JSON of the owning unit's scope
|
|
33
|
+
* map, each value as a one-element list (§5.5).
|
|
34
|
+
*/
|
|
32
35
|
readonly scopesHeader: string;
|
|
33
36
|
}
|
|
34
37
|
|
|
@@ -60,13 +63,15 @@ function parseScopesHeader(header: string): ScopeMap {
|
|
|
60
63
|
}
|
|
61
64
|
const scopes: ScopeMap = {};
|
|
62
65
|
for (const [key, value] of Object.entries(parsed)) {
|
|
66
|
+
// §5.5: the owning unit's scope map, each value as a one-element list.
|
|
63
67
|
if (
|
|
64
68
|
!Array.isArray(value) ||
|
|
65
|
-
value.
|
|
69
|
+
value.length !== 1 ||
|
|
70
|
+
typeof value[0] !== 'string'
|
|
66
71
|
) {
|
|
67
72
|
throw syncError(
|
|
68
73
|
'sync.invalid_request',
|
|
69
|
-
'X-Syncular-Scopes values must be string lists (§
|
|
74
|
+
'X-Syncular-Scopes values must be one-element string lists (§5.5)',
|
|
70
75
|
);
|
|
71
76
|
}
|
|
72
77
|
scopes[key] = value as string[];
|
|
@@ -263,15 +268,36 @@ async function authorizePublication(
|
|
|
263
268
|
}
|
|
264
269
|
resolved = { ok: false };
|
|
265
270
|
}
|
|
266
|
-
|
|
267
|
-
|
|
271
|
+
// §3.2: the effective scopes, and so the digest, depend on the declared
|
|
272
|
+
// variables of the publication's table (the wildcard fence).
|
|
273
|
+
const digests = new Map<string, string | undefined>();
|
|
274
|
+
for (const publication of inContext) {
|
|
275
|
+
const key = `${publication.schemaVersion}\u0000${publication.table}`;
|
|
276
|
+
if (digests.has(key)) continue;
|
|
277
|
+
const table = schemaWindowOf(ctx.schema, ctx.schemaWindow)
|
|
278
|
+
.find((candidate) => candidate.version === publication.schemaVersion)
|
|
279
|
+
?.tables.get(publication.table);
|
|
280
|
+
const outcome = computeEffective(
|
|
281
|
+
table?.declaredVariables ?? new Set(),
|
|
282
|
+
requested,
|
|
283
|
+
resolved,
|
|
284
|
+
);
|
|
285
|
+
digests.set(
|
|
286
|
+
key,
|
|
287
|
+
outcome.status === 'active'
|
|
288
|
+
? await scopeDigest(outcome.effective)
|
|
289
|
+
: undefined,
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
if ([...digests.values()].every((digest) => digest === undefined)) {
|
|
268
293
|
throw syncError('sync.forbidden', 'segment scopes not held (§5.5)');
|
|
269
294
|
}
|
|
270
|
-
const digest = await scopeDigest(outcome.effective);
|
|
271
295
|
// The digest is the grant, but only inside the caller's partition and live
|
|
272
296
|
// epoch: identical bytes published elsewhere never authorize this caller.
|
|
273
297
|
const forDigest = inContext.filter(
|
|
274
|
-
(publication) =>
|
|
298
|
+
(publication) =>
|
|
299
|
+
digests.get(`${publication.schemaVersion}\u0000${publication.table}`) ===
|
|
300
|
+
publication.scopeDigest,
|
|
275
301
|
);
|
|
276
302
|
if (forDigest.length === 0) {
|
|
277
303
|
throw syncError('sync.forbidden', 'scope digest mismatch (§3.5, §5.5)');
|
package/src/signed-url.ts
CHANGED
|
@@ -146,8 +146,8 @@ export async function issueBlobUrl(
|
|
|
146
146
|
* twin of `BlobPresignConfig`: the provider signs a single PUT whose object
|
|
147
147
|
* key embeds the `blobId`. Issuance happens ONLY after host authentication in
|
|
148
148
|
* the upload-grant handler (uploading is host-auth-only, not scope-bearing —
|
|
149
|
-
* §5.9.3). The
|
|
150
|
-
*
|
|
149
|
+
* §5.9.3). The provider MUST bind the PUT to the granted `byteLength` and to
|
|
150
|
+
* the content address, so the object store rejects other bytes. Never a multipart or
|
|
151
151
|
* chunk protocol — single PUT only, this rung and the next (§5.9.3 non-goal).
|
|
152
152
|
* `S3BlobStore.presignBlobPut` (via `s3PresignedBlobUploads`) is the in-tree
|
|
153
153
|
* implementation.
|
|
@@ -286,8 +286,14 @@ export async function verifySegmentToken(
|
|
|
286
286
|
syncError('sync.forbidden', `segment token rejected: ${reason}`);
|
|
287
287
|
const dot = token.indexOf('.');
|
|
288
288
|
if (dot < 0) throw forbidden('malformed token');
|
|
289
|
-
|
|
290
|
-
|
|
289
|
+
let payload: Uint8Array;
|
|
290
|
+
let mac: Uint8Array;
|
|
291
|
+
try {
|
|
292
|
+
payload = fromBase64url(token.slice(0, dot));
|
|
293
|
+
mac = fromBase64url(token.slice(dot + 1));
|
|
294
|
+
} catch {
|
|
295
|
+
throw forbidden('malformed token');
|
|
296
|
+
}
|
|
291
297
|
const computed = await hmac(key, payload);
|
|
292
298
|
if (!constantTimeEqual(mac, computed)) throw forbidden('bad MAC');
|
|
293
299
|
let claims: SegmentTokenClaims;
|