@syncular/server 0.38.0 → 0.39.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/content-encoding.d.ts +24 -23
- package/dist/content-encoding.js +77 -22
- package/dist/d1-storage.js +1 -1
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +207 -64
- package/dist/pull.d.ts +7 -3
- package/dist/pull.js +21 -12
- package/dist/relational-rows.d.ts +16 -10
- package/dist/relational-rows.js +48 -42
- package/dist/sqlite-storage.js +16 -3
- package/package.json +2 -2
- package/src/content-encoding.ts +99 -31
- package/src/d1-storage.ts +1 -1
- package/src/postgres-storage.ts +277 -93
- package/src/pull.ts +31 -18
- package/src/relational-rows.ts +53 -47
- package/src/sqlite-storage.ts +20 -3
package/src/postgres-storage.ts
CHANGED
|
@@ -67,9 +67,10 @@ import {
|
|
|
67
67
|
migratePayload,
|
|
68
68
|
parseLayouts,
|
|
69
69
|
quoteIdent,
|
|
70
|
+
tableColumnNames,
|
|
70
71
|
retiredTableNames,
|
|
71
72
|
rewritePlan,
|
|
72
|
-
|
|
73
|
+
columnSqlType,
|
|
73
74
|
rewriteValues,
|
|
74
75
|
SCHEMA_META_DDL_POSTGRES,
|
|
75
76
|
SYNC_COLUMN_VERSIONS_COLUMN,
|
|
@@ -153,7 +154,7 @@ import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query';
|
|
|
153
154
|
* their table.
|
|
154
155
|
*
|
|
155
156
|
* `sync_row_scopes_by_row (partition, tbl, row_id)` is the one secondary
|
|
156
|
-
* scope index: `
|
|
157
|
+
* scope index: `writeRowsOn` and `deleteRow` replace a row's scope entries
|
|
157
158
|
* with a `(partition, tbl, row_id)` predicate, and `row_id` is the LAST
|
|
158
159
|
* column of the inverted PRIMARY KEY, so without this index that predicate
|
|
159
160
|
* scans the table's whole scope range on every row written (a bulk import
|
|
@@ -645,8 +646,30 @@ async function rewriteRowsOn(
|
|
|
645
646
|
oldLayout: readonly StoredColumnLayout[] | undefined,
|
|
646
647
|
): Promise<void> {
|
|
647
648
|
const select = selectRowsForRewriteSql(table, 'postgres');
|
|
648
|
-
|
|
649
|
-
|
|
649
|
+
// One UPDATE per page: behind a network hop (Hyperdrive, a managed
|
|
650
|
+
// database) every statement is a round trip, and the bump holds the
|
|
651
|
+
// migration lock for its whole duration. The page's rows bind as one
|
|
652
|
+
// VALUES list with typed placeholders (`rewriteValues` order: projection
|
|
653
|
+
// columns, payload, partition, row id).
|
|
654
|
+
const types = [
|
|
655
|
+
...(table.materialize
|
|
656
|
+
? table.columns.map((column) => columnSqlType(column, 'postgres'))
|
|
657
|
+
: []),
|
|
658
|
+
'BYTEA',
|
|
659
|
+
'TEXT',
|
|
660
|
+
'TEXT',
|
|
661
|
+
];
|
|
662
|
+
const projected = table.materialize ? table.columns : [];
|
|
663
|
+
const sets = [
|
|
664
|
+
...projected.map((column, i) => `${quoteIdent(column.name)}=v.c${i}`),
|
|
665
|
+
`${quoteIdent(SYNC_PAYLOAD_COLUMN)}=v.c${projected.length}`,
|
|
666
|
+
].join(', ');
|
|
667
|
+
const aliases = types.map((_, i) => `c${i}`).join(', ');
|
|
668
|
+
const partitionAlias = `c${projected.length + 1}`;
|
|
669
|
+
const rowIdAlias = `c${projected.length + 2}`;
|
|
670
|
+
// A page binds types.length parameters per row; PGlite corrupts its state
|
|
671
|
+
// above 32,767 bound parameters (see PG_CHANGES_PER_STATEMENT).
|
|
672
|
+
const batch = Math.max(1, Math.min(500, Math.floor(30_000 / types.length)));
|
|
650
673
|
let afterPartition = '';
|
|
651
674
|
let afterRowId = '';
|
|
652
675
|
for (;;) {
|
|
@@ -654,21 +677,30 @@ async function rewriteRowsOn(
|
|
|
654
677
|
partition: string;
|
|
655
678
|
row_id: string;
|
|
656
679
|
payload: unknown;
|
|
657
|
-
}>(select, [afterPartition, afterRowId,
|
|
680
|
+
}>(select, [afterPartition, afterRowId, batch]);
|
|
658
681
|
if (rows.length === 0) break;
|
|
659
|
-
|
|
682
|
+
const params: unknown[] = [];
|
|
683
|
+
const values = rows.map((row) => {
|
|
660
684
|
const bytes = asBytes(row.payload);
|
|
661
685
|
const payload =
|
|
662
686
|
oldLayout !== undefined
|
|
663
687
|
? migratePayload(oldLayout, table, bytes)
|
|
664
688
|
: bytes;
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
rewriteValues(table, row.partition, row.row_id, payload, 'postgres'),
|
|
689
|
+
const base = params.length;
|
|
690
|
+
params.push(
|
|
691
|
+
...rewriteValues(table, row.partition, row.row_id, payload, 'postgres'),
|
|
668
692
|
);
|
|
669
|
-
|
|
693
|
+
return `(${types.map((type, i) => `$${base + i + 1}::${type}`).join(',')})`;
|
|
694
|
+
});
|
|
695
|
+
await q.query(
|
|
696
|
+
`UPDATE ${quoteIdent(table.name)} AS t SET ${sets}
|
|
697
|
+
FROM (VALUES ${values.join(',')}) AS v(${aliases})
|
|
698
|
+
WHERE t.${quoteIdent(SYNC_PARTITION_COLUMN)}=v.${partitionAlias}
|
|
699
|
+
AND t.${quoteIdent(SYNC_ROW_ID_COLUMN)}=v.${rowIdAlias}`,
|
|
700
|
+
params,
|
|
701
|
+
);
|
|
670
702
|
const last = rows[rows.length - 1];
|
|
671
|
-
if (last === undefined || rows.length <
|
|
703
|
+
if (last === undefined || rows.length < batch) break;
|
|
672
704
|
afterPartition = last.partition;
|
|
673
705
|
afterRowId = last.row_id;
|
|
674
706
|
}
|
|
@@ -709,37 +741,67 @@ async function scanRowsByIndexOn(
|
|
|
709
741
|
return rows.map(toStoredRow);
|
|
710
742
|
}
|
|
711
743
|
|
|
712
|
-
|
|
744
|
+
/**
|
|
745
|
+
* Write rows of one table in one statement per chunk: the upsert, and the
|
|
746
|
+
* scope index synced to each row's scope map (entries outside it deleted,
|
|
747
|
+
* missing ones inserted). Row ids are distinct, so no index entry is touched
|
|
748
|
+
* twice and the writable CTEs cannot conflict. The scopes bind as text
|
|
749
|
+
* before the JSONB cast (see `appendCommit`). Every chunk is handed to `q`
|
|
750
|
+
* before the first one settles, so a statement issued after this call
|
|
751
|
+
* queues behind all of them on the transaction FIFO.
|
|
752
|
+
*/
|
|
753
|
+
function writeRowsOn(
|
|
713
754
|
q: PgQueryable,
|
|
714
755
|
compiled: CompiledTable,
|
|
715
756
|
partition: string,
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
const
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
757
|
+
rows: readonly {
|
|
758
|
+
readonly row: StoredRow;
|
|
759
|
+
/** `upsertValues` of the row, computed when it was staged. */
|
|
760
|
+
readonly values: readonly unknown[];
|
|
761
|
+
}[],
|
|
762
|
+
): Promise<unknown> {
|
|
763
|
+
const width = tableColumnNames(compiled).length;
|
|
764
|
+
// PGlite corrupts its state above 32,767 bound parameters (see
|
|
765
|
+
// PG_CHANGES_PER_STATEMENT).
|
|
766
|
+
const chunk = Math.max(1, Math.floor(30_000 / width));
|
|
767
|
+
const writes: Promise<unknown>[] = [];
|
|
768
|
+
for (let first = 0; first < rows.length; first += chunk) {
|
|
769
|
+
const slice = rows.slice(first, first + chunk);
|
|
770
|
+
const params = slice.flatMap((entry) => entry.values);
|
|
771
|
+
const n = params.length;
|
|
772
|
+
writes.push(
|
|
773
|
+
q.query(
|
|
774
|
+
`WITH upserted AS (${upsertSql(compiled, 'postgres', slice.length)}),
|
|
775
|
+
batch AS (
|
|
776
|
+
SELECT row_id, scopes
|
|
777
|
+
FROM jsonb_to_recordset($${n + 1}::text::jsonb) AS r(row_id text, scopes jsonb)
|
|
778
|
+
),
|
|
779
|
+
scope AS (
|
|
780
|
+
SELECT batch.row_id, entry.key, entry.value
|
|
781
|
+
FROM batch CROSS JOIN LATERAL jsonb_each_text(batch.scopes) AS entry
|
|
782
|
+
),
|
|
783
|
+
removed AS (
|
|
784
|
+
DELETE FROM sync_row_scopes existing
|
|
785
|
+
USING batch
|
|
786
|
+
WHERE existing.partition=$${n + 2} AND existing.tbl=$${n + 3}
|
|
787
|
+
AND existing.row_id=batch.row_id
|
|
788
|
+
AND batch.scopes->>existing.var IS DISTINCT FROM existing.value
|
|
789
|
+
)
|
|
790
|
+
INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
791
|
+
SELECT $${n + 2}, $${n + 3}, key, value, row_id FROM scope
|
|
792
|
+
ON CONFLICT DO NOTHING`,
|
|
793
|
+
[
|
|
794
|
+
...params,
|
|
795
|
+
JSON.stringify(
|
|
796
|
+
slice.map(({ row }) => ({ row_id: row.rowId, scopes: row.scopes })),
|
|
797
|
+
),
|
|
798
|
+
partition,
|
|
799
|
+
compiled.name,
|
|
800
|
+
],
|
|
801
|
+
),
|
|
802
|
+
);
|
|
803
|
+
}
|
|
804
|
+
return Promise.all(writes);
|
|
743
805
|
}
|
|
744
806
|
|
|
745
807
|
async function getPushResultOn(
|
|
@@ -788,6 +850,8 @@ async function queryAuthoritativeOn(
|
|
|
788
850
|
|
|
789
851
|
class PostgresTransaction implements StorageTransaction {
|
|
790
852
|
#client: PgQueryable;
|
|
853
|
+
/** The FIFO without the staged-write prelude (see the constructor). */
|
|
854
|
+
#serial: PgQueryable;
|
|
791
855
|
#partition: string;
|
|
792
856
|
/** Storage-owned relation rewrite, bound to this transaction's partition. */
|
|
793
857
|
readonly #prepareQuery: (
|
|
@@ -805,6 +869,25 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
805
869
|
*/
|
|
806
870
|
readonly #prefetchedRows = new Map<string, StoredRow | null>();
|
|
807
871
|
readonly #prefetchedTombstones = new Map<string, number | null>();
|
|
872
|
+
/**
|
|
873
|
+
* Upserts staged per table (row id → row and its push operation index).
|
|
874
|
+
* Every statement of this transaction writes them first (see the
|
|
875
|
+
* constructor), so reads and later writes observe them; a commit's rows
|
|
876
|
+
* reach the database as one statement per table instead of one per row.
|
|
877
|
+
* Tables with a unique index write immediately: a violation must name its
|
|
878
|
+
* operation.
|
|
879
|
+
*/
|
|
880
|
+
readonly #staged = new Map<
|
|
881
|
+
CompiledTable,
|
|
882
|
+
Map<
|
|
883
|
+
string,
|
|
884
|
+
{
|
|
885
|
+
readonly row: StoredRow;
|
|
886
|
+
readonly values: readonly unknown[];
|
|
887
|
+
readonly opIndex?: number;
|
|
888
|
+
}
|
|
889
|
+
>
|
|
890
|
+
>();
|
|
808
891
|
/** Resolves/rejects the `transaction(fn)` wrapper (see `begin`). */
|
|
809
892
|
#resolve: () => void;
|
|
810
893
|
#reject: (error: unknown) => void;
|
|
@@ -825,7 +908,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
825
908
|
// issued through this transaction is chained on one FIFO — the same
|
|
826
909
|
// shape as `SqliteServerStorage.queryAuthoritative`.
|
|
827
910
|
let tail: Promise<void> = Promise.resolve();
|
|
828
|
-
|
|
911
|
+
const serial: PgQueryable = {
|
|
829
912
|
query: <Row>(text: string, params?: readonly unknown[]) => {
|
|
830
913
|
const run = tail.then(() => client.query<Row>(text, params));
|
|
831
914
|
tail = run.then(
|
|
@@ -835,6 +918,23 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
835
918
|
return run;
|
|
836
919
|
},
|
|
837
920
|
};
|
|
921
|
+
this.#serial = serial;
|
|
922
|
+
this.#client = {
|
|
923
|
+
query: async <Row>(text: string, params?: readonly unknown[]) => {
|
|
924
|
+
// Queue the staged writes ahead of this statement synchronously, so
|
|
925
|
+
// a concurrently issued statement cannot overtake them.
|
|
926
|
+
const staged = this.#writeStaged();
|
|
927
|
+
const run = serial.query<Row>(text, params);
|
|
928
|
+
try {
|
|
929
|
+
await staged;
|
|
930
|
+
} catch (error) {
|
|
931
|
+
// The aborted transaction fails the queued statement too.
|
|
932
|
+
run.catch(() => undefined);
|
|
933
|
+
throw error;
|
|
934
|
+
}
|
|
935
|
+
return run;
|
|
936
|
+
},
|
|
937
|
+
};
|
|
838
938
|
this.#partition = partition;
|
|
839
939
|
this.#writerVersion = writerVersion;
|
|
840
940
|
this.#resolveTable = resolveTable;
|
|
@@ -847,6 +947,27 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
847
947
|
if (!this.#open) throw new Error('transaction already finished');
|
|
848
948
|
}
|
|
849
949
|
|
|
950
|
+
#writeStaged(): Promise<unknown> {
|
|
951
|
+
if (this.#staged.size === 0) return Promise.resolve();
|
|
952
|
+
const staged = [...this.#staged];
|
|
953
|
+
this.#staged.clear();
|
|
954
|
+
return Promise.all(
|
|
955
|
+
staged.map(([table, rows]) => {
|
|
956
|
+
const entries = [...rows.values()];
|
|
957
|
+
return writeRowsOn(this.#serial, table, this.#partition, entries).catch(
|
|
958
|
+
(error: unknown) => {
|
|
959
|
+
// Staged tables have no unique index, so a violation is a column
|
|
960
|
+
// constraint; it names the first staged operation of the table.
|
|
961
|
+
if (isPostgresConstraintError(error)) {
|
|
962
|
+
throw new StorageConstraintError(error, entries[0]?.opIndex);
|
|
963
|
+
}
|
|
964
|
+
throw error;
|
|
965
|
+
},
|
|
966
|
+
);
|
|
967
|
+
}),
|
|
968
|
+
);
|
|
969
|
+
}
|
|
970
|
+
|
|
850
971
|
getRow(table: string, rowId: string): Promise<StoredRow | undefined> {
|
|
851
972
|
this.#assertOpen();
|
|
852
973
|
const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
|
|
@@ -1039,6 +1160,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1039
1160
|
if (!this.#pushApplySavepoint) {
|
|
1040
1161
|
throw new Error('push rejection requires its apply savepoint');
|
|
1041
1162
|
}
|
|
1163
|
+
this.#staged.clear();
|
|
1042
1164
|
await this.#client.query('ROLLBACK TO SAVEPOINT syncular_push_candidate');
|
|
1043
1165
|
await this.#client.query('RELEASE SAVEPOINT syncular_push_candidate');
|
|
1044
1166
|
this.#pushApplySavepoint = false;
|
|
@@ -1053,13 +1175,28 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1053
1175
|
): Promise<void> {
|
|
1054
1176
|
this.#assertOpen();
|
|
1055
1177
|
this.#prefetchedRows.delete(`${table}\u0000${row.rowId}`);
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1178
|
+
const compiled = this.#resolveTable(table);
|
|
1179
|
+
// Decodes the payload: an undecodable row fails here, at its operation.
|
|
1180
|
+
const values = upsertValues(compiled, this.#partition, row, 'postgres');
|
|
1181
|
+
if (!compiled.indexes.some((index) => index.unique === true)) {
|
|
1182
|
+
let rows = this.#staged.get(compiled);
|
|
1183
|
+
if (rows === undefined) {
|
|
1184
|
+
rows = new Map();
|
|
1185
|
+
this.#staged.set(compiled, rows);
|
|
1186
|
+
}
|
|
1187
|
+
// A later upsert of the same row in this transaction replaces it.
|
|
1188
|
+
rows.delete(row.rowId);
|
|
1189
|
+
rows.set(row.rowId, {
|
|
1061
1190
|
row,
|
|
1062
|
-
|
|
1191
|
+
values,
|
|
1192
|
+
...(context === undefined ? {} : { opIndex: context.opIndex }),
|
|
1193
|
+
});
|
|
1194
|
+
return;
|
|
1195
|
+
}
|
|
1196
|
+
try {
|
|
1197
|
+
await writeRowsOn(this.#client, compiled, this.#partition, [
|
|
1198
|
+
{ row, values },
|
|
1199
|
+
]);
|
|
1063
1200
|
} catch (error) {
|
|
1064
1201
|
if (isPostgresConstraintError(error)) {
|
|
1065
1202
|
throw new StorageConstraintError(error, context?.opIndex);
|
|
@@ -1232,6 +1369,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1232
1369
|
|
|
1233
1370
|
async commit(): Promise<void> {
|
|
1234
1371
|
this.#assertOpen();
|
|
1372
|
+
await this.#writeStaged();
|
|
1235
1373
|
this.#open = false;
|
|
1236
1374
|
// Signal the transaction wrapper to COMMIT; wait for it to actually land.
|
|
1237
1375
|
this.#resolve();
|
|
@@ -1240,6 +1378,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1240
1378
|
|
|
1241
1379
|
async rollback(): Promise<void> {
|
|
1242
1380
|
if (!this.#open) return;
|
|
1381
|
+
this.#staged.clear();
|
|
1243
1382
|
this.#open = false;
|
|
1244
1383
|
this.#reject(new RollbackSignal());
|
|
1245
1384
|
// Swallow — a rollback is a normal outcome, not an error to the caller.
|
|
@@ -1280,11 +1419,11 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1280
1419
|
#schemaVersion: number | undefined;
|
|
1281
1420
|
/**
|
|
1282
1421
|
* Page reads waiting for the end of the current microtask turn, keyed by
|
|
1283
|
-
*
|
|
1422
|
+
* batch stream and partition. A pull starts the window read (or first
|
|
1284
1423
|
* snapshot page) of every subscription before awaiting any of them, so the
|
|
1285
|
-
*
|
|
1286
|
-
*
|
|
1287
|
-
*
|
|
1424
|
+
* reads of all tables leave as ONE set-based statement per stream instead
|
|
1425
|
+
* of one round trip per subscription or table
|
|
1426
|
+
* (SYNCULAR-PULL-ROUNDTRIPS-001).
|
|
1288
1427
|
*/
|
|
1289
1428
|
readonly #pageBatches = new Map<string, PendingPage[]>();
|
|
1290
1429
|
|
|
@@ -1293,37 +1432,42 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1293
1432
|
}
|
|
1294
1433
|
|
|
1295
1434
|
/**
|
|
1296
|
-
* Queue one page read into the batch for (`
|
|
1297
|
-
*
|
|
1298
|
-
*
|
|
1299
|
-
*
|
|
1435
|
+
* Queue one page read into the batch for (`key`, `partition`). `sql` is the
|
|
1436
|
+
* statement every batch of that stream shares, or a builder over the queued
|
|
1437
|
+
* specs when the statement names what the batch contains (the snapshot scan
|
|
1438
|
+
* joins one branch per table present). The batch runs as a single statement
|
|
1439
|
+
* with the queued specs bound as a JSON array (`$2`); each spec gets its
|
|
1440
|
+
* position as `i`, and the result rows are split back by their `sub`
|
|
1441
|
+
* column.
|
|
1300
1442
|
*/
|
|
1301
1443
|
#batchedPage<Row>(
|
|
1302
|
-
|
|
1444
|
+
key: string,
|
|
1445
|
+
sql: string | ((specs: readonly Record<string, unknown>[]) => string),
|
|
1303
1446
|
partition: string,
|
|
1304
1447
|
spec: Record<string, unknown>,
|
|
1305
1448
|
): Promise<Row[]> {
|
|
1306
|
-
const
|
|
1307
|
-
let batch = this.#pageBatches.get(
|
|
1449
|
+
const batchKey = `${key}\u0000${partition}`;
|
|
1450
|
+
let batch = this.#pageBatches.get(batchKey);
|
|
1308
1451
|
if (batch === undefined) {
|
|
1309
1452
|
const queued: PendingPage[] = [];
|
|
1310
1453
|
batch = queued;
|
|
1311
|
-
this.#pageBatches.set(
|
|
1454
|
+
this.#pageBatches.set(batchKey, queued);
|
|
1312
1455
|
queueMicrotask(() => {
|
|
1313
|
-
this.#pageBatches.delete(
|
|
1314
|
-
const specs =
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
(
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1456
|
+
this.#pageBatches.delete(batchKey);
|
|
1457
|
+
const specs = queued.map((entry, i) => ({ ...entry.spec, i }));
|
|
1458
|
+
const statement = typeof sql === 'string' ? sql : sql(specs);
|
|
1459
|
+
this.#exec
|
|
1460
|
+
.query<PgRow>(statement, [partition, JSON.stringify(specs)])
|
|
1461
|
+
.then(
|
|
1462
|
+
({ rows }) => {
|
|
1463
|
+
const grouped = queued.map((): PgRow[] => []);
|
|
1464
|
+
for (const row of rows) grouped[asNumber(row.sub)]?.push(row);
|
|
1465
|
+
queued.forEach((entry, i) => entry.resolve(grouped[i] ?? []));
|
|
1466
|
+
},
|
|
1467
|
+
(error: unknown) => {
|
|
1468
|
+
for (const entry of queued) entry.reject(error);
|
|
1469
|
+
},
|
|
1470
|
+
);
|
|
1327
1471
|
});
|
|
1328
1472
|
}
|
|
1329
1473
|
const target = batch;
|
|
@@ -1538,10 +1682,6 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1538
1682
|
// backfill for flipped-on materialization) — all inside one
|
|
1539
1683
|
// transaction (Postgres DDL is transactional — a failed bump leaves
|
|
1540
1684
|
// no half-state).
|
|
1541
|
-
const layouts = parseLayouts(
|
|
1542
|
-
typeof marker.layouts === 'string' ? marker.layouts : undefined,
|
|
1543
|
-
);
|
|
1544
|
-
const retiredTables = retiredTableNames(schema, layouts);
|
|
1545
1685
|
await this.#exec.transaction(async (client) => {
|
|
1546
1686
|
// RFC 0007 migration barrier: `LOCK TABLE ... EXCLUSIVE` is the FIRST
|
|
1547
1687
|
// statement, before any read, DDL, rewrite, marker or checkpoint
|
|
@@ -1554,6 +1694,31 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1554
1694
|
// when it takes this lock, so a writer already inside completes while
|
|
1555
1695
|
// the migration waits.
|
|
1556
1696
|
await client.query('LOCK TABLE sync_partitions IN EXCLUSIVE MODE');
|
|
1697
|
+
// The probe ran before the lock. A concurrent `ensureSchema` (another
|
|
1698
|
+
// isolate, the Durable Object's first request) may have bumped while
|
|
1699
|
+
// this one waited; re-running the rewrite would decode its new-layout
|
|
1700
|
+
// payloads with the old layout. The locked marker decides.
|
|
1701
|
+
const { rows: lockedMarker } = await client.query<{
|
|
1702
|
+
schema_version: unknown;
|
|
1703
|
+
layouts: unknown;
|
|
1704
|
+
}>('SELECT schema_version, layouts FROM sync_schema_meta WHERE id=1');
|
|
1705
|
+
const lockedVersion =
|
|
1706
|
+
lockedMarker[0]?.schema_version === null ||
|
|
1707
|
+
lockedMarker[0]?.schema_version === undefined
|
|
1708
|
+
? undefined
|
|
1709
|
+
: asNumber(lockedMarker[0].schema_version);
|
|
1710
|
+
if (lockedVersion !== undefined && lockedVersion > schema.version) {
|
|
1711
|
+
throw new Error(
|
|
1712
|
+
`stored schema version ${lockedVersion} is newer than the configured schema (${schema.version}) — refusing to run an older server against a migrated database`,
|
|
1713
|
+
);
|
|
1714
|
+
}
|
|
1715
|
+
if (lockedVersion === schema.version) return;
|
|
1716
|
+
const layouts = parseLayouts(
|
|
1717
|
+
typeof lockedMarker[0]?.layouts === 'string'
|
|
1718
|
+
? lockedMarker[0].layouts
|
|
1719
|
+
: undefined,
|
|
1720
|
+
);
|
|
1721
|
+
const retiredTables = retiredTableNames(schema, layouts);
|
|
1557
1722
|
const existing = new Map<string, ReadonlySet<string>>();
|
|
1558
1723
|
const existingIndexes = new Map<string, ReadonlyMap<string, string>>();
|
|
1559
1724
|
for (const table of schema.tables.values()) {
|
|
@@ -1620,13 +1785,21 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1620
1785
|
);
|
|
1621
1786
|
}
|
|
1622
1787
|
}
|
|
1788
|
+
if (retiredTables.length > 0) {
|
|
1789
|
+
// Neither index leads with `tbl`: one scan per table, not one per
|
|
1790
|
+
// retired table.
|
|
1791
|
+
await client.query(
|
|
1792
|
+
`DELETE FROM sync_row_scopes
|
|
1793
|
+
WHERE tbl IN (SELECT jsonb_array_elements_text($1::text::jsonb))`,
|
|
1794
|
+
[JSON.stringify(retiredTables)],
|
|
1795
|
+
);
|
|
1796
|
+
await client.query(
|
|
1797
|
+
`DELETE FROM sync_blob_refs
|
|
1798
|
+
WHERE tbl IN (SELECT jsonb_array_elements_text($1::text::jsonb))`,
|
|
1799
|
+
[JSON.stringify(retiredTables)],
|
|
1800
|
+
);
|
|
1801
|
+
}
|
|
1623
1802
|
for (const tableName of retiredTables) {
|
|
1624
|
-
await client.query('DELETE FROM sync_row_scopes WHERE tbl=$1', [
|
|
1625
|
-
tableName,
|
|
1626
|
-
]);
|
|
1627
|
-
await client.query('DELETE FROM sync_blob_refs WHERE tbl=$1', [
|
|
1628
|
-
tableName,
|
|
1629
|
-
]);
|
|
1630
1803
|
await client.query(dropTableDdl(tableName));
|
|
1631
1804
|
}
|
|
1632
1805
|
for (const statement of schemaDdl(
|
|
@@ -2359,6 +2532,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2359
2532
|
if (firstVariable === undefined || values.length === 0) return 0;
|
|
2360
2533
|
// Concurrent reads of a pull share one statement across tables.
|
|
2361
2534
|
const [row] = await this.#batchedPage<{ seq: unknown }>(
|
|
2535
|
+
'latestScopedChange',
|
|
2362
2536
|
LATEST_SCOPED_CHANGE_BATCH_SQL,
|
|
2363
2537
|
partition,
|
|
2364
2538
|
{ tbl: query.table, var: firstVariable, vals: values },
|
|
@@ -2388,6 +2562,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2388
2562
|
const batchSize = Math.max(64, query.limitChanges);
|
|
2389
2563
|
while (deliveredChanges < query.limitChanges) {
|
|
2390
2564
|
const records = await this.#batchedPage<CommitWindowRecord>(
|
|
2565
|
+
'commitWindow',
|
|
2391
2566
|
COMMIT_WINDOW_BATCH_SQL,
|
|
2392
2567
|
partition,
|
|
2393
2568
|
{
|
|
@@ -2456,21 +2631,30 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2456
2631
|
if (firstValues.length === 0) return [];
|
|
2457
2632
|
// Candidates via the inverted index (ordered + LIMITed at the covering
|
|
2458
2633
|
// PK) LEFT JOINed to the row table — one round trip per page, never one
|
|
2459
|
-
// per row, and
|
|
2460
|
-
//
|
|
2461
|
-
//
|
|
2462
|
-
const sql = scanRowBatchSql(this.table(query.table));
|
|
2634
|
+
// per row, and concurrent scans across tables share one statement (see
|
|
2635
|
+
// `scanRowBatchSql`). Exact multi-variable verification against the
|
|
2636
|
+
// stored scope map happens below.
|
|
2463
2637
|
const rows: StoredRow[] = [];
|
|
2464
2638
|
let afterRowId = query.afterRowId ?? '';
|
|
2465
2639
|
const batchSize = Math.max(64, query.limit);
|
|
2466
2640
|
while (rows.length < query.limit) {
|
|
2467
|
-
const records = await this.#batchedPage<RowRecord>(
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2641
|
+
const records = await this.#batchedPage<RowRecord>(
|
|
2642
|
+
'scanRows',
|
|
2643
|
+
(specs) =>
|
|
2644
|
+
scanRowBatchSql(
|
|
2645
|
+
[...new Set(specs.map((spec) => String(spec.tbl)))].map((table) =>
|
|
2646
|
+
this.table(table),
|
|
2647
|
+
),
|
|
2648
|
+
),
|
|
2649
|
+
partition,
|
|
2650
|
+
{
|
|
2651
|
+
tbl: query.table,
|
|
2652
|
+
var: firstVariable,
|
|
2653
|
+
vals: firstValues,
|
|
2654
|
+
after: afterRowId,
|
|
2655
|
+
lim: batchSize,
|
|
2656
|
+
},
|
|
2657
|
+
);
|
|
2474
2658
|
if (records.length === 0) break;
|
|
2475
2659
|
for (const record of records) {
|
|
2476
2660
|
afterRowId = record.row_id;
|
package/src/pull.ts
CHANGED
|
@@ -484,8 +484,8 @@ async function* bootstrapSegments(
|
|
|
484
484
|
/**
|
|
485
485
|
* The storage reads a section needs before its first frame. A pull starts
|
|
486
486
|
* them for every subscription before awaiting any, so a storage that batches
|
|
487
|
-
* concurrent page reads (PostgreSQL, D1) answers
|
|
488
|
-
*
|
|
487
|
+
* concurrent page reads (PostgreSQL, D1) answers the first pages of every
|
|
488
|
+
* subscription in one statement, and the horizon re-check runs once per pull.
|
|
489
489
|
*/
|
|
490
490
|
export interface PreparedSection {
|
|
491
491
|
/** §4.5 window read for an incremental cursor; empty when none was read. */
|
|
@@ -499,7 +499,11 @@ export interface PreparedSection {
|
|
|
499
499
|
readonly firstPage: StoredRow[] | undefined;
|
|
500
500
|
/** §4.7 pin of a fresh bootstrap; absent ⇒ `maxCommitSeq`. */
|
|
501
501
|
readonly pin: number | undefined;
|
|
502
|
-
/**
|
|
502
|
+
/**
|
|
503
|
+
* §5.3 stored image for a sqlite-lane bootstrap's reuse key. Read only when
|
|
504
|
+
* the eligibility probe shows a snapshot larger than one rows page, since a
|
|
505
|
+
* smaller table can never match a stored image.
|
|
506
|
+
*/
|
|
503
507
|
readonly image: SegmentRecord | undefined;
|
|
504
508
|
}
|
|
505
509
|
|
|
@@ -574,21 +578,30 @@ export async function prepareSections(
|
|
|
574
578
|
const pin = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
575
579
|
return { asOf: pin, pin };
|
|
576
580
|
});
|
|
577
|
-
const lane = pinned.then(async ({ asOf, pin }) =>
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
581
|
+
const lane = pinned.then(async ({ asOf, pin }) => {
|
|
582
|
+
// §5.3 eligibility: a stored image only matches a snapshot larger than
|
|
583
|
+
// one rows page, so the probe decides before the reuse lookup; a remote
|
|
584
|
+
// `find` on a small table would be a wasted round trip.
|
|
585
|
+
const probe = await firstPage;
|
|
586
|
+
if (probe.length <= limits.limitSnapshotRows) {
|
|
587
|
+
return { pin, image: undefined };
|
|
588
|
+
}
|
|
589
|
+
return {
|
|
590
|
+
pin,
|
|
591
|
+
image: await ctx.segments.find(
|
|
592
|
+
{
|
|
593
|
+
partition: ctx.partition,
|
|
594
|
+
logEpoch,
|
|
595
|
+
table: plan.table.name,
|
|
596
|
+
schemaVersion: schema.version,
|
|
597
|
+
mediaType: 'sqlite',
|
|
598
|
+
scopeDigest: await scopeDigest(plan.effective),
|
|
599
|
+
asOfCommitSeq: asOf,
|
|
600
|
+
},
|
|
601
|
+
clockOf(ctx)(),
|
|
602
|
+
),
|
|
603
|
+
};
|
|
604
|
+
});
|
|
592
605
|
return { firstPage, lane };
|
|
593
606
|
});
|
|
594
607
|
const settled = await Promise.all(
|