@syncular/server 0.38.0 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/content-encoding.d.ts +24 -23
- package/dist/content-encoding.js +77 -22
- package/dist/d1-storage.js +1 -1
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +210 -64
- package/dist/pull.d.ts +7 -3
- package/dist/pull.js +21 -12
- package/dist/relational-rows.d.ts +16 -10
- package/dist/relational-rows.js +48 -42
- package/dist/sqlite-storage.js +16 -3
- package/package.json +2 -2
- package/src/content-encoding.ts +99 -31
- package/src/d1-storage.ts +1 -1
- package/src/postgres-storage.ts +280 -93
- package/src/pull.ts +31 -18
- package/src/relational-rows.ts +53 -47
- package/src/sqlite-storage.ts +20 -3
package/src/postgres-storage.ts
CHANGED
|
@@ -67,9 +67,10 @@ import {
|
|
|
67
67
|
migratePayload,
|
|
68
68
|
parseLayouts,
|
|
69
69
|
quoteIdent,
|
|
70
|
+
tableColumnNames,
|
|
70
71
|
retiredTableNames,
|
|
71
72
|
rewritePlan,
|
|
72
|
-
|
|
73
|
+
columnSqlType,
|
|
73
74
|
rewriteValues,
|
|
74
75
|
SCHEMA_META_DDL_POSTGRES,
|
|
75
76
|
SYNC_COLUMN_VERSIONS_COLUMN,
|
|
@@ -153,7 +154,7 @@ import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query';
|
|
|
153
154
|
* their table.
|
|
154
155
|
*
|
|
155
156
|
* `sync_row_scopes_by_row (partition, tbl, row_id)` is the one secondary
|
|
156
|
-
* scope index: `
|
|
157
|
+
* scope index: `writeRowsOn` and `deleteRow` replace a row's scope entries
|
|
157
158
|
* with a `(partition, tbl, row_id)` predicate, and `row_id` is the LAST
|
|
158
159
|
* column of the inverted PRIMARY KEY, so without this index that predicate
|
|
159
160
|
* scans the table's whole scope range on every row written (a bulk import
|
|
@@ -645,8 +646,30 @@ async function rewriteRowsOn(
|
|
|
645
646
|
oldLayout: readonly StoredColumnLayout[] | undefined,
|
|
646
647
|
): Promise<void> {
|
|
647
648
|
const select = selectRowsForRewriteSql(table, 'postgres');
|
|
648
|
-
|
|
649
|
-
|
|
649
|
+
// One UPDATE per page: behind a network hop (Hyperdrive, a managed
|
|
650
|
+
// database) every statement is a round trip, and the bump holds the
|
|
651
|
+
// migration lock for its whole duration. The page's rows bind as one
|
|
652
|
+
// VALUES list with typed placeholders (`rewriteValues` order: projection
|
|
653
|
+
// columns, payload, partition, row id).
|
|
654
|
+
const types = [
|
|
655
|
+
...(table.materialize
|
|
656
|
+
? table.columns.map((column) => columnSqlType(column, 'postgres'))
|
|
657
|
+
: []),
|
|
658
|
+
'BYTEA',
|
|
659
|
+
'TEXT',
|
|
660
|
+
'TEXT',
|
|
661
|
+
];
|
|
662
|
+
const projected = table.materialize ? table.columns : [];
|
|
663
|
+
const sets = [
|
|
664
|
+
...projected.map((column, i) => `${quoteIdent(column.name)}=v.c${i}`),
|
|
665
|
+
`${quoteIdent(SYNC_PAYLOAD_COLUMN)}=v.c${projected.length}`,
|
|
666
|
+
].join(', ');
|
|
667
|
+
const aliases = types.map((_, i) => `c${i}`).join(', ');
|
|
668
|
+
const partitionAlias = `c${projected.length + 1}`;
|
|
669
|
+
const rowIdAlias = `c${projected.length + 2}`;
|
|
670
|
+
// A page binds types.length parameters per row; PGlite corrupts its state
|
|
671
|
+
// above 32,767 bound parameters (see PG_CHANGES_PER_STATEMENT).
|
|
672
|
+
const batch = Math.max(1, Math.min(500, Math.floor(30_000 / types.length)));
|
|
650
673
|
let afterPartition = '';
|
|
651
674
|
let afterRowId = '';
|
|
652
675
|
for (;;) {
|
|
@@ -654,21 +677,30 @@ async function rewriteRowsOn(
|
|
|
654
677
|
partition: string;
|
|
655
678
|
row_id: string;
|
|
656
679
|
payload: unknown;
|
|
657
|
-
}>(select, [afterPartition, afterRowId,
|
|
680
|
+
}>(select, [afterPartition, afterRowId, batch]);
|
|
658
681
|
if (rows.length === 0) break;
|
|
659
|
-
|
|
682
|
+
const params: unknown[] = [];
|
|
683
|
+
const values = rows.map((row) => {
|
|
660
684
|
const bytes = asBytes(row.payload);
|
|
661
685
|
const payload =
|
|
662
686
|
oldLayout !== undefined
|
|
663
687
|
? migratePayload(oldLayout, table, bytes)
|
|
664
688
|
: bytes;
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
rewriteValues(table, row.partition, row.row_id, payload, 'postgres'),
|
|
689
|
+
const base = params.length;
|
|
690
|
+
params.push(
|
|
691
|
+
...rewriteValues(table, row.partition, row.row_id, payload, 'postgres'),
|
|
668
692
|
);
|
|
669
|
-
|
|
693
|
+
return `(${types.map((type, i) => `$${base + i + 1}::${type}`).join(',')})`;
|
|
694
|
+
});
|
|
695
|
+
await q.query(
|
|
696
|
+
`UPDATE ${quoteIdent(table.name)} AS t SET ${sets}
|
|
697
|
+
FROM (VALUES ${values.join(',')}) AS v(${aliases})
|
|
698
|
+
WHERE t.${quoteIdent(SYNC_PARTITION_COLUMN)}=v.${partitionAlias}
|
|
699
|
+
AND t.${quoteIdent(SYNC_ROW_ID_COLUMN)}=v.${rowIdAlias}`,
|
|
700
|
+
params,
|
|
701
|
+
);
|
|
670
702
|
const last = rows[rows.length - 1];
|
|
671
|
-
if (last === undefined || rows.length <
|
|
703
|
+
if (last === undefined || rows.length < batch) break;
|
|
672
704
|
afterPartition = last.partition;
|
|
673
705
|
afterRowId = last.row_id;
|
|
674
706
|
}
|
|
@@ -709,37 +741,70 @@ async function scanRowsByIndexOn(
|
|
|
709
741
|
return rows.map(toStoredRow);
|
|
710
742
|
}
|
|
711
743
|
|
|
712
|
-
|
|
744
|
+
/**
|
|
745
|
+
* Write rows of one table in one statement per chunk: the upsert, and the
|
|
746
|
+
* scope index synced to each row's scope map (entries outside it deleted,
|
|
747
|
+
* missing ones inserted). Row ids are distinct, so no index entry is touched
|
|
748
|
+
* twice and the writable CTEs cannot conflict. The scopes bind as text
|
|
749
|
+
* before the JSONB cast (see `appendCommit`). Every chunk is handed to `q`
|
|
750
|
+
* before the first one settles, so a statement issued after this call
|
|
751
|
+
* queues behind all of them on the transaction FIFO.
|
|
752
|
+
*/
|
|
753
|
+
function writeRowsOn(
|
|
713
754
|
q: PgQueryable,
|
|
714
755
|
compiled: CompiledTable,
|
|
715
756
|
partition: string,
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
const
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
757
|
+
rows: readonly {
|
|
758
|
+
readonly row: StoredRow;
|
|
759
|
+
/** `upsertValues` of the row, computed when it was staged. */
|
|
760
|
+
readonly values: readonly unknown[];
|
|
761
|
+
}[],
|
|
762
|
+
): Promise<unknown> {
|
|
763
|
+
const width = tableColumnNames(compiled).length;
|
|
764
|
+
// PGlite corrupts its state above 32,767 bound parameters (see
|
|
765
|
+
// PG_CHANGES_PER_STATEMENT).
|
|
766
|
+
const chunk = Math.max(1, Math.floor(30_000 / width));
|
|
767
|
+
const writes: Promise<unknown>[] = [];
|
|
768
|
+
for (let first = 0; first < rows.length; first += chunk) {
|
|
769
|
+
const slice = rows.slice(first, first + chunk);
|
|
770
|
+
const params = slice.flatMap((entry) => entry.values);
|
|
771
|
+
const n = params.length;
|
|
772
|
+
writes.push(
|
|
773
|
+
q.query(
|
|
774
|
+
`WITH upserted AS (${upsertSql(compiled, 'postgres', slice.length)}),
|
|
775
|
+
batch AS (
|
|
776
|
+
SELECT row_id, scopes
|
|
777
|
+
FROM jsonb_to_recordset($${n + 1}::text::jsonb) AS r(row_id text, scopes jsonb)
|
|
778
|
+
),
|
|
779
|
+
scope AS (
|
|
780
|
+
SELECT batch.row_id, entry.key, entry.value
|
|
781
|
+
FROM batch CROSS JOIN LATERAL jsonb_each_text(batch.scopes) AS entry
|
|
782
|
+
),
|
|
783
|
+
removed AS (
|
|
784
|
+
DELETE FROM sync_row_scopes existing
|
|
785
|
+
USING batch
|
|
786
|
+
WHERE existing.partition=$${n + 2} AND existing.tbl=$${n + 3}
|
|
787
|
+
AND existing.row_id=batch.row_id
|
|
788
|
+
AND NOT EXISTS (
|
|
789
|
+
SELECT 1 FROM scope
|
|
790
|
+
WHERE scope.row_id=existing.row_id
|
|
791
|
+
AND scope.key=existing.var AND scope.value=existing.value)
|
|
792
|
+
)
|
|
793
|
+
INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
794
|
+
SELECT $${n + 2}, $${n + 3}, key, value, row_id FROM scope
|
|
795
|
+
ON CONFLICT DO NOTHING`,
|
|
796
|
+
[
|
|
797
|
+
...params,
|
|
798
|
+
JSON.stringify(
|
|
799
|
+
slice.map(({ row }) => ({ row_id: row.rowId, scopes: row.scopes })),
|
|
800
|
+
),
|
|
801
|
+
partition,
|
|
802
|
+
compiled.name,
|
|
803
|
+
],
|
|
804
|
+
),
|
|
805
|
+
);
|
|
806
|
+
}
|
|
807
|
+
return Promise.all(writes);
|
|
743
808
|
}
|
|
744
809
|
|
|
745
810
|
async function getPushResultOn(
|
|
@@ -788,6 +853,8 @@ async function queryAuthoritativeOn(
|
|
|
788
853
|
|
|
789
854
|
class PostgresTransaction implements StorageTransaction {
|
|
790
855
|
#client: PgQueryable;
|
|
856
|
+
/** The FIFO without the staged-write prelude (see the constructor). */
|
|
857
|
+
#serial: PgQueryable;
|
|
791
858
|
#partition: string;
|
|
792
859
|
/** Storage-owned relation rewrite, bound to this transaction's partition. */
|
|
793
860
|
readonly #prepareQuery: (
|
|
@@ -805,6 +872,25 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
805
872
|
*/
|
|
806
873
|
readonly #prefetchedRows = new Map<string, StoredRow | null>();
|
|
807
874
|
readonly #prefetchedTombstones = new Map<string, number | null>();
|
|
875
|
+
/**
|
|
876
|
+
* Upserts staged per table (row id → row and its push operation index).
|
|
877
|
+
* Every statement of this transaction writes them first (see the
|
|
878
|
+
* constructor), so reads and later writes observe them; a commit's rows
|
|
879
|
+
* reach the database as one statement per table instead of one per row.
|
|
880
|
+
* Tables with a unique index write immediately: a violation must name its
|
|
881
|
+
* operation.
|
|
882
|
+
*/
|
|
883
|
+
readonly #staged = new Map<
|
|
884
|
+
CompiledTable,
|
|
885
|
+
Map<
|
|
886
|
+
string,
|
|
887
|
+
{
|
|
888
|
+
readonly row: StoredRow;
|
|
889
|
+
readonly values: readonly unknown[];
|
|
890
|
+
readonly opIndex?: number;
|
|
891
|
+
}
|
|
892
|
+
>
|
|
893
|
+
>();
|
|
808
894
|
/** Resolves/rejects the `transaction(fn)` wrapper (see `begin`). */
|
|
809
895
|
#resolve: () => void;
|
|
810
896
|
#reject: (error: unknown) => void;
|
|
@@ -825,7 +911,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
825
911
|
// issued through this transaction is chained on one FIFO — the same
|
|
826
912
|
// shape as `SqliteServerStorage.queryAuthoritative`.
|
|
827
913
|
let tail: Promise<void> = Promise.resolve();
|
|
828
|
-
|
|
914
|
+
const serial: PgQueryable = {
|
|
829
915
|
query: <Row>(text: string, params?: readonly unknown[]) => {
|
|
830
916
|
const run = tail.then(() => client.query<Row>(text, params));
|
|
831
917
|
tail = run.then(
|
|
@@ -835,6 +921,23 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
835
921
|
return run;
|
|
836
922
|
},
|
|
837
923
|
};
|
|
924
|
+
this.#serial = serial;
|
|
925
|
+
this.#client = {
|
|
926
|
+
query: async <Row>(text: string, params?: readonly unknown[]) => {
|
|
927
|
+
// Queue the staged writes ahead of this statement synchronously, so
|
|
928
|
+
// a concurrently issued statement cannot overtake them.
|
|
929
|
+
const staged = this.#writeStaged();
|
|
930
|
+
const run = serial.query<Row>(text, params);
|
|
931
|
+
try {
|
|
932
|
+
await staged;
|
|
933
|
+
} catch (error) {
|
|
934
|
+
// The aborted transaction fails the queued statement too.
|
|
935
|
+
run.catch(() => undefined);
|
|
936
|
+
throw error;
|
|
937
|
+
}
|
|
938
|
+
return run;
|
|
939
|
+
},
|
|
940
|
+
};
|
|
838
941
|
this.#partition = partition;
|
|
839
942
|
this.#writerVersion = writerVersion;
|
|
840
943
|
this.#resolveTable = resolveTable;
|
|
@@ -847,6 +950,27 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
847
950
|
if (!this.#open) throw new Error('transaction already finished');
|
|
848
951
|
}
|
|
849
952
|
|
|
953
|
+
#writeStaged(): Promise<unknown> {
|
|
954
|
+
if (this.#staged.size === 0) return Promise.resolve();
|
|
955
|
+
const staged = [...this.#staged];
|
|
956
|
+
this.#staged.clear();
|
|
957
|
+
return Promise.all(
|
|
958
|
+
staged.map(([table, rows]) => {
|
|
959
|
+
const entries = [...rows.values()];
|
|
960
|
+
return writeRowsOn(this.#serial, table, this.#partition, entries).catch(
|
|
961
|
+
(error: unknown) => {
|
|
962
|
+
// Staged tables have no unique index, so a violation is a column
|
|
963
|
+
// constraint; it names the first staged operation of the table.
|
|
964
|
+
if (isPostgresConstraintError(error)) {
|
|
965
|
+
throw new StorageConstraintError(error, entries[0]?.opIndex);
|
|
966
|
+
}
|
|
967
|
+
throw error;
|
|
968
|
+
},
|
|
969
|
+
);
|
|
970
|
+
}),
|
|
971
|
+
);
|
|
972
|
+
}
|
|
973
|
+
|
|
850
974
|
getRow(table: string, rowId: string): Promise<StoredRow | undefined> {
|
|
851
975
|
this.#assertOpen();
|
|
852
976
|
const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
|
|
@@ -1039,6 +1163,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1039
1163
|
if (!this.#pushApplySavepoint) {
|
|
1040
1164
|
throw new Error('push rejection requires its apply savepoint');
|
|
1041
1165
|
}
|
|
1166
|
+
this.#staged.clear();
|
|
1042
1167
|
await this.#client.query('ROLLBACK TO SAVEPOINT syncular_push_candidate');
|
|
1043
1168
|
await this.#client.query('RELEASE SAVEPOINT syncular_push_candidate');
|
|
1044
1169
|
this.#pushApplySavepoint = false;
|
|
@@ -1053,13 +1178,28 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1053
1178
|
): Promise<void> {
|
|
1054
1179
|
this.#assertOpen();
|
|
1055
1180
|
this.#prefetchedRows.delete(`${table}\u0000${row.rowId}`);
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1181
|
+
const compiled = this.#resolveTable(table);
|
|
1182
|
+
// Decodes the payload: an undecodable row fails here, at its operation.
|
|
1183
|
+
const values = upsertValues(compiled, this.#partition, row, 'postgres');
|
|
1184
|
+
if (!compiled.indexes.some((index) => index.unique === true)) {
|
|
1185
|
+
let rows = this.#staged.get(compiled);
|
|
1186
|
+
if (rows === undefined) {
|
|
1187
|
+
rows = new Map();
|
|
1188
|
+
this.#staged.set(compiled, rows);
|
|
1189
|
+
}
|
|
1190
|
+
// A later upsert of the same row in this transaction replaces it.
|
|
1191
|
+
rows.delete(row.rowId);
|
|
1192
|
+
rows.set(row.rowId, {
|
|
1061
1193
|
row,
|
|
1062
|
-
|
|
1194
|
+
values,
|
|
1195
|
+
...(context === undefined ? {} : { opIndex: context.opIndex }),
|
|
1196
|
+
});
|
|
1197
|
+
return;
|
|
1198
|
+
}
|
|
1199
|
+
try {
|
|
1200
|
+
await writeRowsOn(this.#client, compiled, this.#partition, [
|
|
1201
|
+
{ row, values },
|
|
1202
|
+
]);
|
|
1063
1203
|
} catch (error) {
|
|
1064
1204
|
if (isPostgresConstraintError(error)) {
|
|
1065
1205
|
throw new StorageConstraintError(error, context?.opIndex);
|
|
@@ -1232,6 +1372,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1232
1372
|
|
|
1233
1373
|
async commit(): Promise<void> {
|
|
1234
1374
|
this.#assertOpen();
|
|
1375
|
+
await this.#writeStaged();
|
|
1235
1376
|
this.#open = false;
|
|
1236
1377
|
// Signal the transaction wrapper to COMMIT; wait for it to actually land.
|
|
1237
1378
|
this.#resolve();
|
|
@@ -1240,6 +1381,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1240
1381
|
|
|
1241
1382
|
async rollback(): Promise<void> {
|
|
1242
1383
|
if (!this.#open) return;
|
|
1384
|
+
this.#staged.clear();
|
|
1243
1385
|
this.#open = false;
|
|
1244
1386
|
this.#reject(new RollbackSignal());
|
|
1245
1387
|
// Swallow — a rollback is a normal outcome, not an error to the caller.
|
|
@@ -1280,11 +1422,11 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1280
1422
|
#schemaVersion: number | undefined;
|
|
1281
1423
|
/**
|
|
1282
1424
|
* Page reads waiting for the end of the current microtask turn, keyed by
|
|
1283
|
-
*
|
|
1425
|
+
* batch stream and partition. A pull starts the window read (or first
|
|
1284
1426
|
* snapshot page) of every subscription before awaiting any of them, so the
|
|
1285
|
-
*
|
|
1286
|
-
*
|
|
1287
|
-
*
|
|
1427
|
+
* reads of all tables leave as ONE set-based statement per stream instead
|
|
1428
|
+
* of one round trip per subscription or table
|
|
1429
|
+
* (SYNCULAR-PULL-ROUNDTRIPS-001).
|
|
1288
1430
|
*/
|
|
1289
1431
|
readonly #pageBatches = new Map<string, PendingPage[]>();
|
|
1290
1432
|
|
|
@@ -1293,37 +1435,42 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1293
1435
|
}
|
|
1294
1436
|
|
|
1295
1437
|
/**
|
|
1296
|
-
* Queue one page read into the batch for (`
|
|
1297
|
-
*
|
|
1298
|
-
*
|
|
1299
|
-
*
|
|
1438
|
+
* Queue one page read into the batch for (`key`, `partition`). `sql` is the
|
|
1439
|
+
* statement every batch of that stream shares, or a builder over the queued
|
|
1440
|
+
* specs when the statement names what the batch contains (the snapshot scan
|
|
1441
|
+
* joins one branch per table present). The batch runs as a single statement
|
|
1442
|
+
* with the queued specs bound as a JSON array (`$2`); each spec gets its
|
|
1443
|
+
* position as `i`, and the result rows are split back by their `sub`
|
|
1444
|
+
* column.
|
|
1300
1445
|
*/
|
|
1301
1446
|
#batchedPage<Row>(
|
|
1302
|
-
|
|
1447
|
+
key: string,
|
|
1448
|
+
sql: string | ((specs: readonly Record<string, unknown>[]) => string),
|
|
1303
1449
|
partition: string,
|
|
1304
1450
|
spec: Record<string, unknown>,
|
|
1305
1451
|
): Promise<Row[]> {
|
|
1306
|
-
const
|
|
1307
|
-
let batch = this.#pageBatches.get(
|
|
1452
|
+
const batchKey = `${key}\u0000${partition}`;
|
|
1453
|
+
let batch = this.#pageBatches.get(batchKey);
|
|
1308
1454
|
if (batch === undefined) {
|
|
1309
1455
|
const queued: PendingPage[] = [];
|
|
1310
1456
|
batch = queued;
|
|
1311
|
-
this.#pageBatches.set(
|
|
1457
|
+
this.#pageBatches.set(batchKey, queued);
|
|
1312
1458
|
queueMicrotask(() => {
|
|
1313
|
-
this.#pageBatches.delete(
|
|
1314
|
-
const specs =
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
(
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1459
|
+
this.#pageBatches.delete(batchKey);
|
|
1460
|
+
const specs = queued.map((entry, i) => ({ ...entry.spec, i }));
|
|
1461
|
+
const statement = typeof sql === 'string' ? sql : sql(specs);
|
|
1462
|
+
this.#exec
|
|
1463
|
+
.query<PgRow>(statement, [partition, JSON.stringify(specs)])
|
|
1464
|
+
.then(
|
|
1465
|
+
({ rows }) => {
|
|
1466
|
+
const grouped = queued.map((): PgRow[] => []);
|
|
1467
|
+
for (const row of rows) grouped[asNumber(row.sub)]?.push(row);
|
|
1468
|
+
queued.forEach((entry, i) => entry.resolve(grouped[i] ?? []));
|
|
1469
|
+
},
|
|
1470
|
+
(error: unknown) => {
|
|
1471
|
+
for (const entry of queued) entry.reject(error);
|
|
1472
|
+
},
|
|
1473
|
+
);
|
|
1327
1474
|
});
|
|
1328
1475
|
}
|
|
1329
1476
|
const target = batch;
|
|
@@ -1538,10 +1685,6 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1538
1685
|
// backfill for flipped-on materialization) — all inside one
|
|
1539
1686
|
// transaction (Postgres DDL is transactional — a failed bump leaves
|
|
1540
1687
|
// no half-state).
|
|
1541
|
-
const layouts = parseLayouts(
|
|
1542
|
-
typeof marker.layouts === 'string' ? marker.layouts : undefined,
|
|
1543
|
-
);
|
|
1544
|
-
const retiredTables = retiredTableNames(schema, layouts);
|
|
1545
1688
|
await this.#exec.transaction(async (client) => {
|
|
1546
1689
|
// RFC 0007 migration barrier: `LOCK TABLE ... EXCLUSIVE` is the FIRST
|
|
1547
1690
|
// statement, before any read, DDL, rewrite, marker or checkpoint
|
|
@@ -1554,6 +1697,31 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1554
1697
|
// when it takes this lock, so a writer already inside completes while
|
|
1555
1698
|
// the migration waits.
|
|
1556
1699
|
await client.query('LOCK TABLE sync_partitions IN EXCLUSIVE MODE');
|
|
1700
|
+
// The probe ran before the lock. A concurrent `ensureSchema` (another
|
|
1701
|
+
// isolate, the Durable Object's first request) may have bumped while
|
|
1702
|
+
// this one waited; re-running the rewrite would decode its new-layout
|
|
1703
|
+
// payloads with the old layout. The locked marker decides.
|
|
1704
|
+
const { rows: lockedMarker } = await client.query<{
|
|
1705
|
+
schema_version: unknown;
|
|
1706
|
+
layouts: unknown;
|
|
1707
|
+
}>('SELECT schema_version, layouts FROM sync_schema_meta WHERE id=1');
|
|
1708
|
+
const lockedVersion =
|
|
1709
|
+
lockedMarker[0]?.schema_version === null ||
|
|
1710
|
+
lockedMarker[0]?.schema_version === undefined
|
|
1711
|
+
? undefined
|
|
1712
|
+
: asNumber(lockedMarker[0].schema_version);
|
|
1713
|
+
if (lockedVersion !== undefined && lockedVersion > schema.version) {
|
|
1714
|
+
throw new Error(
|
|
1715
|
+
`stored schema version ${lockedVersion} is newer than the configured schema (${schema.version}) — refusing to run an older server against a migrated database`,
|
|
1716
|
+
);
|
|
1717
|
+
}
|
|
1718
|
+
if (lockedVersion === schema.version) return;
|
|
1719
|
+
const layouts = parseLayouts(
|
|
1720
|
+
typeof lockedMarker[0]?.layouts === 'string'
|
|
1721
|
+
? lockedMarker[0].layouts
|
|
1722
|
+
: undefined,
|
|
1723
|
+
);
|
|
1724
|
+
const retiredTables = retiredTableNames(schema, layouts);
|
|
1557
1725
|
const existing = new Map<string, ReadonlySet<string>>();
|
|
1558
1726
|
const existingIndexes = new Map<string, ReadonlyMap<string, string>>();
|
|
1559
1727
|
for (const table of schema.tables.values()) {
|
|
@@ -1620,13 +1788,21 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1620
1788
|
);
|
|
1621
1789
|
}
|
|
1622
1790
|
}
|
|
1791
|
+
if (retiredTables.length > 0) {
|
|
1792
|
+
// Neither index leads with `tbl`: one scan per table, not one per
|
|
1793
|
+
// retired table.
|
|
1794
|
+
await client.query(
|
|
1795
|
+
`DELETE FROM sync_row_scopes
|
|
1796
|
+
WHERE tbl IN (SELECT jsonb_array_elements_text($1::text::jsonb))`,
|
|
1797
|
+
[JSON.stringify(retiredTables)],
|
|
1798
|
+
);
|
|
1799
|
+
await client.query(
|
|
1800
|
+
`DELETE FROM sync_blob_refs
|
|
1801
|
+
WHERE tbl IN (SELECT jsonb_array_elements_text($1::text::jsonb))`,
|
|
1802
|
+
[JSON.stringify(retiredTables)],
|
|
1803
|
+
);
|
|
1804
|
+
}
|
|
1623
1805
|
for (const tableName of retiredTables) {
|
|
1624
|
-
await client.query('DELETE FROM sync_row_scopes WHERE tbl=$1', [
|
|
1625
|
-
tableName,
|
|
1626
|
-
]);
|
|
1627
|
-
await client.query('DELETE FROM sync_blob_refs WHERE tbl=$1', [
|
|
1628
|
-
tableName,
|
|
1629
|
-
]);
|
|
1630
1806
|
await client.query(dropTableDdl(tableName));
|
|
1631
1807
|
}
|
|
1632
1808
|
for (const statement of schemaDdl(
|
|
@@ -2359,6 +2535,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2359
2535
|
if (firstVariable === undefined || values.length === 0) return 0;
|
|
2360
2536
|
// Concurrent reads of a pull share one statement across tables.
|
|
2361
2537
|
const [row] = await this.#batchedPage<{ seq: unknown }>(
|
|
2538
|
+
'latestScopedChange',
|
|
2362
2539
|
LATEST_SCOPED_CHANGE_BATCH_SQL,
|
|
2363
2540
|
partition,
|
|
2364
2541
|
{ tbl: query.table, var: firstVariable, vals: values },
|
|
@@ -2388,6 +2565,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2388
2565
|
const batchSize = Math.max(64, query.limitChanges);
|
|
2389
2566
|
while (deliveredChanges < query.limitChanges) {
|
|
2390
2567
|
const records = await this.#batchedPage<CommitWindowRecord>(
|
|
2568
|
+
'commitWindow',
|
|
2391
2569
|
COMMIT_WINDOW_BATCH_SQL,
|
|
2392
2570
|
partition,
|
|
2393
2571
|
{
|
|
@@ -2456,21 +2634,30 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2456
2634
|
if (firstValues.length === 0) return [];
|
|
2457
2635
|
// Candidates via the inverted index (ordered + LIMITed at the covering
|
|
2458
2636
|
// PK) LEFT JOINed to the row table — one round trip per page, never one
|
|
2459
|
-
// per row, and
|
|
2460
|
-
//
|
|
2461
|
-
//
|
|
2462
|
-
const sql = scanRowBatchSql(this.table(query.table));
|
|
2637
|
+
// per row, and concurrent scans across tables share one statement (see
|
|
2638
|
+
// `scanRowBatchSql`). Exact multi-variable verification against the
|
|
2639
|
+
// stored scope map happens below.
|
|
2463
2640
|
const rows: StoredRow[] = [];
|
|
2464
2641
|
let afterRowId = query.afterRowId ?? '';
|
|
2465
2642
|
const batchSize = Math.max(64, query.limit);
|
|
2466
2643
|
while (rows.length < query.limit) {
|
|
2467
|
-
const records = await this.#batchedPage<RowRecord>(
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2644
|
+
const records = await this.#batchedPage<RowRecord>(
|
|
2645
|
+
'scanRows',
|
|
2646
|
+
(specs) =>
|
|
2647
|
+
scanRowBatchSql(
|
|
2648
|
+
[...new Set(specs.map((spec) => String(spec.tbl)))].map((table) =>
|
|
2649
|
+
this.table(table),
|
|
2650
|
+
),
|
|
2651
|
+
),
|
|
2652
|
+
partition,
|
|
2653
|
+
{
|
|
2654
|
+
tbl: query.table,
|
|
2655
|
+
var: firstVariable,
|
|
2656
|
+
vals: firstValues,
|
|
2657
|
+
after: afterRowId,
|
|
2658
|
+
lim: batchSize,
|
|
2659
|
+
},
|
|
2660
|
+
);
|
|
2474
2661
|
if (records.length === 0) break;
|
|
2475
2662
|
for (const record of records) {
|
|
2476
2663
|
afterRowId = record.row_id;
|
package/src/pull.ts
CHANGED
|
@@ -484,8 +484,8 @@ async function* bootstrapSegments(
|
|
|
484
484
|
/**
|
|
485
485
|
* The storage reads a section needs before its first frame. A pull starts
|
|
486
486
|
* them for every subscription before awaiting any, so a storage that batches
|
|
487
|
-
* concurrent page reads (PostgreSQL, D1) answers
|
|
488
|
-
*
|
|
487
|
+
* concurrent page reads (PostgreSQL, D1) answers the first pages of every
|
|
488
|
+
* subscription in one statement, and the horizon re-check runs once per pull.
|
|
489
489
|
*/
|
|
490
490
|
export interface PreparedSection {
|
|
491
491
|
/** §4.5 window read for an incremental cursor; empty when none was read. */
|
|
@@ -499,7 +499,11 @@ export interface PreparedSection {
|
|
|
499
499
|
readonly firstPage: StoredRow[] | undefined;
|
|
500
500
|
/** §4.7 pin of a fresh bootstrap; absent ⇒ `maxCommitSeq`. */
|
|
501
501
|
readonly pin: number | undefined;
|
|
502
|
-
/**
|
|
502
|
+
/**
|
|
503
|
+
* §5.3 stored image for a sqlite-lane bootstrap's reuse key. Read only when
|
|
504
|
+
* the eligibility probe shows a snapshot larger than one rows page, since a
|
|
505
|
+
* smaller table can never match a stored image.
|
|
506
|
+
*/
|
|
503
507
|
readonly image: SegmentRecord | undefined;
|
|
504
508
|
}
|
|
505
509
|
|
|
@@ -574,21 +578,30 @@ export async function prepareSections(
|
|
|
574
578
|
const pin = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
575
579
|
return { asOf: pin, pin };
|
|
576
580
|
});
|
|
577
|
-
const lane = pinned.then(async ({ asOf, pin }) =>
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
581
|
+
const lane = pinned.then(async ({ asOf, pin }) => {
|
|
582
|
+
// §5.3 eligibility: a stored image only matches a snapshot larger than
|
|
583
|
+
// one rows page, so the probe decides before the reuse lookup; a remote
|
|
584
|
+
// `find` on a small table would be a wasted round trip.
|
|
585
|
+
const probe = await firstPage;
|
|
586
|
+
if (probe.length <= limits.limitSnapshotRows) {
|
|
587
|
+
return { pin, image: undefined };
|
|
588
|
+
}
|
|
589
|
+
return {
|
|
590
|
+
pin,
|
|
591
|
+
image: await ctx.segments.find(
|
|
592
|
+
{
|
|
593
|
+
partition: ctx.partition,
|
|
594
|
+
logEpoch,
|
|
595
|
+
table: plan.table.name,
|
|
596
|
+
schemaVersion: schema.version,
|
|
597
|
+
mediaType: 'sqlite',
|
|
598
|
+
scopeDigest: await scopeDigest(plan.effective),
|
|
599
|
+
asOfCommitSeq: asOf,
|
|
600
|
+
},
|
|
601
|
+
clockOf(ctx)(),
|
|
602
|
+
),
|
|
603
|
+
};
|
|
604
|
+
});
|
|
592
605
|
return { firstPage, lane };
|
|
593
606
|
});
|
|
594
607
|
const settled = await Promise.all(
|