@syncular/server 0.37.0 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/content-encoding.d.ts +24 -23
- package/dist/content-encoding.js +77 -22
- package/dist/d1-storage.js +24 -19
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +240 -74
- package/dist/pull.d.ts +7 -3
- package/dist/pull.js +21 -12
- package/dist/relational-rows.d.ts +49 -23
- package/dist/relational-rows.js +133 -59
- package/dist/schema.d.ts +19 -0
- package/dist/schema.js +22 -0
- package/dist/sqlite-bun-driver.js +1 -1
- package/dist/sqlite-node-driver.d.ts +1 -0
- package/dist/sqlite-node-driver.js +13 -3
- package/dist/sqlite-storage.js +36 -21
- package/dist/storage.d.ts +8 -1
- package/dist/storage.js +3 -0
- package/package.json +2 -2
- package/src/content-encoding.ts +99 -31
- package/src/d1-storage.ts +36 -22
- package/src/postgres-storage.ts +338 -109
- package/src/pull.ts +31 -18
- package/src/relational-rows.ts +182 -66
- package/src/schema.ts +55 -0
- package/src/sqlite-bun-driver.ts +1 -1
- package/src/sqlite-node-driver.ts +18 -3
- package/src/sqlite-storage.ts +64 -34
- package/src/storage.ts +9 -1
package/src/postgres-storage.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { validateCommitPruneQuery } from './prune';
|
|
2
2
|
import { StorageQueryError } from './storage-errors';
|
|
3
3
|
import type { CommitPruneQuery, CommitPruneResult } from './storage';
|
|
4
|
-
import { serveGateRefusal } from './storage';
|
|
4
|
+
import { PARTITION_TOUCH_INTERVAL_MS, serveGateRefusal } from './storage';
|
|
5
5
|
import { serveNotReadyError } from './readiness';
|
|
6
6
|
/**
|
|
7
7
|
* Postgres server storage: the production database path.
|
|
@@ -67,9 +67,10 @@ import {
|
|
|
67
67
|
migratePayload,
|
|
68
68
|
parseLayouts,
|
|
69
69
|
quoteIdent,
|
|
70
|
+
tableColumnNames,
|
|
70
71
|
retiredTableNames,
|
|
71
72
|
rewritePlan,
|
|
72
|
-
|
|
73
|
+
columnSqlType,
|
|
73
74
|
rewriteValues,
|
|
74
75
|
SCHEMA_META_DDL_POSTGRES,
|
|
75
76
|
SYNC_COLUMN_VERSIONS_COLUMN,
|
|
@@ -81,6 +82,7 @@ import {
|
|
|
81
82
|
type StoredColumnLayout,
|
|
82
83
|
scanRowBatchSql,
|
|
83
84
|
scanRowPageSql,
|
|
85
|
+
indexSignature,
|
|
84
86
|
schemaDdl,
|
|
85
87
|
selectRowScopesSql,
|
|
86
88
|
selectRowSql,
|
|
@@ -152,7 +154,7 @@ import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query';
|
|
|
152
154
|
* their table.
|
|
153
155
|
*
|
|
154
156
|
* `sync_row_scopes_by_row (partition, tbl, row_id)` is the one secondary
|
|
155
|
-
* scope index: `
|
|
157
|
+
* scope index: `writeRowsOn` and `deleteRow` replace a row's scope entries
|
|
156
158
|
* with a `(partition, tbl, row_id)` predicate, and `row_id` is the LAST
|
|
157
159
|
* column of the inverted PRIMARY KEY, so without this index that predicate
|
|
158
160
|
* scans the table's whole scope range on every row written (a bulk import
|
|
@@ -644,8 +646,30 @@ async function rewriteRowsOn(
|
|
|
644
646
|
oldLayout: readonly StoredColumnLayout[] | undefined,
|
|
645
647
|
): Promise<void> {
|
|
646
648
|
const select = selectRowsForRewriteSql(table, 'postgres');
|
|
647
|
-
|
|
648
|
-
|
|
649
|
+
// One UPDATE per page: behind a network hop (Hyperdrive, a managed
|
|
650
|
+
// database) every statement is a round trip, and the bump holds the
|
|
651
|
+
// migration lock for its whole duration. The page's rows bind as one
|
|
652
|
+
// VALUES list with typed placeholders (`rewriteValues` order: projection
|
|
653
|
+
// columns, payload, partition, row id).
|
|
654
|
+
const types = [
|
|
655
|
+
...(table.materialize
|
|
656
|
+
? table.columns.map((column) => columnSqlType(column, 'postgres'))
|
|
657
|
+
: []),
|
|
658
|
+
'BYTEA',
|
|
659
|
+
'TEXT',
|
|
660
|
+
'TEXT',
|
|
661
|
+
];
|
|
662
|
+
const projected = table.materialize ? table.columns : [];
|
|
663
|
+
const sets = [
|
|
664
|
+
...projected.map((column, i) => `${quoteIdent(column.name)}=v.c${i}`),
|
|
665
|
+
`${quoteIdent(SYNC_PAYLOAD_COLUMN)}=v.c${projected.length}`,
|
|
666
|
+
].join(', ');
|
|
667
|
+
const aliases = types.map((_, i) => `c${i}`).join(', ');
|
|
668
|
+
const partitionAlias = `c${projected.length + 1}`;
|
|
669
|
+
const rowIdAlias = `c${projected.length + 2}`;
|
|
670
|
+
// A page binds types.length parameters per row; PGlite corrupts its state
|
|
671
|
+
// above 32,767 bound parameters (see PG_CHANGES_PER_STATEMENT).
|
|
672
|
+
const batch = Math.max(1, Math.min(500, Math.floor(30_000 / types.length)));
|
|
649
673
|
let afterPartition = '';
|
|
650
674
|
let afterRowId = '';
|
|
651
675
|
for (;;) {
|
|
@@ -653,21 +677,30 @@ async function rewriteRowsOn(
|
|
|
653
677
|
partition: string;
|
|
654
678
|
row_id: string;
|
|
655
679
|
payload: unknown;
|
|
656
|
-
}>(select, [afterPartition, afterRowId,
|
|
680
|
+
}>(select, [afterPartition, afterRowId, batch]);
|
|
657
681
|
if (rows.length === 0) break;
|
|
658
|
-
|
|
682
|
+
const params: unknown[] = [];
|
|
683
|
+
const values = rows.map((row) => {
|
|
659
684
|
const bytes = asBytes(row.payload);
|
|
660
685
|
const payload =
|
|
661
686
|
oldLayout !== undefined
|
|
662
687
|
? migratePayload(oldLayout, table, bytes)
|
|
663
688
|
: bytes;
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
rewriteValues(table, row.partition, row.row_id, payload, 'postgres'),
|
|
689
|
+
const base = params.length;
|
|
690
|
+
params.push(
|
|
691
|
+
...rewriteValues(table, row.partition, row.row_id, payload, 'postgres'),
|
|
667
692
|
);
|
|
668
|
-
|
|
693
|
+
return `(${types.map((type, i) => `$${base + i + 1}::${type}`).join(',')})`;
|
|
694
|
+
});
|
|
695
|
+
await q.query(
|
|
696
|
+
`UPDATE ${quoteIdent(table.name)} AS t SET ${sets}
|
|
697
|
+
FROM (VALUES ${values.join(',')}) AS v(${aliases})
|
|
698
|
+
WHERE t.${quoteIdent(SYNC_PARTITION_COLUMN)}=v.${partitionAlias}
|
|
699
|
+
AND t.${quoteIdent(SYNC_ROW_ID_COLUMN)}=v.${rowIdAlias}`,
|
|
700
|
+
params,
|
|
701
|
+
);
|
|
669
702
|
const last = rows[rows.length - 1];
|
|
670
|
-
if (last === undefined || rows.length <
|
|
703
|
+
if (last === undefined || rows.length < batch) break;
|
|
671
704
|
afterPartition = last.partition;
|
|
672
705
|
afterRowId = last.row_id;
|
|
673
706
|
}
|
|
@@ -708,37 +741,70 @@ async function scanRowsByIndexOn(
|
|
|
708
741
|
return rows.map(toStoredRow);
|
|
709
742
|
}
|
|
710
743
|
|
|
711
|
-
|
|
744
|
+
/**
|
|
745
|
+
* Write rows of one table in one statement per chunk: the upsert, and the
|
|
746
|
+
* scope index synced to each row's scope map (entries outside it deleted,
|
|
747
|
+
* missing ones inserted). Row ids are distinct, so no index entry is touched
|
|
748
|
+
* twice and the writable CTEs cannot conflict. The scopes bind as text
|
|
749
|
+
* before the JSONB cast (see `appendCommit`). Every chunk is handed to `q`
|
|
750
|
+
* before the first one settles, so a statement issued after this call
|
|
751
|
+
* queues behind all of them on the transaction FIFO.
|
|
752
|
+
*/
|
|
753
|
+
function writeRowsOn(
|
|
712
754
|
q: PgQueryable,
|
|
713
755
|
compiled: CompiledTable,
|
|
714
756
|
partition: string,
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
const
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
757
|
+
rows: readonly {
|
|
758
|
+
readonly row: StoredRow;
|
|
759
|
+
/** `upsertValues` of the row, computed when it was staged. */
|
|
760
|
+
readonly values: readonly unknown[];
|
|
761
|
+
}[],
|
|
762
|
+
): Promise<unknown> {
|
|
763
|
+
const width = tableColumnNames(compiled).length;
|
|
764
|
+
// PGlite corrupts its state above 32,767 bound parameters (see
|
|
765
|
+
// PG_CHANGES_PER_STATEMENT).
|
|
766
|
+
const chunk = Math.max(1, Math.floor(30_000 / width));
|
|
767
|
+
const writes: Promise<unknown>[] = [];
|
|
768
|
+
for (let first = 0; first < rows.length; first += chunk) {
|
|
769
|
+
const slice = rows.slice(first, first + chunk);
|
|
770
|
+
const params = slice.flatMap((entry) => entry.values);
|
|
771
|
+
const n = params.length;
|
|
772
|
+
writes.push(
|
|
773
|
+
q.query(
|
|
774
|
+
`WITH upserted AS (${upsertSql(compiled, 'postgres', slice.length)}),
|
|
775
|
+
batch AS (
|
|
776
|
+
SELECT row_id, scopes
|
|
777
|
+
FROM jsonb_to_recordset($${n + 1}::text::jsonb) AS r(row_id text, scopes jsonb)
|
|
778
|
+
),
|
|
779
|
+
scope AS (
|
|
780
|
+
SELECT batch.row_id, entry.key, entry.value
|
|
781
|
+
FROM batch CROSS JOIN LATERAL jsonb_each_text(batch.scopes) AS entry
|
|
782
|
+
),
|
|
783
|
+
removed AS (
|
|
784
|
+
DELETE FROM sync_row_scopes existing
|
|
785
|
+
USING batch
|
|
786
|
+
WHERE existing.partition=$${n + 2} AND existing.tbl=$${n + 3}
|
|
787
|
+
AND existing.row_id=batch.row_id
|
|
788
|
+
AND NOT EXISTS (
|
|
789
|
+
SELECT 1 FROM scope
|
|
790
|
+
WHERE scope.row_id=existing.row_id
|
|
791
|
+
AND scope.key=existing.var AND scope.value=existing.value)
|
|
792
|
+
)
|
|
793
|
+
INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
794
|
+
SELECT $${n + 2}, $${n + 3}, key, value, row_id FROM scope
|
|
795
|
+
ON CONFLICT DO NOTHING`,
|
|
796
|
+
[
|
|
797
|
+
...params,
|
|
798
|
+
JSON.stringify(
|
|
799
|
+
slice.map(({ row }) => ({ row_id: row.rowId, scopes: row.scopes })),
|
|
800
|
+
),
|
|
801
|
+
partition,
|
|
802
|
+
compiled.name,
|
|
803
|
+
],
|
|
804
|
+
),
|
|
805
|
+
);
|
|
806
|
+
}
|
|
807
|
+
return Promise.all(writes);
|
|
742
808
|
}
|
|
743
809
|
|
|
744
810
|
async function getPushResultOn(
|
|
@@ -787,6 +853,8 @@ async function queryAuthoritativeOn(
|
|
|
787
853
|
|
|
788
854
|
class PostgresTransaction implements StorageTransaction {
|
|
789
855
|
#client: PgQueryable;
|
|
856
|
+
/** The FIFO without the staged-write prelude (see the constructor). */
|
|
857
|
+
#serial: PgQueryable;
|
|
790
858
|
#partition: string;
|
|
791
859
|
/** Storage-owned relation rewrite, bound to this transaction's partition. */
|
|
792
860
|
readonly #prepareQuery: (
|
|
@@ -804,6 +872,25 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
804
872
|
*/
|
|
805
873
|
readonly #prefetchedRows = new Map<string, StoredRow | null>();
|
|
806
874
|
readonly #prefetchedTombstones = new Map<string, number | null>();
|
|
875
|
+
/**
|
|
876
|
+
* Upserts staged per table (row id → row and its push operation index).
|
|
877
|
+
* Every statement of this transaction writes them first (see the
|
|
878
|
+
* constructor), so reads and later writes observe them; a commit's rows
|
|
879
|
+
* reach the database as one statement per table instead of one per row.
|
|
880
|
+
* Tables with a unique index write immediately: a violation must name its
|
|
881
|
+
* operation.
|
|
882
|
+
*/
|
|
883
|
+
readonly #staged = new Map<
|
|
884
|
+
CompiledTable,
|
|
885
|
+
Map<
|
|
886
|
+
string,
|
|
887
|
+
{
|
|
888
|
+
readonly row: StoredRow;
|
|
889
|
+
readonly values: readonly unknown[];
|
|
890
|
+
readonly opIndex?: number;
|
|
891
|
+
}
|
|
892
|
+
>
|
|
893
|
+
>();
|
|
807
894
|
/** Resolves/rejects the `transaction(fn)` wrapper (see `begin`). */
|
|
808
895
|
#resolve: () => void;
|
|
809
896
|
#reject: (error: unknown) => void;
|
|
@@ -824,7 +911,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
824
911
|
// issued through this transaction is chained on one FIFO — the same
|
|
825
912
|
// shape as `SqliteServerStorage.queryAuthoritative`.
|
|
826
913
|
let tail: Promise<void> = Promise.resolve();
|
|
827
|
-
|
|
914
|
+
const serial: PgQueryable = {
|
|
828
915
|
query: <Row>(text: string, params?: readonly unknown[]) => {
|
|
829
916
|
const run = tail.then(() => client.query<Row>(text, params));
|
|
830
917
|
tail = run.then(
|
|
@@ -834,6 +921,23 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
834
921
|
return run;
|
|
835
922
|
},
|
|
836
923
|
};
|
|
924
|
+
this.#serial = serial;
|
|
925
|
+
this.#client = {
|
|
926
|
+
query: async <Row>(text: string, params?: readonly unknown[]) => {
|
|
927
|
+
// Queue the staged writes ahead of this statement synchronously, so
|
|
928
|
+
// a concurrently issued statement cannot overtake them.
|
|
929
|
+
const staged = this.#writeStaged();
|
|
930
|
+
const run = serial.query<Row>(text, params);
|
|
931
|
+
try {
|
|
932
|
+
await staged;
|
|
933
|
+
} catch (error) {
|
|
934
|
+
// The aborted transaction fails the queued statement too.
|
|
935
|
+
run.catch(() => undefined);
|
|
936
|
+
throw error;
|
|
937
|
+
}
|
|
938
|
+
return run;
|
|
939
|
+
},
|
|
940
|
+
};
|
|
837
941
|
this.#partition = partition;
|
|
838
942
|
this.#writerVersion = writerVersion;
|
|
839
943
|
this.#resolveTable = resolveTable;
|
|
@@ -846,6 +950,27 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
846
950
|
if (!this.#open) throw new Error('transaction already finished');
|
|
847
951
|
}
|
|
848
952
|
|
|
953
|
+
#writeStaged(): Promise<unknown> {
|
|
954
|
+
if (this.#staged.size === 0) return Promise.resolve();
|
|
955
|
+
const staged = [...this.#staged];
|
|
956
|
+
this.#staged.clear();
|
|
957
|
+
return Promise.all(
|
|
958
|
+
staged.map(([table, rows]) => {
|
|
959
|
+
const entries = [...rows.values()];
|
|
960
|
+
return writeRowsOn(this.#serial, table, this.#partition, entries).catch(
|
|
961
|
+
(error: unknown) => {
|
|
962
|
+
// Staged tables have no unique index, so a violation is a column
|
|
963
|
+
// constraint; it names the first staged operation of the table.
|
|
964
|
+
if (isPostgresConstraintError(error)) {
|
|
965
|
+
throw new StorageConstraintError(error, entries[0]?.opIndex);
|
|
966
|
+
}
|
|
967
|
+
throw error;
|
|
968
|
+
},
|
|
969
|
+
);
|
|
970
|
+
}),
|
|
971
|
+
);
|
|
972
|
+
}
|
|
973
|
+
|
|
849
974
|
getRow(table: string, rowId: string): Promise<StoredRow | undefined> {
|
|
850
975
|
this.#assertOpen();
|
|
851
976
|
const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
|
|
@@ -1038,6 +1163,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1038
1163
|
if (!this.#pushApplySavepoint) {
|
|
1039
1164
|
throw new Error('push rejection requires its apply savepoint');
|
|
1040
1165
|
}
|
|
1166
|
+
this.#staged.clear();
|
|
1041
1167
|
await this.#client.query('ROLLBACK TO SAVEPOINT syncular_push_candidate');
|
|
1042
1168
|
await this.#client.query('RELEASE SAVEPOINT syncular_push_candidate');
|
|
1043
1169
|
this.#pushApplySavepoint = false;
|
|
@@ -1052,13 +1178,28 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1052
1178
|
): Promise<void> {
|
|
1053
1179
|
this.#assertOpen();
|
|
1054
1180
|
this.#prefetchedRows.delete(`${table}\u0000${row.rowId}`);
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1181
|
+
const compiled = this.#resolveTable(table);
|
|
1182
|
+
// Decodes the payload: an undecodable row fails here, at its operation.
|
|
1183
|
+
const values = upsertValues(compiled, this.#partition, row, 'postgres');
|
|
1184
|
+
if (!compiled.indexes.some((index) => index.unique === true)) {
|
|
1185
|
+
let rows = this.#staged.get(compiled);
|
|
1186
|
+
if (rows === undefined) {
|
|
1187
|
+
rows = new Map();
|
|
1188
|
+
this.#staged.set(compiled, rows);
|
|
1189
|
+
}
|
|
1190
|
+
// A later upsert of the same row in this transaction replaces it.
|
|
1191
|
+
rows.delete(row.rowId);
|
|
1192
|
+
rows.set(row.rowId, {
|
|
1060
1193
|
row,
|
|
1061
|
-
|
|
1194
|
+
values,
|
|
1195
|
+
...(context === undefined ? {} : { opIndex: context.opIndex }),
|
|
1196
|
+
});
|
|
1197
|
+
return;
|
|
1198
|
+
}
|
|
1199
|
+
try {
|
|
1200
|
+
await writeRowsOn(this.#client, compiled, this.#partition, [
|
|
1201
|
+
{ row, values },
|
|
1202
|
+
]);
|
|
1062
1203
|
} catch (error) {
|
|
1063
1204
|
if (isPostgresConstraintError(error)) {
|
|
1064
1205
|
throw new StorageConstraintError(error, context?.opIndex);
|
|
@@ -1231,6 +1372,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1231
1372
|
|
|
1232
1373
|
async commit(): Promise<void> {
|
|
1233
1374
|
this.#assertOpen();
|
|
1375
|
+
await this.#writeStaged();
|
|
1234
1376
|
this.#open = false;
|
|
1235
1377
|
// Signal the transaction wrapper to COMMIT; wait for it to actually land.
|
|
1236
1378
|
this.#resolve();
|
|
@@ -1239,6 +1381,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1239
1381
|
|
|
1240
1382
|
async rollback(): Promise<void> {
|
|
1241
1383
|
if (!this.#open) return;
|
|
1384
|
+
this.#staged.clear();
|
|
1242
1385
|
this.#open = false;
|
|
1243
1386
|
this.#reject(new RollbackSignal());
|
|
1244
1387
|
// Swallow — a rollback is a normal outcome, not an error to the caller.
|
|
@@ -1279,11 +1422,11 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1279
1422
|
#schemaVersion: number | undefined;
|
|
1280
1423
|
/**
|
|
1281
1424
|
* Page reads waiting for the end of the current microtask turn, keyed by
|
|
1282
|
-
*
|
|
1425
|
+
* batch stream and partition. A pull starts the window read (or first
|
|
1283
1426
|
* snapshot page) of every subscription before awaiting any of them, so the
|
|
1284
|
-
*
|
|
1285
|
-
*
|
|
1286
|
-
*
|
|
1427
|
+
* reads of all tables leave as ONE set-based statement per stream instead
|
|
1428
|
+
* of one round trip per subscription or table
|
|
1429
|
+
* (SYNCULAR-PULL-ROUNDTRIPS-001).
|
|
1287
1430
|
*/
|
|
1288
1431
|
readonly #pageBatches = new Map<string, PendingPage[]>();
|
|
1289
1432
|
|
|
@@ -1292,37 +1435,42 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1292
1435
|
}
|
|
1293
1436
|
|
|
1294
1437
|
/**
|
|
1295
|
-
* Queue one page read into the batch for (`
|
|
1296
|
-
*
|
|
1297
|
-
*
|
|
1298
|
-
*
|
|
1438
|
+
* Queue one page read into the batch for (`key`, `partition`). `sql` is the
|
|
1439
|
+
* statement every batch of that stream shares, or a builder over the queued
|
|
1440
|
+
* specs when the statement names what the batch contains (the snapshot scan
|
|
1441
|
+
* joins one branch per table present). The batch runs as a single statement
|
|
1442
|
+
* with the queued specs bound as a JSON array (`$2`); each spec gets its
|
|
1443
|
+
* position as `i`, and the result rows are split back by their `sub`
|
|
1444
|
+
* column.
|
|
1299
1445
|
*/
|
|
1300
1446
|
#batchedPage<Row>(
|
|
1301
|
-
|
|
1447
|
+
key: string,
|
|
1448
|
+
sql: string | ((specs: readonly Record<string, unknown>[]) => string),
|
|
1302
1449
|
partition: string,
|
|
1303
1450
|
spec: Record<string, unknown>,
|
|
1304
1451
|
): Promise<Row[]> {
|
|
1305
|
-
const
|
|
1306
|
-
let batch = this.#pageBatches.get(
|
|
1452
|
+
const batchKey = `${key}\u0000${partition}`;
|
|
1453
|
+
let batch = this.#pageBatches.get(batchKey);
|
|
1307
1454
|
if (batch === undefined) {
|
|
1308
1455
|
const queued: PendingPage[] = [];
|
|
1309
1456
|
batch = queued;
|
|
1310
|
-
this.#pageBatches.set(
|
|
1457
|
+
this.#pageBatches.set(batchKey, queued);
|
|
1311
1458
|
queueMicrotask(() => {
|
|
1312
|
-
this.#pageBatches.delete(
|
|
1313
|
-
const specs =
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
(
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1459
|
+
this.#pageBatches.delete(batchKey);
|
|
1460
|
+
const specs = queued.map((entry, i) => ({ ...entry.spec, i }));
|
|
1461
|
+
const statement = typeof sql === 'string' ? sql : sql(specs);
|
|
1462
|
+
this.#exec
|
|
1463
|
+
.query<PgRow>(statement, [partition, JSON.stringify(specs)])
|
|
1464
|
+
.then(
|
|
1465
|
+
({ rows }) => {
|
|
1466
|
+
const grouped = queued.map((): PgRow[] => []);
|
|
1467
|
+
for (const row of rows) grouped[asNumber(row.sub)]?.push(row);
|
|
1468
|
+
queued.forEach((entry, i) => entry.resolve(grouped[i] ?? []));
|
|
1469
|
+
},
|
|
1470
|
+
(error: unknown) => {
|
|
1471
|
+
for (const entry of queued) entry.reject(error);
|
|
1472
|
+
},
|
|
1473
|
+
);
|
|
1326
1474
|
});
|
|
1327
1475
|
}
|
|
1328
1476
|
const target = batch;
|
|
@@ -1537,10 +1685,6 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1537
1685
|
// backfill for flipped-on materialization) — all inside one
|
|
1538
1686
|
// transaction (Postgres DDL is transactional — a failed bump leaves
|
|
1539
1687
|
// no half-state).
|
|
1540
|
-
const layouts = parseLayouts(
|
|
1541
|
-
typeof marker.layouts === 'string' ? marker.layouts : undefined,
|
|
1542
|
-
);
|
|
1543
|
-
const retiredTables = retiredTableNames(schema, layouts);
|
|
1544
1688
|
await this.#exec.transaction(async (client) => {
|
|
1545
1689
|
// RFC 0007 migration barrier: `LOCK TABLE ... EXCLUSIVE` is the FIRST
|
|
1546
1690
|
// statement, before any read, DDL, rewrite, marker or checkpoint
|
|
@@ -1553,8 +1697,33 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1553
1697
|
// when it takes this lock, so a writer already inside completes while
|
|
1554
1698
|
// the migration waits.
|
|
1555
1699
|
await client.query('LOCK TABLE sync_partitions IN EXCLUSIVE MODE');
|
|
1700
|
+
// The probe ran before the lock. A concurrent `ensureSchema` (another
|
|
1701
|
+
// isolate, the Durable Object's first request) may have bumped while
|
|
1702
|
+
// this one waited; re-running the rewrite would decode its new-layout
|
|
1703
|
+
// payloads with the old layout. The locked marker decides.
|
|
1704
|
+
const { rows: lockedMarker } = await client.query<{
|
|
1705
|
+
schema_version: unknown;
|
|
1706
|
+
layouts: unknown;
|
|
1707
|
+
}>('SELECT schema_version, layouts FROM sync_schema_meta WHERE id=1');
|
|
1708
|
+
const lockedVersion =
|
|
1709
|
+
lockedMarker[0]?.schema_version === null ||
|
|
1710
|
+
lockedMarker[0]?.schema_version === undefined
|
|
1711
|
+
? undefined
|
|
1712
|
+
: asNumber(lockedMarker[0].schema_version);
|
|
1713
|
+
if (lockedVersion !== undefined && lockedVersion > schema.version) {
|
|
1714
|
+
throw new Error(
|
|
1715
|
+
`stored schema version ${lockedVersion} is newer than the configured schema (${schema.version}) — refusing to run an older server against a migrated database`,
|
|
1716
|
+
);
|
|
1717
|
+
}
|
|
1718
|
+
if (lockedVersion === schema.version) return;
|
|
1719
|
+
const layouts = parseLayouts(
|
|
1720
|
+
typeof lockedMarker[0]?.layouts === 'string'
|
|
1721
|
+
? lockedMarker[0].layouts
|
|
1722
|
+
: undefined,
|
|
1723
|
+
);
|
|
1724
|
+
const retiredTables = retiredTableNames(schema, layouts);
|
|
1556
1725
|
const existing = new Map<string, ReadonlySet<string>>();
|
|
1557
|
-
const existingIndexes = new Map<string,
|
|
1726
|
+
const existingIndexes = new Map<string, ReadonlyMap<string, string>>();
|
|
1558
1727
|
for (const table of schema.tables.values()) {
|
|
1559
1728
|
const { rows } = await client.query<{ column_name: string }>(
|
|
1560
1729
|
`SELECT column_name FROM information_schema.columns
|
|
@@ -1568,8 +1737,25 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1568
1737
|
// indexes (PRIMARY KEY, UNIQUE constraints) can only be removed
|
|
1569
1738
|
// through their constraint, so DROP INDEX on them would abort
|
|
1570
1739
|
// the migration transaction.
|
|
1571
|
-
const indexes = await client.query<{
|
|
1572
|
-
|
|
1740
|
+
const indexes = await client.query<{
|
|
1741
|
+
index_name: string;
|
|
1742
|
+
is_unique: boolean;
|
|
1743
|
+
is_partial: boolean;
|
|
1744
|
+
columns: (string | null)[];
|
|
1745
|
+
}>(
|
|
1746
|
+
`SELECT index_class.relname AS index_name,
|
|
1747
|
+
index_meta.indisunique AS is_unique,
|
|
1748
|
+
index_meta.indpred IS NOT NULL AS is_partial,
|
|
1749
|
+
ARRAY(
|
|
1750
|
+
SELECT attribute.attname::text
|
|
1751
|
+
FROM unnest(index_meta.indkey::int2[])
|
|
1752
|
+
WITH ORDINALITY AS key(attnum, ord)
|
|
1753
|
+
LEFT JOIN pg_catalog.pg_attribute AS attribute
|
|
1754
|
+
ON attribute.attrelid = table_class.oid
|
|
1755
|
+
AND attribute.attnum = key.attnum
|
|
1756
|
+
WHERE key.ord <= index_meta.indnkeyatts
|
|
1757
|
+
ORDER BY key.ord
|
|
1758
|
+
) AS columns
|
|
1573
1759
|
FROM pg_catalog.pg_class AS table_class
|
|
1574
1760
|
JOIN pg_catalog.pg_namespace AS namespace
|
|
1575
1761
|
ON namespace.oid = table_class.relnamespace
|
|
@@ -1589,17 +1775,34 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1589
1775
|
);
|
|
1590
1776
|
existingIndexes.set(
|
|
1591
1777
|
table.name,
|
|
1592
|
-
new
|
|
1778
|
+
new Map(
|
|
1779
|
+
indexes.rows.map((index) => [
|
|
1780
|
+
index.index_name,
|
|
1781
|
+
indexSignature(
|
|
1782
|
+
index.columns,
|
|
1783
|
+
index.is_unique,
|
|
1784
|
+
index.is_partial,
|
|
1785
|
+
),
|
|
1786
|
+
]),
|
|
1787
|
+
),
|
|
1593
1788
|
);
|
|
1594
1789
|
}
|
|
1595
1790
|
}
|
|
1791
|
+
if (retiredTables.length > 0) {
|
|
1792
|
+
// Neither index leads with `tbl`: one scan per table, not one per
|
|
1793
|
+
// retired table.
|
|
1794
|
+
await client.query(
|
|
1795
|
+
`DELETE FROM sync_row_scopes
|
|
1796
|
+
WHERE tbl IN (SELECT jsonb_array_elements_text($1::text::jsonb))`,
|
|
1797
|
+
[JSON.stringify(retiredTables)],
|
|
1798
|
+
);
|
|
1799
|
+
await client.query(
|
|
1800
|
+
`DELETE FROM sync_blob_refs
|
|
1801
|
+
WHERE tbl IN (SELECT jsonb_array_elements_text($1::text::jsonb))`,
|
|
1802
|
+
[JSON.stringify(retiredTables)],
|
|
1803
|
+
);
|
|
1804
|
+
}
|
|
1596
1805
|
for (const tableName of retiredTables) {
|
|
1597
|
-
await client.query('DELETE FROM sync_row_scopes WHERE tbl=$1', [
|
|
1598
|
-
tableName,
|
|
1599
|
-
]);
|
|
1600
|
-
await client.query('DELETE FROM sync_blob_refs WHERE tbl=$1', [
|
|
1601
|
-
tableName,
|
|
1602
|
-
]);
|
|
1603
1806
|
await client.query(dropTableDdl(tableName));
|
|
1604
1807
|
}
|
|
1605
1808
|
for (const statement of schemaDdl(
|
|
@@ -1667,20 +1870,35 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1667
1870
|
if (initialLogEpoch.length === 0) {
|
|
1668
1871
|
throw new Error('initial log epoch must be non-empty');
|
|
1669
1872
|
}
|
|
1670
|
-
|
|
1873
|
+
type Registry = {
|
|
1671
1874
|
log_epoch: string;
|
|
1672
1875
|
epoch_required: boolean;
|
|
1673
1876
|
last_authenticated_at_ms: unknown;
|
|
1674
|
-
}
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1877
|
+
};
|
|
1878
|
+
let row = (
|
|
1879
|
+
await this.#exec.query<Registry>(
|
|
1880
|
+
`SELECT log_epoch, epoch_required, last_authenticated_at_ms
|
|
1881
|
+
FROM sync_partition_registry WHERE partition=$1`,
|
|
1882
|
+
[partition],
|
|
1883
|
+
)
|
|
1884
|
+
).rows[0];
|
|
1885
|
+
if (
|
|
1886
|
+
row === undefined ||
|
|
1887
|
+
authenticatedAtMs - asNumber(row.last_authenticated_at_ms) >=
|
|
1888
|
+
PARTITION_TOUCH_INTERVAL_MS
|
|
1889
|
+
) {
|
|
1890
|
+
row = (
|
|
1891
|
+
await this.#exec.query<Registry>(
|
|
1892
|
+
`INSERT INTO sync_partition_registry(
|
|
1893
|
+
partition, log_epoch, last_authenticated_at_ms
|
|
1894
|
+
) VALUES ($1,$2,$3)
|
|
1895
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
1896
|
+
last_authenticated_at_ms=EXCLUDED.last_authenticated_at_ms
|
|
1897
|
+
RETURNING log_epoch, epoch_required, last_authenticated_at_ms`,
|
|
1898
|
+
[partition, initialLogEpoch, authenticatedAtMs],
|
|
1899
|
+
)
|
|
1900
|
+
).rows[0];
|
|
1901
|
+
}
|
|
1684
1902
|
if (row === undefined)
|
|
1685
1903
|
throw new Error('partition registry write did not persist');
|
|
1686
1904
|
return {
|
|
@@ -2317,6 +2535,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2317
2535
|
if (firstVariable === undefined || values.length === 0) return 0;
|
|
2318
2536
|
// Concurrent reads of a pull share one statement across tables.
|
|
2319
2537
|
const [row] = await this.#batchedPage<{ seq: unknown }>(
|
|
2538
|
+
'latestScopedChange',
|
|
2320
2539
|
LATEST_SCOPED_CHANGE_BATCH_SQL,
|
|
2321
2540
|
partition,
|
|
2322
2541
|
{ tbl: query.table, var: firstVariable, vals: values },
|
|
@@ -2346,6 +2565,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2346
2565
|
const batchSize = Math.max(64, query.limitChanges);
|
|
2347
2566
|
while (deliveredChanges < query.limitChanges) {
|
|
2348
2567
|
const records = await this.#batchedPage<CommitWindowRecord>(
|
|
2568
|
+
'commitWindow',
|
|
2349
2569
|
COMMIT_WINDOW_BATCH_SQL,
|
|
2350
2570
|
partition,
|
|
2351
2571
|
{
|
|
@@ -2414,21 +2634,30 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2414
2634
|
if (firstValues.length === 0) return [];
|
|
2415
2635
|
// Candidates via the inverted index (ordered + LIMITed at the covering
|
|
2416
2636
|
// PK) LEFT JOINed to the row table — one round trip per page, never one
|
|
2417
|
-
// per row, and
|
|
2418
|
-
//
|
|
2419
|
-
//
|
|
2420
|
-
const sql = scanRowBatchSql(this.table(query.table));
|
|
2637
|
+
// per row, and concurrent scans across tables share one statement (see
|
|
2638
|
+
// `scanRowBatchSql`). Exact multi-variable verification against the
|
|
2639
|
+
// stored scope map happens below.
|
|
2421
2640
|
const rows: StoredRow[] = [];
|
|
2422
2641
|
let afterRowId = query.afterRowId ?? '';
|
|
2423
2642
|
const batchSize = Math.max(64, query.limit);
|
|
2424
2643
|
while (rows.length < query.limit) {
|
|
2425
|
-
const records = await this.#batchedPage<RowRecord>(
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2644
|
+
const records = await this.#batchedPage<RowRecord>(
|
|
2645
|
+
'scanRows',
|
|
2646
|
+
(specs) =>
|
|
2647
|
+
scanRowBatchSql(
|
|
2648
|
+
[...new Set(specs.map((spec) => String(spec.tbl)))].map((table) =>
|
|
2649
|
+
this.table(table),
|
|
2650
|
+
),
|
|
2651
|
+
),
|
|
2652
|
+
partition,
|
|
2653
|
+
{
|
|
2654
|
+
tbl: query.table,
|
|
2655
|
+
var: firstVariable,
|
|
2656
|
+
vals: firstValues,
|
|
2657
|
+
after: afterRowId,
|
|
2658
|
+
lim: batchSize,
|
|
2659
|
+
},
|
|
2660
|
+
);
|
|
2432
2661
|
if (records.length === 0) break;
|
|
2433
2662
|
for (const record of records) {
|
|
2434
2663
|
afterRowId = record.row_id;
|