@syncular/server 0.59.0 → 0.60.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -16
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +42 -40
- package/dist/context.d.ts +6 -8
- package/dist/context.js +1 -3
- package/dist/d1-storage.js +42 -24
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +35 -17
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +486 -223
- package/dist/local-authority.js +7 -9
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +249 -168
- package/dist/pull.d.ts +68 -53
- package/dist/pull.js +258 -287
- package/dist/push.d.ts +12 -3
- package/dist/push.js +95 -32
- package/dist/realtime.d.ts +17 -11
- package/dist/realtime.js +65 -90
- package/dist/relational-rows.d.ts +0 -57
- package/dist/relational-rows.js +4 -100
- package/dist/seed.js +10 -15
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +4 -2
- package/dist/sqlite-dialect.d.ts +36 -59
- package/dist/sqlite-dialect.js +255 -159
- package/dist/sqlite-storage.js +24 -20
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +156 -54
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +87 -59
- package/src/context.ts +7 -11
- package/src/d1-storage.ts +49 -33
- package/src/errors.ts +8 -0
- package/src/events.ts +36 -17
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +628 -286
- package/src/local-authority.ts +10 -11
- package/src/operations.ts +44 -42
- package/src/postgres-storage.ts +336 -226
- package/src/pull.ts +394 -414
- package/src/push.ts +116 -21
- package/src/realtime.ts +85 -129
- package/src/relational-rows.ts +4 -107
- package/src/seed.ts +13 -19
- package/src/segment-download.ts +8 -3
- package/src/sqlite-dialect.ts +363 -204
- package/src/sqlite-storage.ts +45 -25
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +185 -58
package/src/realtime.ts
CHANGED
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
* queues the ack's cursor write without awaiting it, so `drain()` is the
|
|
8
8
|
* awaitable boundary a host uses before closing its storage or ending a
|
|
9
9
|
* hibernatable event (§8.2). Initial
|
|
10
|
-
*
|
|
11
|
-
*
|
|
10
|
+
* registration comes from the live units of the client record (§8.1); a sync
|
|
11
|
+
* round completed on the
|
|
12
12
|
* connection replaces it at round end (§8.7). Deltas are complete SSP2
|
|
13
13
|
* response messages pushed as `0x00`-tagged binary (§8.2/§8.7); sync
|
|
14
14
|
* rounds ride the same socket as `0x01`-tagged byte-stream chunks driven
|
|
@@ -17,10 +17,9 @@
|
|
|
17
17
|
* the `sync` wake-up (§8.3).
|
|
18
18
|
*/
|
|
19
19
|
import {
|
|
20
|
-
type CommitChange,
|
|
21
20
|
DecodeError,
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
decodeRequest,
|
|
22
|
+
encodeResponse,
|
|
24
23
|
encodePresenceError,
|
|
25
24
|
encodePresenceFanout,
|
|
26
25
|
isSupportedProtocolWireVersion,
|
|
@@ -29,8 +28,8 @@ import {
|
|
|
29
28
|
type PresenceKind,
|
|
30
29
|
REALTIME_TAG_DELTA,
|
|
31
30
|
REALTIME_TAG_ROUND,
|
|
32
|
-
type ResponseFrame,
|
|
33
31
|
type ScopeMap,
|
|
32
|
+
unitId,
|
|
34
33
|
type WakeReason,
|
|
35
34
|
} from '@syncular/core';
|
|
36
35
|
import type {
|
|
@@ -55,17 +54,14 @@ import {
|
|
|
55
54
|
import { emitEvent, type SyncularServerEvents } from './events';
|
|
56
55
|
import { composeEvents } from './events-ring';
|
|
57
56
|
import { createSyncResponseStream } from './handler';
|
|
57
|
+
import { commitFrame, coveredTables, planUnit } from './pull';
|
|
58
58
|
import {
|
|
59
59
|
type CompiledSchema,
|
|
60
60
|
type CompiledTable,
|
|
61
61
|
compileSchema,
|
|
62
62
|
} from './schema';
|
|
63
|
-
import { schemaWindowOf
|
|
64
|
-
import {
|
|
65
|
-
computeEffective,
|
|
66
|
-
matchesEffective,
|
|
67
|
-
type ResolvedScopes,
|
|
68
|
-
} from './scopes';
|
|
63
|
+
import { schemaWindowOf } from './schema-window';
|
|
64
|
+
import { matchesEffective, type ResolvedScopes } from './scopes';
|
|
69
65
|
import type { ClientRecord, ServerStorage, StoredCommit } from './storage';
|
|
70
66
|
|
|
71
67
|
/**
|
|
@@ -127,8 +123,11 @@ export interface RealtimeConnectOptions {
|
|
|
127
123
|
readonly onPresence?: (presence: ReadonlyMap<string, string>) => void;
|
|
128
124
|
}
|
|
129
125
|
|
|
126
|
+
/** One active (unit, table) pair of a live unit (§8.1). */
|
|
130
127
|
interface Registration {
|
|
131
|
-
|
|
128
|
+
/** The schema version of the client's latest round: the delta codec (§9). */
|
|
129
|
+
readonly schema: CompiledSchema;
|
|
130
|
+
/** The unit id (§4.1). */
|
|
132
131
|
readonly id: string;
|
|
133
132
|
readonly table: string;
|
|
134
133
|
readonly schemaVersion: number;
|
|
@@ -676,9 +675,8 @@ export class RealtimeSession {
|
|
|
676
675
|
try {
|
|
677
676
|
// §8.7: the round's clientId must match the connection's —
|
|
678
677
|
// registration identity would otherwise be ambiguous.
|
|
679
|
-
const
|
|
680
|
-
|
|
681
|
-
if (decoded.msgKind !== 'request' || header?.type !== 'REQ_HEADER') {
|
|
678
|
+
const header = decodeRequest(requestBytes).frames[0];
|
|
679
|
+
if (header?.type !== 'REQ_HEADER') {
|
|
682
680
|
throw syncError('sync.invalid_request', 'expected a request message');
|
|
683
681
|
}
|
|
684
682
|
if (header.clientId !== this.clientId) {
|
|
@@ -813,8 +811,9 @@ export class RealtimeSession {
|
|
|
813
811
|
* failed, because that round applied nothing.
|
|
814
812
|
*/
|
|
815
813
|
/**
|
|
816
|
-
* A pull outside this connection changed the client's
|
|
817
|
-
* round in flight on the connection reloads
|
|
814
|
+
* A pull outside this connection changed the client's registration list
|
|
815
|
+
* or live set. A round in flight on the connection reloads it at its end
|
|
816
|
+
* anyway.
|
|
818
817
|
*/
|
|
819
818
|
async refreshAfterPull(): Promise<void> {
|
|
820
819
|
if (this.#activeRound !== undefined) return;
|
|
@@ -937,33 +936,15 @@ export class RealtimeSession {
|
|
|
937
936
|
deliverCommit(commit: StoredCommit): void {
|
|
938
937
|
const contiguous = commit.commitSeq === this.lastKnownSeq + 1;
|
|
939
938
|
this.lastKnownSeq = Math.max(this.lastKnownSeq, commit.commitSeq);
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
matchesEffective(change.scopes, registration.effective),
|
|
950
|
-
)
|
|
951
|
-
.map(
|
|
952
|
-
(change): CommitChange => ({
|
|
953
|
-
tableIndex: 0,
|
|
954
|
-
rowId: change.rowId,
|
|
955
|
-
op: change.op,
|
|
956
|
-
...(change.rowVersion !== undefined
|
|
957
|
-
? { rowVersion: change.rowVersion }
|
|
958
|
-
: {}),
|
|
959
|
-
scopes: change.scopes,
|
|
960
|
-
...(change.payload !== undefined
|
|
961
|
-
? { row: this.#hub.projectPayload(registration, change.payload) }
|
|
962
|
-
: {}),
|
|
963
|
-
}),
|
|
964
|
-
);
|
|
965
|
-
if (changes.length > 0) sections.push({ registration, changes });
|
|
966
|
-
}
|
|
939
|
+
// §8.2: one COMMIT holding every change that matches a registration of
|
|
940
|
+
// the connection, once each, in change order.
|
|
941
|
+
const changes = commit.changes.filter((change) =>
|
|
942
|
+
this.registrations.some(
|
|
943
|
+
(registration) =>
|
|
944
|
+
change.table === registration.table &&
|
|
945
|
+
matchesEffective(change.scopes, registration.effective),
|
|
946
|
+
),
|
|
947
|
+
);
|
|
967
948
|
if (!contiguous) {
|
|
968
949
|
// Pushes allocate commitSeq under the partition lock but notify after
|
|
969
950
|
// commit, so racing notifications may arrive out of order. A gap,
|
|
@@ -991,43 +972,30 @@ export class RealtimeSession {
|
|
|
991
972
|
}
|
|
992
973
|
this.#deferredRegistrations = registrations;
|
|
993
974
|
}
|
|
994
|
-
if (
|
|
975
|
+
if (changes.length === 0) return;
|
|
995
976
|
if (this.wakePending) {
|
|
996
977
|
// §8.2: deltas must be cursor-contiguous; while the client is behind
|
|
997
978
|
// we send (coalescible) wake-ups instead of a delta past its cursor.
|
|
998
979
|
this.sendWake('catchup-required');
|
|
999
980
|
return;
|
|
1000
981
|
}
|
|
1001
|
-
const
|
|
1002
|
-
{
|
|
1003
|
-
type: 'RESP_HEADER',
|
|
1004
|
-
logEpoch: this.logEpoch,
|
|
1005
|
-
resetRequired: false,
|
|
1006
|
-
},
|
|
1007
|
-
];
|
|
1008
|
-
for (const section of sections) {
|
|
1009
|
-
frames.push({
|
|
1010
|
-
type: 'SUB_START',
|
|
1011
|
-
id: section.registration.id,
|
|
1012
|
-
status: 'active',
|
|
1013
|
-
reasonCode: '',
|
|
1014
|
-
effectiveScopes: section.registration.effective,
|
|
1015
|
-
bootstrap: false,
|
|
1016
|
-
});
|
|
1017
|
-
frames.push({
|
|
1018
|
-
type: 'COMMIT',
|
|
1019
|
-
commitSeq: commit.commitSeq,
|
|
1020
|
-
createdAtMs: commit.createdAtMs,
|
|
1021
|
-
actorId: commit.actorId,
|
|
1022
|
-
tables: [section.registration.table],
|
|
1023
|
-
changes: section.changes,
|
|
1024
|
-
});
|
|
1025
|
-
frames.push({ type: 'SUB_END', nextCursor: commit.commitSeq });
|
|
1026
|
-
}
|
|
1027
|
-
const bytes = encodeMessage({
|
|
982
|
+
const bytes = encodeResponse({
|
|
1028
983
|
wireVersion: this.wireVersion,
|
|
1029
984
|
msgKind: 'response',
|
|
1030
|
-
frames
|
|
985
|
+
frames: [
|
|
986
|
+
{ type: 'RESP_HEADER', logEpoch: this.logEpoch, resetRequired: false },
|
|
987
|
+
{ type: 'LIVE_START', status: 'active', reasonCode: '' },
|
|
988
|
+
commitFrame(
|
|
989
|
+
{ ...commit, changes },
|
|
990
|
+
this.#hub.schema,
|
|
991
|
+
this.registrations[0]!.schema,
|
|
992
|
+
),
|
|
993
|
+
{
|
|
994
|
+
type: 'LIVE_END',
|
|
995
|
+
throughSeq: commit.commitSeq,
|
|
996
|
+
echoDigest: new Uint8Array(),
|
|
997
|
+
},
|
|
998
|
+
],
|
|
1031
999
|
});
|
|
1032
1000
|
if (bytes.length > this.#maxDeltaBytes) {
|
|
1033
1001
|
this.sendWake('delta-too-large');
|
|
@@ -1037,7 +1005,7 @@ export class RealtimeSession {
|
|
|
1037
1005
|
const tagged = new Uint8Array(bytes.length + 1);
|
|
1038
1006
|
tagged[0] = REALTIME_TAG_DELTA;
|
|
1039
1007
|
tagged.set(bytes, 1);
|
|
1040
|
-
const changeCount =
|
|
1008
|
+
const changeCount = changes.length;
|
|
1041
1009
|
if (this.#roundsRefreshing > 0) {
|
|
1042
1010
|
if (this.#deferredBytes + tagged.length > this.#maxDeltaBytes) {
|
|
1043
1011
|
this.sendWake('catchup-required');
|
|
@@ -1117,12 +1085,9 @@ export class RealtimeHub {
|
|
|
1117
1085
|
this.limits = limitsOf(config);
|
|
1118
1086
|
}
|
|
1119
1087
|
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
registration.codec,
|
|
1124
|
-
payload,
|
|
1125
|
-
);
|
|
1088
|
+
/** The current compiled schema: the storage codec of fanned-out changes. */
|
|
1089
|
+
get schema(): CompiledSchema {
|
|
1090
|
+
return this.#schema;
|
|
1126
1091
|
}
|
|
1127
1092
|
|
|
1128
1093
|
/**
|
|
@@ -1206,7 +1171,7 @@ export class RealtimeHub {
|
|
|
1206
1171
|
}
|
|
1207
1172
|
|
|
1208
1173
|
/**
|
|
1209
|
-
* Resolve the client record's
|
|
1174
|
+
* Resolve the client record's live units into per-connection
|
|
1210
1175
|
* registrations (§8.1 at upgrade; §8.7 at socket-round end).
|
|
1211
1176
|
*/
|
|
1212
1177
|
async loadRegistrations(
|
|
@@ -1242,39 +1207,35 @@ export class RealtimeHub {
|
|
|
1242
1207
|
}
|
|
1243
1208
|
resolved = { ok: false };
|
|
1244
1209
|
}
|
|
1210
|
+
// §8.1: the stored list minus the catching-up units, each live unit's
|
|
1211
|
+
// active pairs under the schema version of the client's latest round.
|
|
1212
|
+
const schema = schemaWindowOf(
|
|
1213
|
+
this.#config.schema,
|
|
1214
|
+
this.#config.schemaWindow,
|
|
1215
|
+
).find((candidate) => candidate.version === record?.schemaVersion);
|
|
1245
1216
|
const registrations: Registration[] = [];
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
const
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
);
|
|
1269
|
-
if (outcome.status !== 'active') continue;
|
|
1270
|
-
registrations.push({
|
|
1271
|
-
id: subscription.id,
|
|
1272
|
-
codec: table,
|
|
1273
|
-
table: subscription.table,
|
|
1274
|
-
schemaVersion:
|
|
1275
|
-
subscription.schemaVersion ?? this.#config.schema.version,
|
|
1276
|
-
effective: outcome.effective,
|
|
1277
|
-
});
|
|
1217
|
+
if (schema === undefined) return registrations;
|
|
1218
|
+
for (const scopes of record?.registration?.units ?? []) {
|
|
1219
|
+
const id = unitId(scopes);
|
|
1220
|
+
if (record?.catchingUp.some((unit) => unit.id === id)) continue;
|
|
1221
|
+
let tables: CompiledTable[];
|
|
1222
|
+
try {
|
|
1223
|
+
tables = coveredTables(schema, scopes);
|
|
1224
|
+
} catch {
|
|
1225
|
+
// A unit the client's schema version no longer serves registers
|
|
1226
|
+
// nothing; the client's next pull fails loud on it (§4.1).
|
|
1227
|
+
continue;
|
|
1228
|
+
}
|
|
1229
|
+
const unit = planUnit(scopes, tables, resolved);
|
|
1230
|
+
for (const [table, effective] of unit.pairs) {
|
|
1231
|
+
registrations.push({
|
|
1232
|
+
schema,
|
|
1233
|
+
id: unit.id,
|
|
1234
|
+
table,
|
|
1235
|
+
schemaVersion: schema.version,
|
|
1236
|
+
effective,
|
|
1237
|
+
});
|
|
1238
|
+
}
|
|
1278
1239
|
}
|
|
1279
1240
|
return registrations;
|
|
1280
1241
|
}
|
|
@@ -1323,8 +1284,8 @@ export class RealtimeHub {
|
|
|
1323
1284
|
}
|
|
1324
1285
|
|
|
1325
1286
|
/**
|
|
1326
|
-
* Register a connected socket (§8.1): load the client's
|
|
1327
|
-
*
|
|
1287
|
+
* Register a connected socket (§8.1): load the client record's live
|
|
1288
|
+
* units, resolve their effective scopes, send `hello`.
|
|
1328
1289
|
*/
|
|
1329
1290
|
connect(options: RealtimeConnectOptions): Promise<RealtimeSession> {
|
|
1330
1291
|
return this.#open(options, undefined);
|
|
@@ -1460,18 +1421,12 @@ export class RealtimeHub {
|
|
|
1460
1421
|
cursor,
|
|
1461
1422
|
latestCursor: latestSeq,
|
|
1462
1423
|
requiresSync:
|
|
1463
|
-
record === undefined ||
|
|
1424
|
+
record?.registration === undefined ||
|
|
1464
1425
|
cursor < latestSeq ||
|
|
1465
|
-
(
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
this.#config.schema,
|
|
1470
|
-
this.#config.schemaWindow,
|
|
1471
|
-
).some(
|
|
1472
|
-
(schema) => schema.version === subscription.schemaVersion,
|
|
1473
|
-
),
|
|
1474
|
-
)),
|
|
1426
|
+
!schemaWindowOf(
|
|
1427
|
+
this.#config.schema,
|
|
1428
|
+
this.#config.schemaWindow,
|
|
1429
|
+
).some((schema) => schema.version === record.schemaVersion),
|
|
1475
1430
|
timestamp: clock(),
|
|
1476
1431
|
},
|
|
1477
1432
|
}),
|
|
@@ -1528,7 +1483,8 @@ export class RealtimeHub {
|
|
|
1528
1483
|
}
|
|
1529
1484
|
|
|
1530
1485
|
/**
|
|
1531
|
-
* RealtimeNotifier: a pull changed a client's
|
|
1486
|
+
* RealtimeNotifier: a pull changed a client's registration list or live
|
|
1487
|
+
* set (§8.1).
|
|
1532
1488
|
* Its sockets reload their registrations with the round-end semantics, so
|
|
1533
1489
|
* an unreadable record keeps the previous registrations.
|
|
1534
1490
|
*/
|
|
@@ -1610,7 +1566,7 @@ function errorResponseBytes(
|
|
|
1610
1566
|
wireVersion: number,
|
|
1611
1567
|
logEpoch: string,
|
|
1612
1568
|
): Uint8Array {
|
|
1613
|
-
return
|
|
1569
|
+
return encodeResponse({
|
|
1614
1570
|
wireVersion,
|
|
1615
1571
|
msgKind: 'response',
|
|
1616
1572
|
frames: [
|
package/src/relational-rows.ts
CHANGED
|
@@ -715,62 +715,6 @@ export function scanRowPageSql(
|
|
|
715
715
|
ORDER BY c.row_id`;
|
|
716
716
|
}
|
|
717
717
|
|
|
718
|
-
/**
|
|
719
|
-
* One-round-trip page read for `readCommitWindow` (sync_* tables only; it
|
|
720
|
-
* lives here beside `scanRowPageSql` because it is the same join-the-index
|
|
721
|
-
* posture, dialect-parameterized the same way): candidates from the inverted
|
|
722
|
-
* change-scope index (ordered + LIMITed at the covering `sync_change_scopes`
|
|
723
|
-
* PK — exactly the old candidate query, so the index-first invariant of
|
|
724
|
-
* `CommitWindowQuery` is unchanged) LEFT JOINed to the commit metadata and
|
|
725
|
-
* to the commit's changes for the table, so a whole window page arrives in
|
|
726
|
-
* ONE statement instead of 1 + 2×candidates round trips. This is the
|
|
727
|
-
* incremental-pull hot path — it runs on every sync round; before the join a
|
|
728
|
-
* 500-candidate catch-up window on Postgres paid ~1000 network round trips.
|
|
729
|
-
*
|
|
730
|
-
* LEFT JOIN (not INNER): a candidate whose commit vanished (scope-index
|
|
731
|
-
* entry without a commit/changes row) must still reach the caller — its
|
|
732
|
-
* `commit_seq` advances the window cursor — so it comes back with NULL
|
|
733
|
-
* meta/change columns rather than disappearing from the page.
|
|
734
|
-
*
|
|
735
|
-
* Rows arrive ordered (commit_seq, idx): oldest-first commits, each commit's
|
|
736
|
-
* change rows consecutive in `idx` order — the caller groups consecutive
|
|
737
|
-
* rows and verifies the full multi-variable scope match in JS.
|
|
738
|
-
*
|
|
739
|
-
* Bind order:
|
|
740
|
-
* - postgres: [partition, tbl, var, ...values, afterSeq, throughSeq,
|
|
741
|
-
* limit] (the joins reuse `$1`/`$2`);
|
|
742
|
-
* - sqlite: [partition, tbl, var, values as JSON array text, afterSeq,
|
|
743
|
-
* throughSeq, limit, partition, partition, tbl] (positional `?`;
|
|
744
|
-
* partition/tbl bind again for the joins). The values expand through
|
|
745
|
-
* `json_each` like `scanRowPageSql`.
|
|
746
|
-
*/
|
|
747
|
-
export function commitWindowPageSql(
|
|
748
|
-
valueCount: number,
|
|
749
|
-
dialect: RelationalDialect,
|
|
750
|
-
): string {
|
|
751
|
-
const p = (n: number) => (dialect === 'sqlite' ? '?' : `$${n}`);
|
|
752
|
-
const values =
|
|
753
|
-
dialect === 'sqlite'
|
|
754
|
-
? 'SELECT value FROM json_each(?)'
|
|
755
|
-
: Array.from({ length: valueCount }, (_, i) => p(4 + i)).join(',');
|
|
756
|
-
const after = p(4 + valueCount);
|
|
757
|
-
const through = p(5 + valueCount);
|
|
758
|
-
const limit = p(6 + valueCount);
|
|
759
|
-
const joinPartition = dialect === 'sqlite' ? '?' : '$1';
|
|
760
|
-
const joinTbl = dialect === 'sqlite' ? '?' : '$2';
|
|
761
|
-
return `SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
762
|
-
ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
763
|
-
FROM (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
764
|
-
WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values})
|
|
765
|
-
AND commit_seq>${after} AND commit_seq<=${through}
|
|
766
|
-
ORDER BY commit_seq LIMIT ${limit}) c
|
|
767
|
-
LEFT JOIN sync_commits m
|
|
768
|
-
ON m.partition=${joinPartition} AND m.commit_seq=c.commit_seq
|
|
769
|
-
LEFT JOIN sync_changes ch
|
|
770
|
-
ON ch.partition=${joinPartition} AND ch.commit_seq=c.commit_seq AND ch.tbl=${joinTbl}
|
|
771
|
-
ORDER BY c.commit_seq, ch.idx`;
|
|
772
|
-
}
|
|
773
|
-
|
|
774
718
|
/**
|
|
775
719
|
* PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
|
|
776
720
|
* page of every subscription scanning any table in a pull. `$2` is a JSON
|
|
@@ -809,29 +753,6 @@ export function scanRowBatchSql(tables: readonly CompiledTable[]): string {
|
|
|
809
753
|
ORDER BY sub, row_id`;
|
|
810
754
|
}
|
|
811
755
|
|
|
812
|
-
/**
|
|
813
|
-
* PostgreSQL set-based form of `commitWindowPageSql`: one statement answers
|
|
814
|
-
* the window page of every subscription in a pull, across all tables. `$2`
|
|
815
|
-
* is a JSON array of `{i, tbl, var, vals, after, through, lim}` records, each
|
|
816
|
-
* running the same index-first candidate subquery through a LATERAL join.
|
|
817
|
-
* Rows arrive ordered by (i, commit_seq, idx).
|
|
818
|
-
*
|
|
819
|
-
* Params: [partition, specs (JSON text)], the specs bound as text like
|
|
820
|
-
* `scanRowBatchSql`.
|
|
821
|
-
*/
|
|
822
|
-
export const COMMIT_WINDOW_BATCH_SQL = `SELECT s.i AS sub, c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
823
|
-
ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
824
|
-
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after bigint, through bigint, lim int)
|
|
825
|
-
CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
826
|
-
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)
|
|
827
|
-
AND commit_seq>s.after AND commit_seq<=s.through
|
|
828
|
-
ORDER BY commit_seq LIMIT s.lim) c
|
|
829
|
-
LEFT JOIN sync_commits m
|
|
830
|
-
ON m.partition=$1 AND m.commit_seq=c.commit_seq
|
|
831
|
-
LEFT JOIN sync_changes ch
|
|
832
|
-
ON ch.partition=$1 AND ch.commit_seq=c.commit_seq AND ch.tbl=s.tbl
|
|
833
|
-
ORDER BY s.i, c.commit_seq, ch.idx`;
|
|
834
|
-
|
|
835
756
|
/**
|
|
836
757
|
* PostgreSQL push read-ahead across every table a commit targets, in one
|
|
837
758
|
* statement: each requested row id with its stored row (NULL columns when
|
|
@@ -867,32 +788,6 @@ export function prefetchRowsBatchSql(
|
|
|
867
788
|
${projection}`;
|
|
868
789
|
}
|
|
869
790
|
|
|
870
|
-
/**
|
|
871
|
-
* SQLite/D1: the newest `commit_seq` in the inverted change-scope index for
|
|
872
|
-
* one variable's values, as one index-only max per value (a `(partition,
|
|
873
|
-
* tbl, var, value)` prefix), so the cost is independent of the table's
|
|
874
|
-
* history. The values arrive as one JSON array, so the statement has no
|
|
875
|
-
* compound SELECT and four bound parameters for any value count (D1 caps a
|
|
876
|
-
* compound SELECT at 5 terms and a statement at 100 parameters).
|
|
877
|
-
* Params: [partition, tbl, var, values as JSON array text].
|
|
878
|
-
*/
|
|
879
|
-
export const LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL = `SELECT max((SELECT max(commit_seq) FROM sync_change_scopes
|
|
880
|
-
WHERE partition=? AND tbl=? AND var=? AND value=v.value)) AS seq
|
|
881
|
-
FROM json_each(?) AS v`;
|
|
882
|
-
|
|
883
|
-
/**
|
|
884
|
-
* PostgreSQL set-based form of `LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL` for the
|
|
885
|
-
* subscriptions of a pull, across all tables. `$2` is a JSON array of
|
|
886
|
-
* `{i, tbl, var, vals}` records. Params: [partition, specs (JSON text)].
|
|
887
|
-
*/
|
|
888
|
-
export const LATEST_SCOPED_CHANGE_BATCH_SQL = `SELECT s.i AS sub,
|
|
889
|
-
(SELECT max(latest.seq) FROM (
|
|
890
|
-
SELECT (SELECT max(commit_seq) FROM sync_change_scopes
|
|
891
|
-
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=v.value) AS seq
|
|
892
|
-
FROM jsonb_array_elements_text(s.vals) AS v(value)) AS latest) AS seq
|
|
893
|
-
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals jsonb)
|
|
894
|
-
ORDER BY s.i`;
|
|
895
|
-
|
|
896
791
|
/**
|
|
897
792
|
* SELECT the rows that reference one blob with their stored scope maps, in
|
|
898
793
|
* one statement for any number of references: the by-blob index range
|
|
@@ -1270,8 +1165,10 @@ export function serverIndexDdl(
|
|
|
1270
1165
|
const drops: string[] = [];
|
|
1271
1166
|
const creates: string[] = [];
|
|
1272
1167
|
for (const table of schema.tables.values()) {
|
|
1273
|
-
|
|
1274
|
-
|
|
1168
|
+
// Every table exists at the current version; a table the introspection
|
|
1169
|
+
// reported no index for (Postgres lists none for a primary-key-only
|
|
1170
|
+
// table) has none.
|
|
1171
|
+
const existing = existingIndexesByTable.get(table.name) ?? new Map();
|
|
1275
1172
|
const desired = desiredIndexes(table);
|
|
1276
1173
|
for (const indexName of existing.keys()) {
|
|
1277
1174
|
if (
|
package/src/seed.ts
CHANGED
|
@@ -10,10 +10,11 @@
|
|
|
10
10
|
* the seed replays the cached push result and writes nothing twice.
|
|
11
11
|
*/
|
|
12
12
|
import {
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
decodeResponse,
|
|
14
|
+
encodeRequest,
|
|
15
15
|
encodeSparseRow,
|
|
16
16
|
PROTOCOL_WIRE_VERSION,
|
|
17
|
+
registrationDigest,
|
|
17
18
|
type PushOperation,
|
|
18
19
|
type PushResultFrame,
|
|
19
20
|
type RequestFrame,
|
|
@@ -192,9 +193,9 @@ export async function seedMutations(
|
|
|
192
193
|
partition: target.partition,
|
|
193
194
|
actorId: target.actorId,
|
|
194
195
|
};
|
|
195
|
-
const acquisition =
|
|
196
|
+
const acquisition = decodeResponse(
|
|
196
197
|
await handleSyncRequest(
|
|
197
|
-
|
|
198
|
+
encodeRequest({
|
|
198
199
|
wireVersion: PROTOCOL_WIRE_VERSION,
|
|
199
200
|
msgKind: 'request',
|
|
200
201
|
frames: [
|
|
@@ -209,6 +210,9 @@ export async function seedMutations(
|
|
|
209
210
|
limitSnapshotRows: 0,
|
|
210
211
|
maxSnapshotPages: 0,
|
|
211
212
|
accept: 0b0011,
|
|
213
|
+
cursor: -1,
|
|
214
|
+
registrationDigest: registrationDigest([]),
|
|
215
|
+
echoDigest: new Uint8Array(),
|
|
212
216
|
},
|
|
213
217
|
],
|
|
214
218
|
}),
|
|
@@ -216,10 +220,7 @@ export async function seedMutations(
|
|
|
216
220
|
),
|
|
217
221
|
);
|
|
218
222
|
const acquisitionHeader = acquisition.frames[0];
|
|
219
|
-
if (
|
|
220
|
-
acquisition.msgKind !== 'response' ||
|
|
221
|
-
acquisitionHeader?.type !== 'RESP_HEADER'
|
|
222
|
-
) {
|
|
223
|
+
if (acquisitionHeader?.type !== 'RESP_HEADER') {
|
|
223
224
|
throw new SyncError(
|
|
224
225
|
'sync.invalid_request',
|
|
225
226
|
'seedMutations: epoch acquisition returned an invalid response',
|
|
@@ -232,15 +233,8 @@ export async function seedMutations(
|
|
|
232
233
|
schemaVersion: config.schema.version,
|
|
233
234
|
logEpoch: acquisitionHeader.logEpoch,
|
|
234
235
|
},
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
{
|
|
238
|
-
type: 'PULL_HEADER',
|
|
239
|
-
limitCommits: 0,
|
|
240
|
-
limitSnapshotRows: 0,
|
|
241
|
-
maxSnapshotPages: 0,
|
|
242
|
-
accept: 0b0011,
|
|
243
|
-
},
|
|
236
|
+
// §6.10: a database-less producer sends no outboxId and clientSeq 0.
|
|
237
|
+
{ type: 'PUSH_COMMIT', clientCommitId, clientSeq: 0, operations },
|
|
244
238
|
];
|
|
245
239
|
type SeedTerminalEvent = Extract<
|
|
246
240
|
SyncularServerEvent,
|
|
@@ -259,7 +253,7 @@ export async function seedMutations(
|
|
|
259
253
|
},
|
|
260
254
|
};
|
|
261
255
|
const response = await handleSyncRequest(
|
|
262
|
-
|
|
256
|
+
encodeRequest({
|
|
263
257
|
wireVersion: PROTOCOL_WIRE_VERSION,
|
|
264
258
|
msgKind: 'request',
|
|
265
259
|
frames,
|
|
@@ -274,7 +268,7 @@ export async function seedMutations(
|
|
|
274
268
|
);
|
|
275
269
|
|
|
276
270
|
// Fail loud: surface the first rejected/failed operation.
|
|
277
|
-
const message =
|
|
271
|
+
const message = decodeResponse(response);
|
|
278
272
|
const result = message.frames.find(
|
|
279
273
|
(frame): frame is PushResultFrame =>
|
|
280
274
|
frame.type === 'PUSH_RESULT' && frame.clientCommitId === clientCommitId,
|
package/src/segment-download.ts
CHANGED
|
@@ -28,7 +28,10 @@ import {
|
|
|
28
28
|
|
|
29
29
|
export interface SegmentDownloadRequest {
|
|
30
30
|
readonly segmentId: string;
|
|
31
|
-
/**
|
|
31
|
+
/**
|
|
32
|
+
* The `X-Syncular-Scopes` header: canonical JSON of the owning unit's scope
|
|
33
|
+
* map, each value as a one-element list (§5.5).
|
|
34
|
+
*/
|
|
32
35
|
readonly scopesHeader: string;
|
|
33
36
|
}
|
|
34
37
|
|
|
@@ -60,13 +63,15 @@ function parseScopesHeader(header: string): ScopeMap {
|
|
|
60
63
|
}
|
|
61
64
|
const scopes: ScopeMap = {};
|
|
62
65
|
for (const [key, value] of Object.entries(parsed)) {
|
|
66
|
+
// §5.5: the owning unit's scope map, each value as a one-element list.
|
|
63
67
|
if (
|
|
64
68
|
!Array.isArray(value) ||
|
|
65
|
-
value.
|
|
69
|
+
value.length !== 1 ||
|
|
70
|
+
typeof value[0] !== 'string'
|
|
66
71
|
) {
|
|
67
72
|
throw syncError(
|
|
68
73
|
'sync.invalid_request',
|
|
69
|
-
'X-Syncular-Scopes values must be string lists (§
|
|
74
|
+
'X-Syncular-Scopes values must be one-element string lists (§5.5)',
|
|
70
75
|
);
|
|
71
76
|
}
|
|
72
77
|
scopes[key] = value as string[];
|