@syncular/server 0.59.0 → 0.60.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -16
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +42 -40
- package/dist/context.d.ts +6 -8
- package/dist/context.js +1 -3
- package/dist/d1-storage.js +42 -24
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +35 -17
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +486 -223
- package/dist/local-authority.js +7 -9
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +249 -168
- package/dist/pull.d.ts +68 -53
- package/dist/pull.js +258 -287
- package/dist/push.d.ts +12 -3
- package/dist/push.js +95 -32
- package/dist/realtime.d.ts +17 -11
- package/dist/realtime.js +65 -90
- package/dist/relational-rows.d.ts +0 -57
- package/dist/relational-rows.js +4 -100
- package/dist/seed.js +10 -15
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +4 -2
- package/dist/sqlite-dialect.d.ts +36 -59
- package/dist/sqlite-dialect.js +255 -159
- package/dist/sqlite-storage.js +24 -20
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +156 -54
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +87 -59
- package/src/context.ts +7 -11
- package/src/d1-storage.ts +49 -33
- package/src/errors.ts +8 -0
- package/src/events.ts +36 -17
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +628 -286
- package/src/local-authority.ts +10 -11
- package/src/operations.ts +44 -42
- package/src/postgres-storage.ts +336 -226
- package/src/pull.ts +394 -414
- package/src/push.ts +116 -21
- package/src/realtime.ts +85 -129
- package/src/relational-rows.ts +4 -107
- package/src/seed.ts +13 -19
- package/src/segment-download.ts +8 -3
- package/src/sqlite-dialect.ts +363 -204
- package/src/sqlite-storage.ts +45 -25
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +185 -58
package/dist/realtime.js
CHANGED
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
* queues the ack's cursor write without awaiting it, so `drain()` is the
|
|
8
8
|
* awaitable boundary a host uses before closing its storage or ending a
|
|
9
9
|
* hibernatable event (§8.2). Initial
|
|
10
|
-
*
|
|
11
|
-
*
|
|
10
|
+
* registration comes from the live units of the client record (§8.1); a sync
|
|
11
|
+
* round completed on the
|
|
12
12
|
* connection replaces it at round end (§8.7). Deltas are complete SSP2
|
|
13
13
|
* response messages pushed as `0x00`-tagged binary (§8.2/§8.7); sync
|
|
14
14
|
* rounds ride the same socket as `0x01`-tagged byte-stream chunks driven
|
|
@@ -16,15 +16,16 @@
|
|
|
16
16
|
* one handler, two framings); the only JSON data-plane server event is
|
|
17
17
|
* the `sync` wake-up (§8.3).
|
|
18
18
|
*/
|
|
19
|
-
import { DecodeError,
|
|
19
|
+
import { DecodeError, decodeRequest, encodeResponse, encodePresenceError, encodePresenceFanout, isSupportedProtocolWireVersion, MessageStreamScanner, PROTOCOL_WIRE_VERSION, REALTIME_TAG_DELTA, REALTIME_TAG_ROUND, unitId, } from '@syncular/core';
|
|
20
20
|
import { limitsOf, REMOTE_COMMAND_CLIENT_ID_PREFIX, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
|
|
21
21
|
import { adapterSyncError, reportError, SyncError, syncError, } from './errors.js';
|
|
22
22
|
import { emitEvent } from './events.js';
|
|
23
23
|
import { composeEvents } from './events-ring.js';
|
|
24
24
|
import { createSyncResponseStream } from './handler.js';
|
|
25
|
+
import { commitFrame, coveredTables, planUnit } from './pull.js';
|
|
25
26
|
import { compileSchema, } from './schema.js';
|
|
26
|
-
import { schemaWindowOf
|
|
27
|
-
import {
|
|
27
|
+
import { schemaWindowOf } from './schema-window.js';
|
|
28
|
+
import { matchesEffective } from './scopes.js';
|
|
28
29
|
const DEFAULT_MAX_DELTA_BYTES = 1024 * 1024;
|
|
29
30
|
const DEFAULT_MAX_PRESENCE_BYTES = 8192;
|
|
30
31
|
const DEFAULT_MAX_UNACKED_DELTAS = 256;
|
|
@@ -497,9 +498,8 @@ export class RealtimeSession {
|
|
|
497
498
|
try {
|
|
498
499
|
// §8.7: the round's clientId must match the connection's —
|
|
499
500
|
// registration identity would otherwise be ambiguous.
|
|
500
|
-
const
|
|
501
|
-
|
|
502
|
-
if (decoded.msgKind !== 'request' || header?.type !== 'REQ_HEADER') {
|
|
501
|
+
const header = decodeRequest(requestBytes).frames[0];
|
|
502
|
+
if (header?.type !== 'REQ_HEADER') {
|
|
503
503
|
throw syncError('sync.invalid_request', 'expected a request message');
|
|
504
504
|
}
|
|
505
505
|
if (header.clientId !== this.clientId) {
|
|
@@ -610,8 +610,9 @@ export class RealtimeSession {
|
|
|
610
610
|
* failed, because that round applied nothing.
|
|
611
611
|
*/
|
|
612
612
|
/**
|
|
613
|
-
* A pull outside this connection changed the client's
|
|
614
|
-
* round in flight on the connection reloads
|
|
613
|
+
* A pull outside this connection changed the client's registration list
|
|
614
|
+
* or live set. A round in flight on the connection reloads it at its end
|
|
615
|
+
* anyway.
|
|
615
616
|
*/
|
|
616
617
|
async refreshAfterPull() {
|
|
617
618
|
if (this.#activeRound !== undefined)
|
|
@@ -713,26 +714,10 @@ export class RealtimeSession {
|
|
|
713
714
|
deliverCommit(commit) {
|
|
714
715
|
const contiguous = commit.commitSeq === this.lastKnownSeq + 1;
|
|
715
716
|
this.lastKnownSeq = Math.max(this.lastKnownSeq, commit.commitSeq);
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
matchesEffective(change.scopes, registration.effective))
|
|
721
|
-
.map((change) => ({
|
|
722
|
-
tableIndex: 0,
|
|
723
|
-
rowId: change.rowId,
|
|
724
|
-
op: change.op,
|
|
725
|
-
...(change.rowVersion !== undefined
|
|
726
|
-
? { rowVersion: change.rowVersion }
|
|
727
|
-
: {}),
|
|
728
|
-
scopes: change.scopes,
|
|
729
|
-
...(change.payload !== undefined
|
|
730
|
-
? { row: this.#hub.projectPayload(registration, change.payload) }
|
|
731
|
-
: {}),
|
|
732
|
-
}));
|
|
733
|
-
if (changes.length > 0)
|
|
734
|
-
sections.push({ registration, changes });
|
|
735
|
-
}
|
|
717
|
+
// §8.2: one COMMIT holding every change that matches a registration of
|
|
718
|
+
// the connection, once each, in change order.
|
|
719
|
+
const changes = commit.changes.filter((change) => this.registrations.some((registration) => change.table === registration.table &&
|
|
720
|
+
matchesEffective(change.scopes, registration.effective)));
|
|
736
721
|
if (!contiguous) {
|
|
737
722
|
// Pushes allocate commitSeq under the partition lock but notify after
|
|
738
723
|
// commit, so racing notifications may arrive out of order. A gap,
|
|
@@ -758,7 +743,7 @@ export class RealtimeSession {
|
|
|
758
743
|
}
|
|
759
744
|
this.#deferredRegistrations = registrations;
|
|
760
745
|
}
|
|
761
|
-
if (
|
|
746
|
+
if (changes.length === 0)
|
|
762
747
|
return;
|
|
763
748
|
if (this.wakePending) {
|
|
764
749
|
// §8.2: deltas must be cursor-contiguous; while the client is behind
|
|
@@ -766,36 +751,19 @@ export class RealtimeSession {
|
|
|
766
751
|
this.sendWake('catchup-required');
|
|
767
752
|
return;
|
|
768
753
|
}
|
|
769
|
-
const
|
|
770
|
-
{
|
|
771
|
-
type: 'RESP_HEADER',
|
|
772
|
-
logEpoch: this.logEpoch,
|
|
773
|
-
resetRequired: false,
|
|
774
|
-
},
|
|
775
|
-
];
|
|
776
|
-
for (const section of sections) {
|
|
777
|
-
frames.push({
|
|
778
|
-
type: 'SUB_START',
|
|
779
|
-
id: section.registration.id,
|
|
780
|
-
status: 'active',
|
|
781
|
-
reasonCode: '',
|
|
782
|
-
effectiveScopes: section.registration.effective,
|
|
783
|
-
bootstrap: false,
|
|
784
|
-
});
|
|
785
|
-
frames.push({
|
|
786
|
-
type: 'COMMIT',
|
|
787
|
-
commitSeq: commit.commitSeq,
|
|
788
|
-
createdAtMs: commit.createdAtMs,
|
|
789
|
-
actorId: commit.actorId,
|
|
790
|
-
tables: [section.registration.table],
|
|
791
|
-
changes: section.changes,
|
|
792
|
-
});
|
|
793
|
-
frames.push({ type: 'SUB_END', nextCursor: commit.commitSeq });
|
|
794
|
-
}
|
|
795
|
-
const bytes = encodeMessage({
|
|
754
|
+
const bytes = encodeResponse({
|
|
796
755
|
wireVersion: this.wireVersion,
|
|
797
756
|
msgKind: 'response',
|
|
798
|
-
frames
|
|
757
|
+
frames: [
|
|
758
|
+
{ type: 'RESP_HEADER', logEpoch: this.logEpoch, resetRequired: false },
|
|
759
|
+
{ type: 'LIVE_START', status: 'active', reasonCode: '' },
|
|
760
|
+
commitFrame({ ...commit, changes }, this.#hub.schema, this.registrations[0].schema),
|
|
761
|
+
{
|
|
762
|
+
type: 'LIVE_END',
|
|
763
|
+
throughSeq: commit.commitSeq,
|
|
764
|
+
echoDigest: new Uint8Array(),
|
|
765
|
+
},
|
|
766
|
+
],
|
|
799
767
|
});
|
|
800
768
|
if (bytes.length > this.#maxDeltaBytes) {
|
|
801
769
|
this.sendWake('delta-too-large');
|
|
@@ -805,7 +773,7 @@ export class RealtimeSession {
|
|
|
805
773
|
const tagged = new Uint8Array(bytes.length + 1);
|
|
806
774
|
tagged[0] = REALTIME_TAG_DELTA;
|
|
807
775
|
tagged.set(bytes, 1);
|
|
808
|
-
const changeCount =
|
|
776
|
+
const changeCount = changes.length;
|
|
809
777
|
if (this.#roundsRefreshing > 0) {
|
|
810
778
|
if (this.#deferredBytes + tagged.length > this.#maxDeltaBytes) {
|
|
811
779
|
this.sendWake('catchup-required');
|
|
@@ -871,8 +839,9 @@ export class RealtimeHub {
|
|
|
871
839
|
schemaWindowOf(config.schema, config.schemaWindow);
|
|
872
840
|
this.limits = limitsOf(config);
|
|
873
841
|
}
|
|
874
|
-
|
|
875
|
-
|
|
842
|
+
/** The current compiled schema: the storage codec of fanned-out changes. */
|
|
843
|
+
get schema() {
|
|
844
|
+
return this.#schema;
|
|
876
845
|
}
|
|
877
846
|
/**
|
|
878
847
|
* Compose `sink` into the hub's `events`, so socket rounds and sessions
|
|
@@ -938,7 +907,7 @@ export class RealtimeHub {
|
|
|
938
907
|
}
|
|
939
908
|
}
|
|
940
909
|
/**
|
|
941
|
-
* Resolve the client record's
|
|
910
|
+
* Resolve the client record's live units into per-connection
|
|
942
911
|
* registrations (§8.1 at upgrade; §8.7 at socket-round end).
|
|
943
912
|
*/
|
|
944
913
|
async loadRegistrations(partition, actorId, clientId, record) {
|
|
@@ -970,29 +939,35 @@ export class RealtimeHub {
|
|
|
970
939
|
}
|
|
971
940
|
resolved = { ok: false };
|
|
972
941
|
}
|
|
942
|
+
// §8.1: the stored list minus the catching-up units, each live unit's
|
|
943
|
+
// active pairs under the schema version of the client's latest round.
|
|
944
|
+
const schema = schemaWindowOf(this.#config.schema, this.#config.schemaWindow).find((candidate) => candidate.version === record?.schemaVersion);
|
|
973
945
|
const registrations = [];
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
: window.find((schema) => schema.version === subscription.schemaVersion);
|
|
980
|
-
const table = selected?.tables.get(subscription.table);
|
|
981
|
-
if (table === undefined)
|
|
982
|
-
continue;
|
|
983
|
-
const keysValid = Object.keys(subscription.scopes).every((key) => table.declaredVariables.has(key));
|
|
984
|
-
if (!keysValid)
|
|
946
|
+
if (schema === undefined)
|
|
947
|
+
return registrations;
|
|
948
|
+
for (const scopes of record?.registration?.units ?? []) {
|
|
949
|
+
const id = unitId(scopes);
|
|
950
|
+
if (record?.catchingUp.some((unit) => unit.id === id))
|
|
985
951
|
continue;
|
|
986
|
-
|
|
987
|
-
|
|
952
|
+
let tables;
|
|
953
|
+
try {
|
|
954
|
+
tables = coveredTables(schema, scopes);
|
|
955
|
+
}
|
|
956
|
+
catch {
|
|
957
|
+
// A unit the client's schema version no longer serves registers
|
|
958
|
+
// nothing; the client's next pull fails loud on it (§4.1).
|
|
988
959
|
continue;
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
960
|
+
}
|
|
961
|
+
const unit = planUnit(scopes, tables, resolved);
|
|
962
|
+
for (const [table, effective] of unit.pairs) {
|
|
963
|
+
registrations.push({
|
|
964
|
+
schema,
|
|
965
|
+
id: unit.id,
|
|
966
|
+
table,
|
|
967
|
+
schemaVersion: schema.version,
|
|
968
|
+
effective,
|
|
969
|
+
});
|
|
970
|
+
}
|
|
996
971
|
}
|
|
997
972
|
return registrations;
|
|
998
973
|
}
|
|
@@ -1030,8 +1005,8 @@ export class RealtimeHub {
|
|
|
1030
1005
|
return this.requestContextFor(session);
|
|
1031
1006
|
}
|
|
1032
1007
|
/**
|
|
1033
|
-
* Register a connected socket (§8.1): load the client's
|
|
1034
|
-
*
|
|
1008
|
+
* Register a connected socket (§8.1): load the client record's live
|
|
1009
|
+
* units, resolve their effective scopes, send `hello`.
|
|
1035
1010
|
*/
|
|
1036
1011
|
connect(options) {
|
|
1037
1012
|
return this.#open(options, undefined);
|
|
@@ -1123,10 +1098,9 @@ export class RealtimeHub {
|
|
|
1123
1098
|
clientId: options.clientId,
|
|
1124
1099
|
cursor,
|
|
1125
1100
|
latestCursor: latestSeq,
|
|
1126
|
-
requiresSync: record === undefined ||
|
|
1101
|
+
requiresSync: record?.registration === undefined ||
|
|
1127
1102
|
cursor < latestSeq ||
|
|
1128
|
-
(this.#config.schemaWindow
|
|
1129
|
-
record.subscriptions.some((subscription) => !schemaWindowOf(this.#config.schema, this.#config.schemaWindow).some((schema) => schema.version === subscription.schemaVersion))),
|
|
1103
|
+
!schemaWindowOf(this.#config.schema, this.#config.schemaWindow).some((schema) => schema.version === record.schemaVersion),
|
|
1130
1104
|
timestamp: clock(),
|
|
1131
1105
|
},
|
|
1132
1106
|
}));
|
|
@@ -1182,7 +1156,8 @@ export class RealtimeHub {
|
|
|
1182
1156
|
}
|
|
1183
1157
|
}
|
|
1184
1158
|
/**
|
|
1185
|
-
* RealtimeNotifier: a pull changed a client's
|
|
1159
|
+
* RealtimeNotifier: a pull changed a client's registration list or live
|
|
1160
|
+
* set (§8.1).
|
|
1186
1161
|
* Its sockets reload their registrations with the round-end semantics, so
|
|
1187
1162
|
* an unreadable record keeps the previous registrations.
|
|
1188
1163
|
*/
|
|
@@ -1255,7 +1230,7 @@ export function createRealtimeHub(config) {
|
|
|
1255
1230
|
* request-level failure becomes a minimal RESP_HEADER/ERROR/END
|
|
1256
1231
|
* response message delivered as the round's response stream. */
|
|
1257
1232
|
function errorResponseBytes(error, wireVersion, logEpoch) {
|
|
1258
|
-
return
|
|
1233
|
+
return encodeResponse({
|
|
1259
1234
|
wireVersion,
|
|
1260
1235
|
msgKind: 'response',
|
|
1261
1236
|
frames: [
|
|
@@ -230,36 +230,6 @@ export declare function indexRowPageStatement(table: CompiledTable, index: Index
|
|
|
230
230
|
* cap is unreachable through scope values.
|
|
231
231
|
*/
|
|
232
232
|
export declare function scanRowPageSql(table: CompiledTable, valueCount: number, dialect: RelationalDialect): string;
|
|
233
|
-
/**
|
|
234
|
-
* One-round-trip page read for `readCommitWindow` (sync_* tables only; it
|
|
235
|
-
* lives here beside `scanRowPageSql` because it is the same join-the-index
|
|
236
|
-
* posture, dialect-parameterized the same way): candidates from the inverted
|
|
237
|
-
* change-scope index (ordered + LIMITed at the covering `sync_change_scopes`
|
|
238
|
-
* PK — exactly the old candidate query, so the index-first invariant of
|
|
239
|
-
* `CommitWindowQuery` is unchanged) LEFT JOINed to the commit metadata and
|
|
240
|
-
* to the commit's changes for the table, so a whole window page arrives in
|
|
241
|
-
* ONE statement instead of 1 + 2×candidates round trips. This is the
|
|
242
|
-
* incremental-pull hot path — it runs on every sync round; before the join a
|
|
243
|
-
* 500-candidate catch-up window on Postgres paid ~1000 network round trips.
|
|
244
|
-
*
|
|
245
|
-
* LEFT JOIN (not INNER): a candidate whose commit vanished (scope-index
|
|
246
|
-
* entry without a commit/changes row) must still reach the caller — its
|
|
247
|
-
* `commit_seq` advances the window cursor — so it comes back with NULL
|
|
248
|
-
* meta/change columns rather than disappearing from the page.
|
|
249
|
-
*
|
|
250
|
-
* Rows arrive ordered (commit_seq, idx): oldest-first commits, each commit's
|
|
251
|
-
* change rows consecutive in `idx` order — the caller groups consecutive
|
|
252
|
-
* rows and verifies the full multi-variable scope match in JS.
|
|
253
|
-
*
|
|
254
|
-
* Bind order:
|
|
255
|
-
* - postgres: [partition, tbl, var, ...values, afterSeq, throughSeq,
|
|
256
|
-
* limit] (the joins reuse `$1`/`$2`);
|
|
257
|
-
* - sqlite: [partition, tbl, var, values as JSON array text, afterSeq,
|
|
258
|
-
* throughSeq, limit, partition, partition, tbl] (positional `?`;
|
|
259
|
-
* partition/tbl bind again for the joins). The values expand through
|
|
260
|
-
* `json_each` like `scanRowPageSql`.
|
|
261
|
-
*/
|
|
262
|
-
export declare function commitWindowPageSql(valueCount: number, dialect: RelationalDialect): string;
|
|
263
233
|
/**
|
|
264
234
|
* PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
|
|
265
235
|
* page of every subscription scanning any table in a pull. `$2` is a JSON
|
|
@@ -275,17 +245,6 @@ export declare function commitWindowPageSql(valueCount: number, dialect: Relatio
|
|
|
275
245
|
* parameter (Bun.sql) cannot turn the array into a JSON string.
|
|
276
246
|
*/
|
|
277
247
|
export declare function scanRowBatchSql(tables: readonly CompiledTable[]): string;
|
|
278
|
-
/**
|
|
279
|
-
* PostgreSQL set-based form of `commitWindowPageSql`: one statement answers
|
|
280
|
-
* the window page of every subscription in a pull, across all tables. `$2`
|
|
281
|
-
* is a JSON array of `{i, tbl, var, vals, after, through, lim}` records, each
|
|
282
|
-
* running the same index-first candidate subquery through a LATERAL join.
|
|
283
|
-
* Rows arrive ordered by (i, commit_seq, idx).
|
|
284
|
-
*
|
|
285
|
-
* Params: [partition, specs (JSON text)], the specs bound as text like
|
|
286
|
-
* `scanRowBatchSql`.
|
|
287
|
-
*/
|
|
288
|
-
export declare const COMMIT_WINDOW_BATCH_SQL = "SELECT s.i AS sub, c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,\n ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload\n FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after bigint, through bigint, lim int)\n CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes\n WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)\n AND commit_seq>s.after AND commit_seq<=s.through\n ORDER BY commit_seq LIMIT s.lim) c\n LEFT JOIN sync_commits m\n ON m.partition=$1 AND m.commit_seq=c.commit_seq\n LEFT JOIN sync_changes ch\n ON ch.partition=$1 AND ch.commit_seq=c.commit_seq AND ch.tbl=s.tbl\n ORDER BY s.i, c.commit_seq, ch.idx";
|
|
289
248
|
/**
|
|
290
249
|
* PostgreSQL push read-ahead across every table a commit targets, in one
|
|
291
250
|
* statement: each requested row id with its stored row (NULL columns when
|
|
@@ -296,22 +255,6 @@ export declare function prefetchRowsBatchSql(tables: readonly CompiledTable[], p
|
|
|
296
255
|
readonly partition: string;
|
|
297
256
|
readonly specs: string;
|
|
298
257
|
}): string;
|
|
299
|
-
/**
|
|
300
|
-
* SQLite/D1: the newest `commit_seq` in the inverted change-scope index for
|
|
301
|
-
* one variable's values, as one index-only max per value (a `(partition,
|
|
302
|
-
* tbl, var, value)` prefix), so the cost is independent of the table's
|
|
303
|
-
* history. The values arrive as one JSON array, so the statement has no
|
|
304
|
-
* compound SELECT and four bound parameters for any value count (D1 caps a
|
|
305
|
-
* compound SELECT at 5 terms and a statement at 100 parameters).
|
|
306
|
-
* Params: [partition, tbl, var, values as JSON array text].
|
|
307
|
-
*/
|
|
308
|
-
export declare const LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL = "SELECT max((SELECT max(commit_seq) FROM sync_change_scopes\n WHERE partition=? AND tbl=? AND var=? AND value=v.value)) AS seq\n FROM json_each(?) AS v";
|
|
309
|
-
/**
|
|
310
|
-
* PostgreSQL set-based form of `LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL` for the
|
|
311
|
-
* subscriptions of a pull, across all tables. `$2` is a JSON array of
|
|
312
|
-
* `{i, tbl, var, vals}` records. Params: [partition, specs (JSON text)].
|
|
313
|
-
*/
|
|
314
|
-
export declare const LATEST_SCOPED_CHANGE_BATCH_SQL = "SELECT s.i AS sub,\n (SELECT max(latest.seq) FROM (\n SELECT (SELECT max(commit_seq) FROM sync_change_scopes\n WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=v.value) AS seq\n FROM jsonb_array_elements_text(s.vals) AS v(value)) AS latest) AS seq\n FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals jsonb)\n ORDER BY s.i";
|
|
315
258
|
/**
|
|
316
259
|
* SELECT the rows that reference one blob with their stored scope maps, in
|
|
317
260
|
* one statement for any number of references: the by-blob index range
|
package/dist/relational-rows.js
CHANGED
|
@@ -573,57 +573,6 @@ export function scanRowPageSql(table, valueCount, dialect) {
|
|
|
573
573
|
ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${joinPartition} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
|
|
574
574
|
ORDER BY c.row_id`;
|
|
575
575
|
}
|
|
576
|
-
/**
|
|
577
|
-
* One-round-trip page read for `readCommitWindow` (sync_* tables only; it
|
|
578
|
-
* lives here beside `scanRowPageSql` because it is the same join-the-index
|
|
579
|
-
* posture, dialect-parameterized the same way): candidates from the inverted
|
|
580
|
-
* change-scope index (ordered + LIMITed at the covering `sync_change_scopes`
|
|
581
|
-
* PK — exactly the old candidate query, so the index-first invariant of
|
|
582
|
-
* `CommitWindowQuery` is unchanged) LEFT JOINed to the commit metadata and
|
|
583
|
-
* to the commit's changes for the table, so a whole window page arrives in
|
|
584
|
-
* ONE statement instead of 1 + 2×candidates round trips. This is the
|
|
585
|
-
* incremental-pull hot path — it runs on every sync round; before the join a
|
|
586
|
-
* 500-candidate catch-up window on Postgres paid ~1000 network round trips.
|
|
587
|
-
*
|
|
588
|
-
* LEFT JOIN (not INNER): a candidate whose commit vanished (scope-index
|
|
589
|
-
* entry without a commit/changes row) must still reach the caller — its
|
|
590
|
-
* `commit_seq` advances the window cursor — so it comes back with NULL
|
|
591
|
-
* meta/change columns rather than disappearing from the page.
|
|
592
|
-
*
|
|
593
|
-
* Rows arrive ordered (commit_seq, idx): oldest-first commits, each commit's
|
|
594
|
-
* change rows consecutive in `idx` order — the caller groups consecutive
|
|
595
|
-
* rows and verifies the full multi-variable scope match in JS.
|
|
596
|
-
*
|
|
597
|
-
* Bind order:
|
|
598
|
-
* - postgres: [partition, tbl, var, ...values, afterSeq, throughSeq,
|
|
599
|
-
* limit] (the joins reuse `$1`/`$2`);
|
|
600
|
-
* - sqlite: [partition, tbl, var, values as JSON array text, afterSeq,
|
|
601
|
-
* throughSeq, limit, partition, partition, tbl] (positional `?`;
|
|
602
|
-
* partition/tbl bind again for the joins). The values expand through
|
|
603
|
-
* `json_each` like `scanRowPageSql`.
|
|
604
|
-
*/
|
|
605
|
-
export function commitWindowPageSql(valueCount, dialect) {
|
|
606
|
-
const p = (n) => (dialect === 'sqlite' ? '?' : `$${n}`);
|
|
607
|
-
const values = dialect === 'sqlite'
|
|
608
|
-
? 'SELECT value FROM json_each(?)'
|
|
609
|
-
: Array.from({ length: valueCount }, (_, i) => p(4 + i)).join(',');
|
|
610
|
-
const after = p(4 + valueCount);
|
|
611
|
-
const through = p(5 + valueCount);
|
|
612
|
-
const limit = p(6 + valueCount);
|
|
613
|
-
const joinPartition = dialect === 'sqlite' ? '?' : '$1';
|
|
614
|
-
const joinTbl = dialect === 'sqlite' ? '?' : '$2';
|
|
615
|
-
return `SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
616
|
-
ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
617
|
-
FROM (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
618
|
-
WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values})
|
|
619
|
-
AND commit_seq>${after} AND commit_seq<=${through}
|
|
620
|
-
ORDER BY commit_seq LIMIT ${limit}) c
|
|
621
|
-
LEFT JOIN sync_commits m
|
|
622
|
-
ON m.partition=${joinPartition} AND m.commit_seq=c.commit_seq
|
|
623
|
-
LEFT JOIN sync_changes ch
|
|
624
|
-
ON ch.partition=${joinPartition} AND ch.commit_seq=c.commit_seq AND ch.tbl=${joinTbl}
|
|
625
|
-
ORDER BY c.commit_seq, ch.idx`;
|
|
626
|
-
}
|
|
627
576
|
/**
|
|
628
577
|
* PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
|
|
629
578
|
* page of every subscription scanning any table in a pull. `$2` is a JSON
|
|
@@ -661,28 +610,6 @@ export function scanRowBatchSql(tables) {
|
|
|
661
610
|
${projection}
|
|
662
611
|
ORDER BY sub, row_id`;
|
|
663
612
|
}
|
|
664
|
-
/**
|
|
665
|
-
* PostgreSQL set-based form of `commitWindowPageSql`: one statement answers
|
|
666
|
-
* the window page of every subscription in a pull, across all tables. `$2`
|
|
667
|
-
* is a JSON array of `{i, tbl, var, vals, after, through, lim}` records, each
|
|
668
|
-
* running the same index-first candidate subquery through a LATERAL join.
|
|
669
|
-
* Rows arrive ordered by (i, commit_seq, idx).
|
|
670
|
-
*
|
|
671
|
-
* Params: [partition, specs (JSON text)], the specs bound as text like
|
|
672
|
-
* `scanRowBatchSql`.
|
|
673
|
-
*/
|
|
674
|
-
export const COMMIT_WINDOW_BATCH_SQL = `SELECT s.i AS sub, c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
675
|
-
ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
676
|
-
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after bigint, through bigint, lim int)
|
|
677
|
-
CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
678
|
-
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)
|
|
679
|
-
AND commit_seq>s.after AND commit_seq<=s.through
|
|
680
|
-
ORDER BY commit_seq LIMIT s.lim) c
|
|
681
|
-
LEFT JOIN sync_commits m
|
|
682
|
-
ON m.partition=$1 AND m.commit_seq=c.commit_seq
|
|
683
|
-
LEFT JOIN sync_changes ch
|
|
684
|
-
ON ch.partition=$1 AND ch.commit_seq=c.commit_seq AND ch.tbl=s.tbl
|
|
685
|
-
ORDER BY s.i, c.commit_seq, ch.idx`;
|
|
686
613
|
/**
|
|
687
614
|
* PostgreSQL push read-ahead across every table a commit targets, in one
|
|
688
615
|
* statement: each requested row id with its stored row (NULL columns when
|
|
@@ -714,30 +641,6 @@ export function prefetchRowsBatchSql(tables, params = {
|
|
|
714
641
|
)
|
|
715
642
|
${projection}`;
|
|
716
643
|
}
|
|
717
|
-
/**
|
|
718
|
-
* SQLite/D1: the newest `commit_seq` in the inverted change-scope index for
|
|
719
|
-
* one variable's values, as one index-only max per value (a `(partition,
|
|
720
|
-
* tbl, var, value)` prefix), so the cost is independent of the table's
|
|
721
|
-
* history. The values arrive as one JSON array, so the statement has no
|
|
722
|
-
* compound SELECT and four bound parameters for any value count (D1 caps a
|
|
723
|
-
* compound SELECT at 5 terms and a statement at 100 parameters).
|
|
724
|
-
* Params: [partition, tbl, var, values as JSON array text].
|
|
725
|
-
*/
|
|
726
|
-
export const LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL = `SELECT max((SELECT max(commit_seq) FROM sync_change_scopes
|
|
727
|
-
WHERE partition=? AND tbl=? AND var=? AND value=v.value)) AS seq
|
|
728
|
-
FROM json_each(?) AS v`;
|
|
729
|
-
/**
|
|
730
|
-
* PostgreSQL set-based form of `LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL` for the
|
|
731
|
-
* subscriptions of a pull, across all tables. `$2` is a JSON array of
|
|
732
|
-
* `{i, tbl, var, vals}` records. Params: [partition, specs (JSON text)].
|
|
733
|
-
*/
|
|
734
|
-
export const LATEST_SCOPED_CHANGE_BATCH_SQL = `SELECT s.i AS sub,
|
|
735
|
-
(SELECT max(latest.seq) FROM (
|
|
736
|
-
SELECT (SELECT max(commit_seq) FROM sync_change_scopes
|
|
737
|
-
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=v.value) AS seq
|
|
738
|
-
FROM jsonb_array_elements_text(s.vals) AS v(value)) AS latest) AS seq
|
|
739
|
-
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals jsonb)
|
|
740
|
-
ORDER BY s.i`;
|
|
741
644
|
/**
|
|
742
645
|
* SELECT the rows that reference one blob with their stored scope maps, in
|
|
743
646
|
* one statement for any number of references: the by-blob index range
|
|
@@ -1024,9 +927,10 @@ export function serverIndexDdl(schema, existingIndexesByTable) {
|
|
|
1024
927
|
const drops = [];
|
|
1025
928
|
const creates = [];
|
|
1026
929
|
for (const table of schema.tables.values()) {
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
930
|
+
// Every table exists at the current version; a table the introspection
|
|
931
|
+
// reported no index for (Postgres lists none for a primary-key-only
|
|
932
|
+
// table) has none.
|
|
933
|
+
const existing = existingIndexesByTable.get(table.name) ?? new Map();
|
|
1030
934
|
const desired = desiredIndexes(table);
|
|
1031
935
|
for (const indexName of existing.keys()) {
|
|
1032
936
|
if (indexName.startsWith(SYNC_INDEX_PREFIX) &&
|
package/dist/seed.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* Idempotent by construction (§2.3): the commit id is stable, so re-running
|
|
10
10
|
* the seed replays the cached push result and writes nothing twice.
|
|
11
11
|
*/
|
|
12
|
-
import {
|
|
12
|
+
import { decodeResponse, encodeRequest, encodeSparseRow, PROTOCOL_WIRE_VERSION, registrationDigest, } from '@syncular/core';
|
|
13
13
|
import { SyncError } from './errors.js';
|
|
14
14
|
import { composeEvents } from './events-ring.js';
|
|
15
15
|
import { handleSyncRequest } from './handler.js';
|
|
@@ -111,7 +111,7 @@ export async function seedMutations(config, target, mutations) {
|
|
|
111
111
|
partition: target.partition,
|
|
112
112
|
actorId: target.actorId,
|
|
113
113
|
};
|
|
114
|
-
const acquisition =
|
|
114
|
+
const acquisition = decodeResponse(await handleSyncRequest(encodeRequest({
|
|
115
115
|
wireVersion: PROTOCOL_WIRE_VERSION,
|
|
116
116
|
msgKind: 'request',
|
|
117
117
|
frames: [
|
|
@@ -126,12 +126,14 @@ export async function seedMutations(config, target, mutations) {
|
|
|
126
126
|
limitSnapshotRows: 0,
|
|
127
127
|
maxSnapshotPages: 0,
|
|
128
128
|
accept: 0b0011,
|
|
129
|
+
cursor: -1,
|
|
130
|
+
registrationDigest: registrationDigest([]),
|
|
131
|
+
echoDigest: new Uint8Array(),
|
|
129
132
|
},
|
|
130
133
|
],
|
|
131
134
|
}), requestContext));
|
|
132
135
|
const acquisitionHeader = acquisition.frames[0];
|
|
133
|
-
if (
|
|
134
|
-
acquisitionHeader?.type !== 'RESP_HEADER') {
|
|
136
|
+
if (acquisitionHeader?.type !== 'RESP_HEADER') {
|
|
135
137
|
throw new SyncError('sync.invalid_request', 'seedMutations: epoch acquisition returned an invalid response');
|
|
136
138
|
}
|
|
137
139
|
const frames = [
|
|
@@ -141,15 +143,8 @@ export async function seedMutations(config, target, mutations) {
|
|
|
141
143
|
schemaVersion: config.schema.version,
|
|
142
144
|
logEpoch: acquisitionHeader.logEpoch,
|
|
143
145
|
},
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
{
|
|
147
|
-
type: 'PULL_HEADER',
|
|
148
|
-
limitCommits: 0,
|
|
149
|
-
limitSnapshotRows: 0,
|
|
150
|
-
maxSnapshotPages: 0,
|
|
151
|
-
accept: 0b0011,
|
|
152
|
-
},
|
|
146
|
+
// §6.10: a database-less producer sends no outboxId and clientSeq 0.
|
|
147
|
+
{ type: 'PUSH_COMMIT', clientCommitId, clientSeq: 0, operations },
|
|
153
148
|
];
|
|
154
149
|
let terminalEvent;
|
|
155
150
|
const capture = {
|
|
@@ -161,7 +156,7 @@ export async function seedMutations(config, target, mutations) {
|
|
|
161
156
|
}
|
|
162
157
|
},
|
|
163
158
|
};
|
|
164
|
-
const response = await handleSyncRequest(
|
|
159
|
+
const response = await handleSyncRequest(encodeRequest({
|
|
165
160
|
wireVersion: PROTOCOL_WIRE_VERSION,
|
|
166
161
|
msgKind: 'request',
|
|
167
162
|
frames,
|
|
@@ -172,7 +167,7 @@ export async function seedMutations(config, target, mutations) {
|
|
|
172
167
|
: composeEvents(config.events, capture),
|
|
173
168
|
});
|
|
174
169
|
// Fail loud: surface the first rejected/failed operation.
|
|
175
|
-
const message =
|
|
170
|
+
const message = decodeResponse(response);
|
|
176
171
|
const result = message.frames.find((frame) => frame.type === 'PUSH_RESULT' && frame.clientCommitId === clientCommitId);
|
|
177
172
|
if (result === undefined) {
|
|
178
173
|
throw new SyncError('sync.invalid_request', 'seedMutations: the sync response carried no push result');
|
|
@@ -2,7 +2,10 @@ import type { SyncRequestContext } from './context.js';
|
|
|
2
2
|
import { type SegmentRecord } from './segment-store.js';
|
|
3
3
|
export interface SegmentDownloadRequest {
|
|
4
4
|
readonly segmentId: string;
|
|
5
|
-
/**
|
|
5
|
+
/**
|
|
6
|
+
* The `X-Syncular-Scopes` header: canonical JSON of the owning unit's scope
|
|
7
|
+
* map, each value as a one-element list (§5.5).
|
|
8
|
+
*/
|
|
6
9
|
readonly scopesHeader: string;
|
|
7
10
|
}
|
|
8
11
|
export interface SegmentDownloadResult {
|
package/dist/segment-download.js
CHANGED
|
@@ -18,9 +18,11 @@ function parseScopesHeader(header) {
|
|
|
18
18
|
}
|
|
19
19
|
const scopes = {};
|
|
20
20
|
for (const [key, value] of Object.entries(parsed)) {
|
|
21
|
+
// §5.5: the owning unit's scope map, each value as a one-element list.
|
|
21
22
|
if (!Array.isArray(value) ||
|
|
22
|
-
value.
|
|
23
|
-
|
|
23
|
+
value.length !== 1 ||
|
|
24
|
+
typeof value[0] !== 'string') {
|
|
25
|
+
throw syncError('sync.invalid_request', 'X-Syncular-Scopes values must be one-element string lists (§5.5)');
|
|
24
26
|
}
|
|
25
27
|
scopes[key] = value;
|
|
26
28
|
}
|